Creating the SQLAlchemyPipeline class
This commit is contained in:
parent
eaf854c3eb
commit
0db07013ce
27
scrarls.py
27
scrarls.py
@ -84,6 +84,33 @@ class TvShowItem(scrapy.Item):
|
|||||||
download_url: scrapy.Field = scrapy.Field()
|
download_url: scrapy.Field = scrapy.Field()
|
||||||
|
|
||||||
|
|
||||||
|
class SQLAlchemyPipeline:
|
||||||
|
def __init__(self):
|
||||||
|
self.engine: Engine = create_engine('sqlite:///tvshows.db', echo=True)
|
||||||
|
Base.metadata.create_all(self.engine)
|
||||||
|
self.Session = sessionmaker(bind=self.engine)
|
||||||
|
|
||||||
|
def process_item(self, item, spider):
|
||||||
|
session = self.Session()
|
||||||
|
try:
|
||||||
|
stmt = select(TvShowDB).where(TvShowDB.post_id == item["post_id"])
|
||||||
|
show = session.scalars(stmt).first()
|
||||||
|
print(f"{show=}")
|
||||||
|
if not show:
|
||||||
|
show = TvShowDB(**item)
|
||||||
|
session.add(show)
|
||||||
|
else:
|
||||||
|
for key, value in item.items():
|
||||||
|
setattr(show, key, value)
|
||||||
|
session.commit()
|
||||||
|
except Exception as e:
|
||||||
|
session.rollback()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
class TvShow(CrawlSpider):
|
class TvShow(CrawlSpider):
|
||||||
name: str = "rlsb_tvshow"
|
name: str = "rlsb_tvshow"
|
||||||
allowed_domains: list[str] = ["rlsbb.ru"]
|
allowed_domains: list[str] = ["rlsbb.ru"]
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user