Creating the SQLAlchemyPipeline class
This commit is contained in:
parent
eaf854c3eb
commit
0db07013ce
27
scrarls.py
27
scrarls.py
@ -84,6 +84,33 @@ class TvShowItem(scrapy.Item):
|
||||
download_url: scrapy.Field = scrapy.Field()
|
||||
|
||||
|
||||
class SQLAlchemyPipeline:
|
||||
def __init__(self):
|
||||
self.engine: Engine = create_engine('sqlite:///tvshows.db', echo=True)
|
||||
Base.metadata.create_all(self.engine)
|
||||
self.Session = sessionmaker(bind=self.engine)
|
||||
|
||||
def process_item(self, item, spider):
|
||||
session = self.Session()
|
||||
try:
|
||||
stmt = select(TvShowDB).where(TvShowDB.post_id == item["post_id"])
|
||||
show = session.scalars(stmt).first()
|
||||
print(f"{show=}")
|
||||
if not show:
|
||||
show = TvShowDB(**item)
|
||||
session.add(show)
|
||||
else:
|
||||
for key, value in item.items():
|
||||
setattr(show, key, value)
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
return item
|
||||
|
||||
|
||||
class TvShow(CrawlSpider):
|
||||
name: str = "rlsb_tvshow"
|
||||
allowed_domains: list[str] = ["rlsbb.ru"]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user