Adding a first iteration to crawl over pages from the tv-shows category

2025-12-24 23:00:32 +01:00
parent 598db18f54
commit 887a641be5
3 changed files with 585 additions and 1 deletions
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+import sys
+
+import scrapy
+from scrapy.http import Response
+from scrapy.crawler import CrawlerProcess
+from scrapy.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor
+
+
+class TvShow(CrawlSpider):
+    name = "rlsb_tvshow"
+    allowed_domains = ["rlsbb.ru"]
+    start_urls = ["https://rlsbb.ru/category/tv-shows/"]
+    custom_settings = {
+        'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36',
+        'AUTOTHROTTLE_ENABLED': True,
+        'DOWNLOAD_DELAY': 10,
+    }
+    rules = [
+        Rule(LinkExtractor(allow=r"/tv-shows/page/"), callback="parse", follow=True)
+    ]
+
+    def parse(self, response):
+        pass
+
+
+def main():
+    process = CrawlerProcess()
+    process.crawl(TvShow)
+    process.start()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())