edipretoro/rlsbb_scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactoring the scraper to use our RlsbbScraperItem

Browse Source
This commit is contained in:
edipretoro 2024-12-09 10:20:20 +01:00
parent 1f4601b8cd
commit 7d4dd8edbf
1 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
rlsbb_scraper/spiders/rlsbb.py
View File

@ -1,5 +1,7 @@
import scrapy import scrapy
from rlsbb_scraper.items import RlsbbScraperItem
class RlsBBMagsSpider(scrapy.Spider): class RlsBBMagsSpider(scrapy.Spider):
name = "rlsbb_mags" name = "rlsbb_mags"
@ -21,11 +23,12 @@ class RlsBBMagsSpider(scrapy.Spider):
def parse(self, response): def parse(self, response):
for article in response.css("article"): for article in response.css("article"):
yield { item = RlsbbScraperItem()
'id': article.attrib['id'], item['article_id'] = article.attrib['id'],
'article_title': article.css('h1.entry-title > a::text').get(), item['article_title'] = article.css('h1.entry-title > a::text').get(),
'title': article.css('.entry-summary > p > strong::text').get(), item['title'] = article.css('.entry-summary > p > strong::text').get(),
'date': article.css('.entry-meta-header-before::text').getall()[1].strip(), item['date'] = article.css('.entry-meta-header-before::text').getall()[1].strip(),
'image_url': article.css('.entry-summary > p > img::attr(src)').get(), item['summary'] = article.xpath('.//div[@class="entry-summary"]/node()').extract(),
'download_url': article.css('.entry-summary > p > a[href ^= "https://rapidgator"]::attr(href)').get() item['image_url'] = article.css('.entry-summary > p > img::attr(src)').get(),
} item['download_url'] = article.css('.entry-summary > p > a[href ^= "https://rapidgator"]::attr(href)').get()
yield item