edipretoro/scrarls
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extracting the date from the post

Browse Source
This commit is contained in:
edipretoro 2025-12-29 21:39:51 +01:00
parent 3f6d1bfb4f
commit d099cd262d
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrarls.py
View File

@ -1,6 +1,8 @@
#!/usr/bin/env python #!/usr/bin/env python
import re
import sys import sys
from datetime import datetime
import scrapy import scrapy
from scrapy.crawler import CrawlerProcess from scrapy.crawler import CrawlerProcess
@ -37,12 +39,16 @@ class TvShow(CrawlSpider):
item['article_id'] = article.attrib['id'], item['article_id'] = article.attrib['id'],
item['article_title'] = article.css('h1.entry-title > a::text').get(), item['article_title'] = article.css('h1.entry-title > a::text').get(),
item['title'] = article.css('.entry-summary > p:nth-child(4) > strong::text').get(), item['title'] = article.css('.entry-summary > p:nth-child(4) > strong::text').get(),
item['date'] = article.css('.entry-meta-header-before::text').getall()[1].strip(), item['date'] = self.parse_date(article.css('.entry-meta-header-before::text').getall()[1].strip()),
item['summary'] = article.xpath('.//div[@class="entry-summary"]/node()').extract(), item['summary'] = article.xpath('.//div[@class="entry-summary"]/node()').extract(),
item['image_url'] = article.css('.entry-summary > p > img::attr(src)').get(), item['image_url'] = article.css('.entry-summary > p > img::attr(src)').get(),
item['download_url'] = article.css('.entry-summary > p > a[href ^= "https://rapidgator"]::attr(href)').get() item['download_url'] = article.css('.entry-summary > p > a[href ^= "https://rapidgator"]::attr(href)').get()
yield item yield item
def parse_date(self, formatted_date: str):
formatted_date = re.sub(r'(\d)(st|nd|rd|th)', r'\1', formatted_date)
return datetime.strptime(formatted_date, "Posted on %B %d, %Y at %H:%M in")
def main(): def main():
process = CrawlerProcess() process = CrawlerProcess()