Compare commits

...

3 Commits

8 changed files with 14 additions and 14 deletions

View File

@ -6,7 +6,7 @@
import scrapy import scrapy
class RslbbScraperItem(scrapy.Item): class RlsbbScraperItem(scrapy.Item):
# define the fields for your item here like: # define the fields for your item here like:
# name = scrapy.Field() # name = scrapy.Field()
pass pass

View File

@ -9,7 +9,7 @@ from scrapy import signals
from itemadapter import is_item, ItemAdapter from itemadapter import is_item, ItemAdapter
class RslbbScraperSpiderMiddleware: class RlsbbScraperSpiderMiddleware:
# Not all methods need to be defined. If a method is not defined, # Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the # scrapy acts as if the spider middleware does not modify the
# passed objects. # passed objects.
@ -56,7 +56,7 @@ class RslbbScraperSpiderMiddleware:
spider.logger.info("Spider opened: %s" % spider.name) spider.logger.info("Spider opened: %s" % spider.name)
class RslbbScraperDownloaderMiddleware: class RlsbbScraperDownloaderMiddleware:
# Not all methods need to be defined. If a method is not defined, # Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the # scrapy acts as if the downloader middleware does not modify the
# passed objects. # passed objects.

View File

@ -8,6 +8,6 @@
from itemadapter import ItemAdapter from itemadapter import ItemAdapter
class RslbbScraperPipeline: class RlsbbScraperPipeline:
def process_item(self, item, spider): def process_item(self, item, spider):
return item return item

View File

@ -1,4 +1,4 @@
# Scrapy settings for rslbb_scraper project # Scrapy settings for rlsbb_scraper project
# #
# For simplicity, this file contains only settings considered important or # For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation: # commonly used. You can find more settings consulting the documentation:
@ -7,14 +7,14 @@
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html # https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html # https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = "rslbb_scraper" BOT_NAME = "rlsbb_scraper"
SPIDER_MODULES = ["rslbb_scraper.spiders"] SPIDER_MODULES = ["rlsbb_scraper.spiders"]
NEWSPIDER_MODULE = "rslbb_scraper.spiders" NEWSPIDER_MODULE = "rlsbb_scraper.spiders"
# Crawl responsibly by identifying yourself (and your website) on the user-agent # Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = "rslbb_scraper (+http://www.yourdomain.com)" #USER_AGENT = "rlsbb_scraper (+http://www.yourdomain.com)"
# Obey robots.txt rules # Obey robots.txt rules
ROBOTSTXT_OBEY = True ROBOTSTXT_OBEY = True
@ -45,13 +45,13 @@ ROBOTSTXT_OBEY = True
# Enable or disable spider middlewares # Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = { #SPIDER_MIDDLEWARES = {
# "rslbb_scraper.middlewares.RslbbScraperSpiderMiddleware": 543, # "rlsbb_scraper.middlewares.rlsbbScraperSpiderMiddleware": 543,
#} #}
# Enable or disable downloader middlewares # Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = { #DOWNLOADER_MIDDLEWARES = {
# "rslbb_scraper.middlewares.RslbbScraperDownloaderMiddleware": 543, # "rlsbb_scraper.middlewares.rlsbbScraperDownloaderMiddleware": 543,
#} #}
# Enable or disable extensions # Enable or disable extensions
@ -63,7 +63,7 @@ ROBOTSTXT_OBEY = True
# Configure item pipelines # Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = { #ITEM_PIPELINES = {
# "rslbb_scraper.pipelines.RslbbScraperPipeline": 300, # "rlsbb_scraper.pipelines.rlsbbScraperPipeline": 300,
#} #}
# Enable and configure the AutoThrottle extension (disabled by default) # Enable and configure the AutoThrottle extension (disabled by default)

View File

@ -4,8 +4,8 @@
# https://scrapyd.readthedocs.io/en/latest/deploy.html # https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings] [settings]
default = rslbb_scraper.settings default = rlsbb_scraper.settings
[deploy] [deploy]
#url = http://localhost:6800/ #url = http://localhost:6800/
project = rslbb_scraper project = rlsbb_scraper