Compare commits
3 Commits
4cc5438e5c
...
6fa55f572a
| Author | SHA1 | Date | |
|---|---|---|---|
| 6fa55f572a | |||
| 54f600f50d | |||
| 86607d90cd |
@ -6,7 +6,7 @@
|
|||||||
import scrapy
|
import scrapy
|
||||||
|
|
||||||
|
|
||||||
class RslbbScraperItem(scrapy.Item):
|
class RlsbbScraperItem(scrapy.Item):
|
||||||
# define the fields for your item here like:
|
# define the fields for your item here like:
|
||||||
# name = scrapy.Field()
|
# name = scrapy.Field()
|
||||||
pass
|
pass
|
||||||
@ -9,7 +9,7 @@ from scrapy import signals
|
|||||||
from itemadapter import is_item, ItemAdapter
|
from itemadapter import is_item, ItemAdapter
|
||||||
|
|
||||||
|
|
||||||
class RslbbScraperSpiderMiddleware:
|
class RlsbbScraperSpiderMiddleware:
|
||||||
# Not all methods need to be defined. If a method is not defined,
|
# Not all methods need to be defined. If a method is not defined,
|
||||||
# scrapy acts as if the spider middleware does not modify the
|
# scrapy acts as if the spider middleware does not modify the
|
||||||
# passed objects.
|
# passed objects.
|
||||||
@ -56,7 +56,7 @@ class RslbbScraperSpiderMiddleware:
|
|||||||
spider.logger.info("Spider opened: %s" % spider.name)
|
spider.logger.info("Spider opened: %s" % spider.name)
|
||||||
|
|
||||||
|
|
||||||
class RslbbScraperDownloaderMiddleware:
|
class RlsbbScraperDownloaderMiddleware:
|
||||||
# Not all methods need to be defined. If a method is not defined,
|
# Not all methods need to be defined. If a method is not defined,
|
||||||
# scrapy acts as if the downloader middleware does not modify the
|
# scrapy acts as if the downloader middleware does not modify the
|
||||||
# passed objects.
|
# passed objects.
|
||||||
@ -8,6 +8,6 @@
|
|||||||
from itemadapter import ItemAdapter
|
from itemadapter import ItemAdapter
|
||||||
|
|
||||||
|
|
||||||
class RslbbScraperPipeline:
|
class RlsbbScraperPipeline:
|
||||||
def process_item(self, item, spider):
|
def process_item(self, item, spider):
|
||||||
return item
|
return item
|
||||||
@ -1,4 +1,4 @@
|
|||||||
# Scrapy settings for rslbb_scraper project
|
# Scrapy settings for rlsbb_scraper project
|
||||||
#
|
#
|
||||||
# For simplicity, this file contains only settings considered important or
|
# For simplicity, this file contains only settings considered important or
|
||||||
# commonly used. You can find more settings consulting the documentation:
|
# commonly used. You can find more settings consulting the documentation:
|
||||||
@ -7,14 +7,14 @@
|
|||||||
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||||
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
||||||
|
|
||||||
BOT_NAME = "rslbb_scraper"
|
BOT_NAME = "rlsbb_scraper"
|
||||||
|
|
||||||
SPIDER_MODULES = ["rslbb_scraper.spiders"]
|
SPIDER_MODULES = ["rlsbb_scraper.spiders"]
|
||||||
NEWSPIDER_MODULE = "rslbb_scraper.spiders"
|
NEWSPIDER_MODULE = "rlsbb_scraper.spiders"
|
||||||
|
|
||||||
|
|
||||||
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||||
#USER_AGENT = "rslbb_scraper (+http://www.yourdomain.com)"
|
#USER_AGENT = "rlsbb_scraper (+http://www.yourdomain.com)"
|
||||||
|
|
||||||
# Obey robots.txt rules
|
# Obey robots.txt rules
|
||||||
ROBOTSTXT_OBEY = True
|
ROBOTSTXT_OBEY = True
|
||||||
@ -45,13 +45,13 @@ ROBOTSTXT_OBEY = True
|
|||||||
# Enable or disable spider middlewares
|
# Enable or disable spider middlewares
|
||||||
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
||||||
#SPIDER_MIDDLEWARES = {
|
#SPIDER_MIDDLEWARES = {
|
||||||
# "rslbb_scraper.middlewares.RslbbScraperSpiderMiddleware": 543,
|
# "rlsbb_scraper.middlewares.rlsbbScraperSpiderMiddleware": 543,
|
||||||
#}
|
#}
|
||||||
|
|
||||||
# Enable or disable downloader middlewares
|
# Enable or disable downloader middlewares
|
||||||
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||||
#DOWNLOADER_MIDDLEWARES = {
|
#DOWNLOADER_MIDDLEWARES = {
|
||||||
# "rslbb_scraper.middlewares.RslbbScraperDownloaderMiddleware": 543,
|
# "rlsbb_scraper.middlewares.rlsbbScraperDownloaderMiddleware": 543,
|
||||||
#}
|
#}
|
||||||
|
|
||||||
# Enable or disable extensions
|
# Enable or disable extensions
|
||||||
@ -63,7 +63,7 @@ ROBOTSTXT_OBEY = True
|
|||||||
# Configure item pipelines
|
# Configure item pipelines
|
||||||
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
||||||
#ITEM_PIPELINES = {
|
#ITEM_PIPELINES = {
|
||||||
# "rslbb_scraper.pipelines.RslbbScraperPipeline": 300,
|
# "rlsbb_scraper.pipelines.rlsbbScraperPipeline": 300,
|
||||||
#}
|
#}
|
||||||
|
|
||||||
# Enable and configure the AutoThrottle extension (disabled by default)
|
# Enable and configure the AutoThrottle extension (disabled by default)
|
||||||
@ -4,8 +4,8 @@
|
|||||||
# https://scrapyd.readthedocs.io/en/latest/deploy.html
|
# https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||||
|
|
||||||
[settings]
|
[settings]
|
||||||
default = rslbb_scraper.settings
|
default = rlsbb_scraper.settings
|
||||||
|
|
||||||
[deploy]
|
[deploy]
|
||||||
#url = http://localhost:6800/
|
#url = http://localhost:6800/
|
||||||
project = rslbb_scraper
|
project = rlsbb_scraper
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user