Loading descarga_por_dia/alChile/alChile/items.py +8 −2 Changes for descarga_por_dia/alChile/alChile/items.py: 8 added lines, 2 removed lines. Original line number Diff line number Diff line Loading @@ -8,7 +8,13 @@ import scrapy class AlchileItem(scrapy.Item): class NoticiasItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() pass title = scrapy.Field() text = scrapy.Field() date = scrapy.Field() location = scrapy.Field() author = scrapy.Field() topic = scrapy.Field() url = scrapy.Field() descarga_por_dia/alChile/alChile/pipelines.py +65 −1 Changes for descarga_por_dia/alChile/alChile/pipelines.py: 65 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -5,7 +5,71 @@ # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html import json from collections import OrderedDict class JsonWriterPipeline(object): def __init__(self, filename): self.filename = filename @classmethod def from_crawler(cls, crawler): # Here you get whatever value was passed through the "filename" command line parameter settings = crawler.settings filename = settings.get('filename') # Instantiate the pipeline with the file name return cls(filename) def open_spider(self, spider): self.counter = 0 self.file = open(self.filename, 'w') self.file.write("[") def close_spider(self, spider): self.file.write("]") self.file.close() class AlchilePipeline(object): def process_item(self, item, spider): # print("this is my item", item) row = [] try: row.append(("date", item['date'])) except: pass try: row.append(("topic", item['topic'])) except: pass try: row.append(("title", item['title'])) except: pass try: row.append(("author", item['author'])) except: pass try: row.append(("location", item['location'])) except: pass try: row.append(("text", item['text'])) except: pass try: row.append(("url", item['url'])) except: pass line = OrderedDict(row) self.counter += 1 if self.counter == 1: self.file.write(json.dumps(line)) elif self.counter > 1: self.file.write(",\n" + json.dumps(line)) return item descarga_por_dia/alChile/alChile/settings.py +6 −6 Changes for descarga_por_dia/alChile/alChile/settings.py: 6 added lines, 6 removed lines. Original line number Diff line number Diff line Loading @@ -19,7 +19,7 @@ NEWSPIDER_MODULE = 'alChile.spiders' #USER_AGENT = 'alChile (+http://www.yourdomain.com)' # Obey robots.txt rules ROBOTSTXT_OBEY = True # ROBOTSTXT_OBEY = True # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32 Loading @@ -27,13 +27,13 @@ ROBOTSTXT_OBEY = True # Configure a delay for requests for the same website (default: 0) # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 DOWNLOAD_DELAY = 0.5 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default) #COOKIES_ENABLED = False COOKIES_ENABLED = False # Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False Loading Loading @@ -64,9 +64,9 @@ ROBOTSTXT_OBEY = True # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { # 'alChile.pipelines.AlchilePipeline': 300, #} ITEM_PIPELINES = { 'alChile.pipelines.JsonWriterPipeline': 300, } # Enable and configure the AutoThrottle extension (disabled by default) # See http://doc.scrapy.org/en/latest/topics/autothrottle.html Loading descarga_por_dia/alChile/alChile/settings.pyc +122 B (440 B) File changed.No diff preview for this file type. View original file View changed file descarga_por_dia/alChile/alChile/spiders/noticias.py +12 −22 Changes for descarga_por_dia/alChile/alChile/spiders/noticias.py: 12 added lines, 22 removed lines. Original line number Diff line number Diff line import scrapy, re from alChile.items import NoticiasItem """ scrapy crawl noticias -t json --nolog -o noticias.json -a year=2017 -a month=3 -a day=22 MEDIO: Al Chile, Yucatan USO: scrapy crawl noticias --nolog -s filename=2017-03-22.json -a year=2017 -a month=3 -a day=22 """ TAG_RE = re.compile(r'<[^>]+>') Loading @@ -9,16 +13,6 @@ def remove_tags(text): return TAG_RE.sub('', text) class NoticiasItem(scrapy.Item): title = scrapy.Field() text = scrapy.Field() date = scrapy.Field() location = scrapy.Field() author = scrapy.Field() topic = scrapy.Field() url = scrapy.Field() class QuotesSpider(scrapy.Spider): name = "noticias" Loading @@ -26,26 +20,22 @@ class QuotesSpider(scrapy.Spider): year = getattr(self, 'year', None) month = getattr(self, 'month', None) day = getattr(self, 'day', None) self.baseURL = 'http://alchile.com.mx/' + year + '/' + month + '/' + day yield scrapy.Request(url=self.baseURL, callback=self.parse) def parse(self, response): pagination = response.css('div.page-nav').css('a.last::attr(href)').extract() if ( len(pagination) > 0 ): pagination = pagination[0].strip('/') pages = int(pagination[pagination.rfind('/')+1:]) for page in range(0,pages): if ( page == 0 ): yield scrapy.Request(url=response.url, callback=self.parse_page, dont_filter=True) else: yield scrapy.Request(url=self.baseURL+"/page/"+str(page+1), callback=self.parse_page) pagination = response.css('div.page-nav').css('a::attr(href)').extract() if len(pagination) > 0: pagination = pagination[-2].strip('/') pages = int(pagination[pagination.rfind('/')+1:]) else: yield scrapy.Request(url=response.url, callback=self.parse_page, dont_filter=True) for page in range(1,pages): yield scrapy.Request(url=self.baseURL+"/page/"+str(page+1), callback=self.parse_page) def parse_page(self, response): Loading Loading
descarga_por_dia/alChile/alChile/items.py +8 −2 Changes for descarga_por_dia/alChile/alChile/items.py: 8 added lines, 2 removed lines. Original line number Diff line number Diff line Loading @@ -8,7 +8,13 @@ import scrapy class AlchileItem(scrapy.Item): class NoticiasItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() pass title = scrapy.Field() text = scrapy.Field() date = scrapy.Field() location = scrapy.Field() author = scrapy.Field() topic = scrapy.Field() url = scrapy.Field()
descarga_por_dia/alChile/alChile/pipelines.py +65 −1 Changes for descarga_por_dia/alChile/alChile/pipelines.py: 65 added lines, 1 removed line. Original line number Diff line number Diff line Loading @@ -5,7 +5,71 @@ # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html import json from collections import OrderedDict class JsonWriterPipeline(object): def __init__(self, filename): self.filename = filename @classmethod def from_crawler(cls, crawler): # Here you get whatever value was passed through the "filename" command line parameter settings = crawler.settings filename = settings.get('filename') # Instantiate the pipeline with the file name return cls(filename) def open_spider(self, spider): self.counter = 0 self.file = open(self.filename, 'w') self.file.write("[") def close_spider(self, spider): self.file.write("]") self.file.close() class AlchilePipeline(object): def process_item(self, item, spider): # print("this is my item", item) row = [] try: row.append(("date", item['date'])) except: pass try: row.append(("topic", item['topic'])) except: pass try: row.append(("title", item['title'])) except: pass try: row.append(("author", item['author'])) except: pass try: row.append(("location", item['location'])) except: pass try: row.append(("text", item['text'])) except: pass try: row.append(("url", item['url'])) except: pass line = OrderedDict(row) self.counter += 1 if self.counter == 1: self.file.write(json.dumps(line)) elif self.counter > 1: self.file.write(",\n" + json.dumps(line)) return item
descarga_por_dia/alChile/alChile/settings.py +6 −6 Changes for descarga_por_dia/alChile/alChile/settings.py: 6 added lines, 6 removed lines. Original line number Diff line number Diff line Loading @@ -19,7 +19,7 @@ NEWSPIDER_MODULE = 'alChile.spiders' #USER_AGENT = 'alChile (+http://www.yourdomain.com)' # Obey robots.txt rules ROBOTSTXT_OBEY = True # ROBOTSTXT_OBEY = True # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32 Loading @@ -27,13 +27,13 @@ ROBOTSTXT_OBEY = True # Configure a delay for requests for the same website (default: 0) # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 DOWNLOAD_DELAY = 0.5 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default) #COOKIES_ENABLED = False COOKIES_ENABLED = False # Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False Loading Loading @@ -64,9 +64,9 @@ ROBOTSTXT_OBEY = True # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { # 'alChile.pipelines.AlchilePipeline': 300, #} ITEM_PIPELINES = { 'alChile.pipelines.JsonWriterPipeline': 300, } # Enable and configure the AutoThrottle extension (disabled by default) # See http://doc.scrapy.org/en/latest/topics/autothrottle.html Loading
descarga_por_dia/alChile/alChile/settings.pyc +122 B (440 B) File changed.No diff preview for this file type. View original file View changed file
descarga_por_dia/alChile/alChile/spiders/noticias.py +12 −22 Changes for descarga_por_dia/alChile/alChile/spiders/noticias.py: 12 added lines, 22 removed lines. Original line number Diff line number Diff line import scrapy, re from alChile.items import NoticiasItem """ scrapy crawl noticias -t json --nolog -o noticias.json -a year=2017 -a month=3 -a day=22 MEDIO: Al Chile, Yucatan USO: scrapy crawl noticias --nolog -s filename=2017-03-22.json -a year=2017 -a month=3 -a day=22 """ TAG_RE = re.compile(r'<[^>]+>') Loading @@ -9,16 +13,6 @@ def remove_tags(text): return TAG_RE.sub('', text) class NoticiasItem(scrapy.Item): title = scrapy.Field() text = scrapy.Field() date = scrapy.Field() location = scrapy.Field() author = scrapy.Field() topic = scrapy.Field() url = scrapy.Field() class QuotesSpider(scrapy.Spider): name = "noticias" Loading @@ -26,26 +20,22 @@ class QuotesSpider(scrapy.Spider): year = getattr(self, 'year', None) month = getattr(self, 'month', None) day = getattr(self, 'day', None) self.baseURL = 'http://alchile.com.mx/' + year + '/' + month + '/' + day yield scrapy.Request(url=self.baseURL, callback=self.parse) def parse(self, response): pagination = response.css('div.page-nav').css('a.last::attr(href)').extract() if ( len(pagination) > 0 ): pagination = pagination[0].strip('/') pages = int(pagination[pagination.rfind('/')+1:]) for page in range(0,pages): if ( page == 0 ): yield scrapy.Request(url=response.url, callback=self.parse_page, dont_filter=True) else: yield scrapy.Request(url=self.baseURL+"/page/"+str(page+1), callback=self.parse_page) pagination = response.css('div.page-nav').css('a::attr(href)').extract() if len(pagination) > 0: pagination = pagination[-2].strip('/') pages = int(pagination[pagination.rfind('/')+1:]) else: yield scrapy.Request(url=response.url, callback=self.parse_page, dont_filter=True) for page in range(1,pages): yield scrapy.Request(url=self.baseURL+"/page/"+str(page+1), callback=self.parse_page) def parse_page(self, response): Loading