Skip to content

crawlersNoticias
crawlersNoticias
Commit fa10af53 authored by Renán Sosa Guillen's avatar Renán Sosa Guillen
Browse files
Options

crawlers

parent c9b94d38
Show whitespace changes
Inline Side-by-side
Showing with 1227 additions and 96 deletions
descarga_hacia_atras/diarioDeChiapas/diarioDeChiapas/items.py
View file @ fa10af53
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
# Define here the models for your scraped items # Define here the models for your scraped items
# #
# See documentation in: # See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html # https://doc.scrapy.org/en/latest/topics/items.html
import scrapy import scrapy
......
descarga_hacia_atras/diarioDeChiapas/diarioDeChiapas/pipelines.py
View file @ fa10af53
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
# Define your item pipelines here # Define your item pipelines here
# #
# Don't forget to add your pipeline to the ITEM_PIPELINES setting # Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json import json
from collections import OrderedDict from collections import OrderedDict
......
descarga_hacia_atras/diarioDeChiapas/diarioDeChiapas/spiders/noticias.py
View file @ fa10af53
...@@ -2,17 +2,17 @@ ...@@ -2,17 +2,17 @@
""" """
MEDIA: MEDIA:
Amandala, Belice Diario de Chiapas, Chiapas
USAGE USAGE
$ cd diarioDeChiapas $ cd diarioDeChiapas
// Si se quiere obtener todas las noticias desde las más actuales hasta las más antiguas. // ------------------------------------------------------------------------------------------------------------
## Get all the news from the most current to the oldest. It's necessary to use the parse_date_files.py file
for the news contained in noticias.json being splitted into files by date. ##
$ scrapy crawl noticias --nolog -s filename=noticias.json $ scrapy crawl noticias --nolog -s filename=noticias.json
------------------------------------------------------------------------------------------------- ------------------------------------------------------------------------------------------------------------
// Si se quiere obtener todas las noticias desde las más actuales hasta una fecha específica. // ## Get all the news from the most current to a specific date. ##
$ scrapy crawl noticias --nolog -s filename=noticias.json -a year=2018 -a month=3 -a day=5 $ scrapy crawl noticias --nolog -s filename=2018-08-30.json -a year=2018 -a month=8 -a day=30
-------------------------------------------------------------------------------------------------
Después será necesario hacer uso del archivo parse_date_files.py para que las noticias contenidas
en noticias.json sean separadas en archivos por fecha.
""" """
import scrapy, re, json import scrapy, re, json
...@@ -24,17 +24,23 @@ TAG_RE = re.compile(r'<[^>]+>') ...@@ -24,17 +24,23 @@ TAG_RE = re.compile(r'<[^>]+>')
def remove_tags(text): def remove_tags(text):
return TAG_RE.sub('', text) return TAG_RE.sub('', text)
DAT_RE = re.compile(r'\d{4}\/\d{2}\/\d{2}')
class ImportantData(scrapy.Item): class ImportantData(scrapy.Item):
continue_searching = scrapy.Field() """
last_link = scrapy.Field() Useful data for the flow of the implementation
section = scrapy.Field() """
url = scrapy.Field() to_next_page = scrapy.Field()
is_last_link = scrapy.Field()
news_section = scrapy.Field()
return_url = scrapy.Field()
class QuotesSpider(scrapy.Spider): class QuotesSpider(scrapy.Spider):
"""
Basic Scrapy Spider class
"""
name = "noticias" name = "noticias"
def start_requests(self): def start_requests(self):
...@@ -50,7 +56,7 @@ class QuotesSpider(scrapy.Spider): ...@@ -50,7 +56,7 @@ class QuotesSpider(scrapy.Spider):
baseURL = "http://www.diariodechiapas.com/landing/" baseURL = "http://www.diariodechiapas.com/landing/"
section_list = ["editorial", "portada", "metropoli", "region", "la-roja", section_list = ["editorial", "portada", "metropoli", "region", "la-roja",
"deportes", "boga", "ae", "opinion-dia", "trascendio"] "deportes", "boga", "ae", "trascendio"]
# section_list = ["editorial"] # section_list = ["editorial"]
if self.stopDate is None: if self.stopDate is None:
...@@ -60,7 +66,7 @@ class QuotesSpider(scrapy.Spider): ...@@ -60,7 +66,7 @@ class QuotesSpider(scrapy.Spider):
else: else:
for s in section_list: for s in section_list:
flow_info = ImportantData() flow_info = ImportantData()
flow_info['continue_searching'] = False flow_info['to_next_page'] = False
request = scrapy.Request(url=baseURL + s, callback=self.parse_with_stop_date) request = scrapy.Request(url=baseURL + s, callback=self.parse_with_stop_date)
request.meta['item'] = flow_info request.meta['item'] = flow_info
...@@ -68,6 +74,7 @@ class QuotesSpider(scrapy.Spider): ...@@ -68,6 +74,7 @@ class QuotesSpider(scrapy.Spider):
yield request yield request
def parse(self, response): def parse(self, response):
link_list = response.xpath('//section[@class="page__content"]').css('section.post').xpath('./a[@class="post__link"]/@href').extract() link_list = response.xpath('//section[@class="page__content"]').css('section.post').xpath('./a[@class="post__link"]/@href').extract()
section = response.xpath('//section[@class="wrapper"]/h1').extract_first() section = response.xpath('//section[@class="wrapper"]/h1').extract_first()
...@@ -75,7 +82,7 @@ class QuotesSpider(scrapy.Spider): ...@@ -75,7 +82,7 @@ class QuotesSpider(scrapy.Spider):
for link in link_list: for link in link_list:
flow_info = ImportantData() flow_info = ImportantData()
flow_info['section'] = section flow_info['news_section'] = section
request = scrapy.Request(url=link, callback=self.parse_item) request = scrapy.Request(url=link, callback=self.parse_item)
request.meta['item'] = flow_info request.meta['item'] = flow_info
...@@ -87,18 +94,22 @@ class QuotesSpider(scrapy.Spider): ...@@ -87,18 +94,22 @@ class QuotesSpider(scrapy.Spider):
yield scrapy.Request(url=next_page, callback=self.parse) yield scrapy.Request(url=next_page, callback=self.parse)
def parse_with_stop_date(self, response): def parse_with_stop_date(self, response):
flow_info = response.meta['item'] flow_info = response.meta['item']
if not flow_info['continue_searching']: if not flow_info['to_next_page']:
link_list = response.xpath('//section[@class="page__content"]').css('section.post').xpath('./a[@class="post__link"]/@href').extract() link_list = response.xpath('//section[@class="page__content"]').css('section.post').xpath('./a[@class="post__link"]/@href').extract()
section = response.xpath('//section[@class="wrapper"]/h1').extract_first()
if section is not None : section = remove_tags(section)
for link in link_list: for link in link_list:
flow_info = ImportantData() flow_info = ImportantData()
flow_info['url'] = response.url flow_info['news_section'] = section
flow_info['return_url'] = response.url
if link == link_list[-1] : flow_info['last_link'] = True if link == link_list[-1] : flow_info['is_last_link'] = True
else: flow_info['last_link'] = False else : flow_info['is_last_link'] = False
request = scrapy.Request(url=link, callback=self.parse_item_with_stop_date) request = scrapy.Request(url=link, callback=self.parse_item_with_stop_date)
request.meta['item'] = flow_info request.meta['item'] = flow_info
...@@ -108,7 +119,7 @@ class QuotesSpider(scrapy.Spider): ...@@ -108,7 +119,7 @@ class QuotesSpider(scrapy.Spider):
else: else:
next_page = response.css('div.wp-pagenavi').css('a.nextpostslink').css('::attr(href)').extract_first() next_page = response.css('div.wp-pagenavi').css('a.nextpostslink').css('::attr(href)').extract_first()
if next_page is not None: if next_page is not None:
flow_info['continue_searching'] = False flow_info['to_next_page'] = False
request = scrapy.Request(url=next_page, callback=self.parse_with_stop_date) request = scrapy.Request(url=next_page, callback=self.parse_with_stop_date)
request.meta['item'] = flow_info request.meta['item'] = flow_info
...@@ -116,6 +127,7 @@ class QuotesSpider(scrapy.Spider): ...@@ -116,6 +127,7 @@ class QuotesSpider(scrapy.Spider):
yield request yield request
def parse_item(self, response): def parse_item(self, response):
flow_info = response.meta['item'] flow_info = response.meta['item']
item = NoticiasItem() item = NoticiasItem()
...@@ -127,8 +139,9 @@ class QuotesSpider(scrapy.Spider): ...@@ -127,8 +139,9 @@ class QuotesSpider(scrapy.Spider):
for p in response.xpath('//section[@class="single__content"]').css('p').extract(): for p in response.xpath('//section[@class="single__content"]').css('p').extract():
text += remove_tags(p) + "\n" text += remove_tags(p) + "\n"
## News item info ##
item['date'] = response.xpath('//meta[@property="article:published_time"]/@content').extract_first() item['date'] = response.xpath('//meta[@property="article:published_time"]/@content').extract_first()
item['topic'] = flow_info['section'] item['topic'] = flow_info['news_section']
item['title'] = title item['title'] = title
item['text'] = text.strip() item['text'] = text.strip()
item['url'] = response.url item['url'] = response.url
...@@ -136,37 +149,36 @@ class QuotesSpider(scrapy.Spider): ...@@ -136,37 +149,36 @@ class QuotesSpider(scrapy.Spider):
yield item yield item
def parse_item_with_stop_date(self, response): def parse_item_with_stop_date(self, response):
d = response.xpath('//meta[@property="article:published_time"]/@content').extract_first() news_date = response.xpath('//meta[@property="article:published_time"]/@content').extract_first()
dt = datetime.strptime(d, '%Y-%m-%d').date() news_date = news_date[:news_date.find('T')]
news_date = datetime.strptime(news_date, '%Y-%m-%d').date()
if dt >= self.stopDate: if news_date >= self.stopDate:
flow_info = response.meta['item'] flow_info = response.meta['item']
item = NoticiasItem() item = NoticiasItem()
text = '' text = ''
item['date'] = datetime.strptime(d, '%Y-%m-%d').isoformat("T") title = response.xpath('//section[@class="single__content"]/h1').extract_first()
item['title'] = remove_tags(response.xpath('//div[@class="active"]/h1/a').extract_first()) if title is not None : title = remove_tags(title)
try:
topic = response.css('div.date').css('span.date').css('a::text').extract()[0]
except:
topic = None
item['topic'] = topic
for p in response.css('div.content').css('p').extract(): for p in response.xpath('//section[@class="single__content"]').css('p').extract():
text += remove_tags(p) + "\n" text += remove_tags(p) + "\n"
## News item info ##
item['date'] = response.xpath('//meta[@property="article:published_time"]/@content').extract_first()
item['topic'] = flow_info['news_section']
item['title'] = title
item['text'] = text.strip() item['text'] = text.strip()
item['url'] = response.url item['url'] = response.url
yield item yield item
if flow_info['last_link']: if flow_info['is_last_link']:
flow_info['continue_searching'] = True flow_info['to_next_page'] = True
request = scrapy.Request(url=flow_info['url'], callback=self.parse_with_stop_date, dont_filter=True) request = scrapy.Request(url=flow_info['return_url'], callback=self.parse_with_stop_date, dont_filter=True)
request.meta['item'] = flow_info request.meta['item'] = flow_info
yield request yield request
descarga_hacia_atras/diarioIndependiente/diarioIndependiente/items.py
View file @ fa10af53
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
# Define here the models for your scraped items # Define here the models for your scraped items
# #
# See documentation in: # See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html # https://doc.scrapy.org/en/latest/topics/items.html
import scrapy import scrapy
......
descarga_hacia_atras/diarioIndependiente/diarioIndependiente/pipelines.py
View file @ fa10af53
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
# Define your item pipelines here # Define your item pipelines here
# #
# Don't forget to add your pipeline to the ITEM_PIPELINES setting # Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json import json
from collections import OrderedDict from collections import OrderedDict
......
descarga_hacia_atras/diarioIndependiente/diarioIndependiente/spiders/noticias.py
View file @ fa10af53
...@@ -8,7 +8,7 @@ USAGE: ...@@ -8,7 +8,7 @@ USAGE:
$ cd elIndependiente/ $ cd elIndependiente/
------------------------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------------------------
## Get all the news from the most current to the oldest. It's necessary to use the parse_date_files.py file ## Get all the news from the most current to the oldest. It's necessary to use the parse_date_files.py file
for the news contained in noticias.json being splitted into files by date. for the news contained in noticias.json being splitted into files by date. ##
$ scrapy crawl noticias --nolog -s filename=noticias.json $ scrapy crawl noticias --nolog -s filename=noticias.json
------------------------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------------------------
## Get all the news from the most current to a specific date. ## ## Get all the news from the most current to a specific date. ##
...@@ -30,9 +30,9 @@ class ImportantFlowData(scrapy.Item): ...@@ -30,9 +30,9 @@ class ImportantFlowData(scrapy.Item):
""" """
Useful data for the flow of the implementation Useful data for the flow of the implementation
""" """
TO_NEXT_PAGE = scrapy.Field() to_next_page = scrapy.Field()
IS_LAST_LINK = scrapy.Field() is_last_link = scrapy.Field()
URL = scrapy.Field() return_url = scrapy.Field()
...@@ -66,7 +66,7 @@ class QuotesSpider(scrapy.Spider): ...@@ -66,7 +66,7 @@ class QuotesSpider(scrapy.Spider):
else: else:
flow_info = ImportantFlowData() flow_info = ImportantFlowData()
flow_info['TO_NEXT_PAGE'] = False flow_info['to_next_page'] = False
request = scrapy.Request(url=base_url, callback=self.parse_with_stop_date) request = scrapy.Request(url=base_url, callback=self.parse_with_stop_date)
request.meta['item'] = flow_info request.meta['item'] = flow_info
...@@ -97,17 +97,16 @@ class QuotesSpider(scrapy.Spider): ...@@ -97,17 +97,16 @@ class QuotesSpider(scrapy.Spider):
def parse_with_stop_date(self, response): def parse_with_stop_date(self, response):
flow_info = response.meta['item'] flow_info = response.meta['item']
TO_NEXT_PAGE = flow_info['TO_NEXT_PAGE']
if not TO_NEXT_PAGE: if not flow_info['to_next_page']:
link_list = response.xpath('//div[@id="colNoticias"]').css('article.card__article').xpath('./h2/a/@href').extract() link_list = response.xpath('//div[@id="colNoticias"]').css('article.card__article').xpath('./h2/a/@href').extract()
for link in link_list: for link in link_list:
flow_info = ImportantFlowData() flow_info = ImportantFlowData()
flow_info['URL'] = response.url flow_info['return_url'] = response.url
if link == link_list[-1] : flow_info['IS_LAST_LINK'] = True if link == link_list[-1] : flow_info['is_last_link'] = True
else : flow_info['IS_LAST_LINK'] = False else : flow_info['is_last_link'] = False
request = scrapy.Request(url=link, callback=self.parse_item_with_stop_date) request = scrapy.Request(url=link, callback=self.parse_item_with_stop_date)
request.meta['item'] = flow_info request.meta['item'] = flow_info
...@@ -118,7 +117,7 @@ class QuotesSpider(scrapy.Spider): ...@@ -118,7 +117,7 @@ class QuotesSpider(scrapy.Spider):
next_page = response.css('div.paginacion').xpath('./ul/li/a[@rel="next"]/@href').extract_first() next_page = response.css('div.paginacion').xpath('./ul/li/a[@rel="next"]/@href').extract_first()
if next_page is not None: if next_page is not None:
flow_info['TO_NEXT_PAGE'] = False flow_info['to_next_page'] = False
request = scrapy.Request(url=next_page, callback=self.parse_with_stop_date) request = scrapy.Request(url=next_page, callback=self.parse_with_stop_date)
request.meta['item'] = flow_info request.meta['item'] = flow_info
...@@ -141,6 +140,7 @@ class QuotesSpider(scrapy.Spider): ...@@ -141,6 +140,7 @@ class QuotesSpider(scrapy.Spider):
for p in response.css('div.cuerpo_noticia').css('p').extract(): for p in response.css('div.cuerpo_noticia').css('p').extract():
text += remove_tags(p) + "\n" text += remove_tags(p) + "\n"
## News item info ##
item['date'] = datetime.strptime(news_date, '%Y-%m-%d').isoformat("T") item['date'] = datetime.strptime(news_date, '%Y-%m-%d').isoformat("T")
item['title'] = remove_tags(response.css('h1.colorRojo').extract_first()) item['title'] = remove_tags(response.css('h1.colorRojo').extract_first())
item['topic'] = remove_tags(topic) item['topic'] = remove_tags(topic)
...@@ -160,26 +160,25 @@ class QuotesSpider(scrapy.Spider): ...@@ -160,26 +160,25 @@ class QuotesSpider(scrapy.Spider):
item = NoticiasItem() item = NoticiasItem()
text = '' text = ''
try:
topic = response.xpath('//span[@class="badge"]').extract_first() topic = response.xpath('//span[@class="badge"]').extract_first()
except: if topic is not None : remove_tags(topic)
topic = None
for p in response.css('div.cuerpo_noticia').css('p').extract(): for p in response.css('div.cuerpo_noticia').css('p').extract():
text += remove_tags(p) + "\n" text += remove_tags(p) + "\n"
## News item info ##
item['date'] = datetime.strptime(news_date.isoformat(), '%Y-%m-%d').isoformat("T") item['date'] = datetime.strptime(news_date.isoformat(), '%Y-%m-%d').isoformat("T")
item['title'] = remove_tags(response.css('h1.colorRojo').extract_first()) item['title'] = remove_tags(response.css('h1.colorRojo').extract_first())
item['topic'] = remove_tags(topic) item['topic'] = topic
item['text'] = text.strip() item['text'] = text.strip()
item['url'] = response.url item['url'] = response.url
yield item yield item
if flow_info['IS_LAST_LINK']: if flow_info['is_last_link']:
flow_info['TO_NEXT_PAGE'] = True flow_info['to_next_page'] = True
request = scrapy.Request(url=flow_info['URL'], callback=self.parse_with_stop_date, dont_filter=True) request = scrapy.Request(url=flow_info['return_url'], callback=self.parse_with_stop_date, dont_filter=True)
request.meta['item'] = flow_info request.meta['item'] = flow_info
yield request yield request
descarga_hacia_atras/heraldoChihuahua/heraldoChihuahua/items.py 0 → 100644
View file @ fa10af53
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class NoticiasItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
title = scrapy.Field()
text = scrapy.Field()
date = scrapy.Field()
location = scrapy.Field()
author = scrapy.Field()
topic = scrapy.Field()
url = scrapy.Field()
descarga_hacia_atras/heraldoChihuahua/heraldoChihuahua/middlewares.py 0 → 100644
View file @ fa10af53
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
class HeraldochihuahuaSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class HeraldochihuahuaDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
descarga_hacia_atras/heraldoChihuahua/heraldoChihuahua/pipelines.py 0 → 100644
View file @ fa10af53
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
from collections import OrderedDict
class JsonWriterPipeline(object):
def __init__(self, filename):
self.filename = filename
@classmethod
def from_crawler(cls, crawler):
# Here you get whatever value was passed through the "filename" command line parameter
settings = crawler.settings
filename = settings.get('filename')
# Instantiate the pipeline with the file name
return cls(filename)
def open_spider(self, spider):
self.counter = 0
self.file = open(self.filename, 'w')
self.file.write("[")
def close_spider(self, spider):
self.file.write("]")
self.file.close()
def process_item(self, item, spider):
# print("this is my item", item)
row = []
try:
row.append(("date", item['date']))
except:
pass
try:
row.append(("topic", item['topic']))
except:
pass
try:
row.append(("title", item['title']))
except:
pass
try:
row.append(("author", item['author']))
except:
pass
try:
row.append(("location", item['location']))
except:
pass
try:
row.append(("text", item['text']))
except:
pass
try:
row.append(("url", item['url']))
except:
pass
line = OrderedDict(row)
self.counter += 1
if self.counter == 1:
self.file.write(json.dumps(line))
elif self.counter > 1:
self.file.write(",\n" + json.dumps(line))
return item
descarga_hacia_atras/heraldoChihuahua/heraldoChihuahua/settings.py 0 → 100644
View file @ fa10af53
# -*- coding: utf-8 -*-
# Scrapy settings for heraldoChihuahua project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'heraldoChihuahua'
SPIDER_MODULES = ['heraldoChihuahua.spiders']
NEWSPIDER_MODULE = 'heraldoChihuahua.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'heraldoChihuahua (+http://www.yourdomain.com)'
# Obey robots.txt rules
# ROBOTSTXT_OBEY = True
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 0.5
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'heraldoChihuahua.middlewares.HeraldochihuahuaSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'heraldoChihuahua.middlewares.HeraldochihuahuaDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'heraldoChihuahua.pipelines.JsonWriterPipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
descarga_hacia_atras/heraldoChihuahua/heraldoChihuahua/spiders/__init__.py 0 → 100644
View file @ fa10af53
# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.
descarga_hacia_atras/heraldoChihuahua/heraldoChihuahua/spiders/noticias.py 0 → 100644
View file @ fa10af53
# -*- coding: utf-8 -*-
"""
MEDIA:
El Heraldo de Chihuahua, Chihuahua
USAGE
$ cd heraldoChihuahua
------------------------------------------------------------------------------------------------------------
## Get all the news from the most current to the oldest. It's necessary to use the parse_date_files.py file
for the news contained in noticias.json being splitted into files by date. ##
$ scrapy crawl noticias --nolog -s filename=noticias.json
------------------------------------------------------------------------------------------------------------
## Get all the news from the most current to a specific date. ##
$ scrapy crawl noticias --nolog -s filename=2018-08-30.json -a year=2018 -a month=8 -a day=30
"""
import scrapy, re, json, urllib
from datetime import datetime, date, tzinfo, timedelta
from collections import OrderedDict
from heraldoChihuahua.items import NoticiasItem
TAG_RE = re.compile(r'<[^>]+>')
def remove_tags(text):
return TAG_RE.sub('', text)
DAY_NUM = re.compile(r'[0-9]{1,2}')
class UTC(tzinfo):
"""
Class for Time Zone
"""
def utcoffset(self, dt):
## Time zone for Chihuahua: UTC-7 ##
return timedelta(hours=-7)
def tzname(self, dt):
## Time zone name ##
return 'UTC-7'
class ImportantData(scrapy.Item):
"""
Useful data for the flow of the implementation
"""
to_next_page = scrapy.Field()
is_last_link = scrapy.Field()
news_section = scrapy.Field()
news_page = scrapy.Field()
return_url = scrapy.Field()
class QuotesSpider(scrapy.Spider):
"""
Basic Scrapy Spider class
"""
name = "noticias"
def start_requests(self):
self.tz = UTC()
year = getattr(self, "year", None)
month = getattr(self, "month", None)
day = getattr(self, "day", None)
if year is not None and month is not None and day is not None:
self.stopDate = date(int(year), int(month), int(day))
else:
self.stopDate = None
self.baseURL = "https://www.elheraldodechihuahua.com.mx/"
section_list = ["local", "policiaca", "mexico", "republica", "mundo", "finanzas",
"analisis", "gossip", "circulos", "cultura", "doble-via", "deportes"]
# section_list = ["local"]
self.month_parser = dict(enero='01', febrero='02', marzo='03', abril='04', mayo='05', junio='06',
julio='07', agosto='08', septiembre='09', octubre='10', noviembre='11', diciembre='12')
if self.stopDate is None:
for s in section_list:
flow_info = ImportantData()
flow_info['news_page'] = 8
flow_info['news_section'] = s
request = scrapy.Request(url=self.baseURL + s, callback=self.parse)
request.meta['item'] = flow_info
yield request
else:
for s in section_list:
flow_info = ImportantData()
flow_info['to_next_page'] = False
flow_info['news_page'] = 8
flow_info['news_section'] = s
request = scrapy.Request(url=self.baseURL + s, callback=self.parse_with_stop_date)
request.meta['item'] = flow_info
yield request
def parse(self, response):
flow_info = response.meta['item']
link_list = response.css('div.hard-news-row').css('h4.title > a::attr(href)').extract()
if len(link_list) <= 0 : link_list = response.css('h4.title > a::attr(href)').extract()
for link in link_list:
yield scrapy.Request(url=link, callback=self.parse_item)
load_more = response.css('div.load-more').extract_first()
if load_more is not None:
flow_info['news_page'] += 4
params = OrderedDict()
params['widgetContentId'] = '148'
params['widgetName'] = 'viewPicker'
params['offset'] = str(flow_info['news_page'])
url_params = urllib.urlencode(params)
next_load = self.baseURL + flow_info['news_section'] + "/widget/?" + url_params
request = scrapy.Request(url=next_load, callback=self.parse)
request.meta['item'] = flow_info
yield request
def parse_with_stop_date(self, response):
flow_info = response.meta['item']
if not flow_info['to_next_page']:
link_list = response.css('div.hard-news-row').css('h4.title > a::attr(href)').extract()
if len(link_list) <= 0 : link_list = response.css('h4.title > a::attr(href)').extract()
news_page = flow_info['news_page']
news_section = flow_info['news_section']
for link in link_list:
flow_info = ImportantData()
flow_info['return_url'] = response.url
flow_info['news_page'] = news_page
flow_info['news_section'] = news_section
if link == link_list[-1] : flow_info['is_last_link'] = True
else : flow_info['is_last_link'] = False
request = scrapy.Request(url=link, callback=self.parse_item_with_stop_date)
request.meta['item'] = flow_info
yield request
else:
load_more = response.css('div.load-more').extract_first()
if load_more is not None:
flow_info['to_next_page'] = False
flow_info['news_page'] += 4
params = OrderedDict()
params['widgetContentId'] = '148'
params['widgetName'] = 'viewPicker'
params['offset'] = str(flow_info['news_page'])
url_params = urllib.urlencode(params)
next_load = self.baseURL + flow_info['news_section'] + "/widget/?" + url_params
request = scrapy.Request(url=next_load, callback=self.parse_with_stop_date)
request.meta['item'] = flow_info
yield request
def parse_item(self, response):
published_date = response.css('p.published-date').extract_first()
if published_date is not None:
published_date = remove_tags(published_date)
published_date = published_date.split('/')
published_date = published_date[1].strip().replace('\n', '')
date_lst = published_date.split(' de ')
element = DAY_NUM.search(date_lst[0])
if element:
item = NoticiasItem()
text = ''
date_lst[0] = element.group()
date_lst[1] = self.month_parser[date_lst[1].lower()]
date_lst = map(int, date_lst)
news_date = datetime(date_lst[2], date_lst[1], date_lst[0], tzinfo=self.tz).isoformat("T")
topic = response.css('div.breadcrumb > a').extract_first()
if topic is not None : topic = remove_tags(topic).strip("\n")
title = response.css('h1.title').extract_first()
if title is not None : title = remove_tags(title).strip("\n")
for p in response.css('div.content-body').xpath('./div[contains(@id, "content-body")]/p').extract():
text += remove_tags(p) + "\n"
## News item info ##
item['date'] = news_date
item['topic'] = topic
item['title'] = title
item['text'] = text.strip()
item['url'] = response.url
yield item
def parse_item_with_stop_date(self, response):
published_date = response.css('p.published-date').extract_first()
if published_date is not None:
published_date = remove_tags(published_date)
published_date = published_date.split('/')
published_date = published_date[1].strip().replace('\n', '')
date_lst = published_date.split(' de ')
element = DAY_NUM.search(date_lst[0])
if element:
date_lst[0] = element.group()
date_lst[1] = self.month_parser[date_lst[1].lower()]
news_date = "-".join(date_lst)
news_date = datetime.strptime(news_date, '%d-%m-%Y').date()
if news_date >= self.stopDate:
flow_info = response.meta['item']
item = NoticiasItem()
text = ''
date_lst = map(int, date_lst)
news_date = datetime(date_lst[2], date_lst[1], date_lst[0], tzinfo=self.tz).isoformat("T")
topic = response.css('div.breadcrumb > a').extract_first()
if topic is not None : topic = remove_tags(topic).strip("\n")
title = response.css('h1.title').extract_first()
if title is not None : title = remove_tags(title).strip("\n")
for p in response.css('div.content-body').xpath('./div[contains(@id, "content-body")]/p').extract():
text += remove_tags(p) + "\n"
## News item info ##
item['date'] = news_date
item['topic'] = topic
item['title'] = title
item['text'] = text.strip()
item['url'] = response.url
yield item
if flow_info['is_last_link']:
flow_info['to_next_page'] = True
request = scrapy.Request(url=flow_info['return_url'], callback=self.parse_with_stop_date, dont_filter=True)
request.meta['item'] = flow_info
yield request
descarga_hacia_atras/heraldoChihuahua/scrapy.cfg 0 → 100644
View file @ fa10af53
# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
default = heraldoChihuahua.settings
[deploy]
#url = http://localhost:6800/
project = heraldoChihuahua
descarga_hacia_atras/tintaFresca/scrapy.cfg 0 → 100644
View file @ fa10af53
# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
default = tintaFresca.settings
[deploy]
#url = http://localhost:6800/
project = tintaFresca
descarga_hacia_atras/tintaFresca/tintaFresca/items.py 0 → 100644
View file @ fa10af53
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class NoticiasItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
title = scrapy.Field()
text = scrapy.Field()
date = scrapy.Field()
location = scrapy.Field()
author = scrapy.Field()
topic = scrapy.Field()
url = scrapy.Field()
descarga_hacia_atras/tintaFresca/tintaFresca/middlewares.py 0 → 100644
View file @ fa10af53
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
class TintafrescaSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class TintafrescaDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
descarga_hacia_atras/tintaFresca/tintaFresca/pipelines.py 0 → 100644
View file @ fa10af53
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
from collections import OrderedDict
class JsonWriterPipeline(object):
def __init__(self, filename):
self.filename = filename
@classmethod
def from_crawler(cls, crawler):
# Here you get whatever value was passed through the "filename" command line parameter
settings = crawler.settings
filename = settings.get('filename')
# Instantiate the pipeline with the file name
return cls(filename)
def open_spider(self, spider):
self.counter = 0
self.file = open(self.filename, 'w')
self.file.write("[")
def close_spider(self, spider):
self.file.write("]")
self.file.close()
def process_item(self, item, spider):
# print("this is my item", item)
row = []
try:
row.append(("date", item['date']))
except:
pass
try:
row.append(("topic", item['topic']))
except:
pass
try:
row.append(("title", item['title']))
except:
pass
try:
row.append(("author", item['author']))
except:
pass
try:
row.append(("location", item['location']))
except:
pass
try:
row.append(("text", item['text']))
except:
pass
try:
row.append(("url", item['url']))
except:
pass
line = OrderedDict(row)
self.counter += 1
if self.counter == 1:
self.file.write(json.dumps(line))
elif self.counter > 1:
self.file.write(",\n" + json.dumps(line))
return item
descarga_hacia_atras/tintaFresca/tintaFresca/settings.py 0 → 100644
View file @ fa10af53
# -*- coding: utf-8 -*-
# Scrapy settings for tintaFresca project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'tintaFresca'
SPIDER_MODULES = ['tintaFresca.spiders']
NEWSPIDER_MODULE = 'tintaFresca.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'tintaFresca (+http://www.yourdomain.com)'
# Obey robots.txt rules
# ROBOTSTXT_OBEY = True
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 0.5
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'tintaFresca.middlewares.TintafrescaSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'tintaFresca.middlewares.TintafrescaDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'tintaFresca.pipelines.JsonWriterPipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
descarga_hacia_atras/tintaFresca/tintaFresca/spiders/__init__.py 0 → 100644
View file @ fa10af53
# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.
descarga_hacia_atras/tintaFresca/tintaFresca/spiders/noticias.py 0 → 100644
View file @ fa10af53
# -*- coding: utf-8 -*-
"""
MEDIA:
Tinta Fresca, Chiapas
USAGE
$ cd tintaFresca
------------------------------------------------------------------------------------------------------------
## Get all the news from the most current to the oldest. It's necessary to use the parse_date_files.py file
for the news contained in noticias.json being splitted into files by date. ##
$ scrapy crawl noticias --nolog -s filename=noticias.json
------------------------------------------------------------------------------------------------------------
## Get all the news from the most current to a specific date. ##
$ scrapy crawl noticias --nolog -s filename=2018-08-30.json -a year=2018 -a month=8 -a day=30
"""
import scrapy, re, json
from datetime import datetime, date, tzinfo, timedelta
from tintaFresca.items import NoticiasItem
TAG_RE = re.compile(r'<[^>]+>')
def remove_tags(text):
return TAG_RE.sub('', text)
class UTC(tzinfo):
"""
Class for Time Zone
"""
def utcoffset(self, dt):
## Time zone for Chiapas: UTC-6 ##
return timedelta(hours=-6)
def tzname(self, dt):
## Time zone name ##
return 'UTC-6'
class ImportantData(scrapy.Item):
"""
Useful data for the flow of the implementation
"""
to_next_page = scrapy.Field()
is_last_link = scrapy.Field()
news_section = scrapy.Field()
return_url = scrapy.Field()
class QuotesSpider(scrapy.Spider):
"""
Basic Scrapy Spider class
"""
name = "noticias"
def start_requests(self):
self.tz = UTC()
year = getattr(self, "year", None)
month = getattr(self, "month", None)
day = getattr(self, "day", None)
if year is not None and month is not None and day is not None:
self.stopDate = date(int(year), int(month), int(day))
else:
self.stopDate = None
baseURL = "http://tintafresca.com.mx/"
# section_list = ["letras_en_su_tinta/page1/", "tgz/page1/", "patria_chica/page1/", "hecho_en_chiapas/page1/", "show/page1/", "rafaga/page1/"]
section_list = ["tgz/page1/", "patria_chica/page1/", "hecho_en_chiapas/page1/", "show/page1/"]
self.month_parser = dict(Enero='01', Febrero='02', Marzo='03', Abril='04', Mayo='05', Junio='06',
Julio='07', Agosto='08', Septiembre='09', Octubre='10', Noviembre='11', Diciembre='12')
if self.stopDate is None:
for s in section_list:
yield scrapy.Request(url=baseURL + s, callback=self.parse)
else:
for s in section_list:
flow_info = ImportantData()
flow_info['to_next_page'] = False
request = scrapy.Request(url=baseURL + s, callback=self.parse_with_stop_date)
request.meta['item'] = flow_info
yield request
def parse(self, response):
link_list = response.css('ul.tintas').css('a.leer::attr(href)').extract()
# section = response.xpath('//div[@id="ruta"]').extract_first()
# if section is not None:
# section = remove_tags(section)
# section = section.replace("Inicio &gt; ", '')
for link in link_list:
flow_info = ImportantData()
# flow_info['news_section'] = section
request = scrapy.Request(url=link, callback=self.parse_item)
request.meta['item'] = flow_info
yield request
next_page = response.xpath('//ul[@class="pagination"]/li[3]/a/@href').extract_first()
if next_page is not None:
yield scrapy.Request(url=next_page, callback=self.parse)
def parse_with_stop_date(self, response):
flow_info = response.meta['item']
if not flow_info['to_next_page']:
link_list = response.css('ul.tintas').css('a.leer::attr(href)').extract()
# section = response.xpath('//div[@id="ruta"]').extract_first()
# if section is not None:
# section = remove_tags(section)
# section = section.replace("Inicio &gt; ", '')
for link in link_list:
flow_info = ImportantData()
# flow_info['news_section'] = section
flow_info['return_url'] = response.url
if link == link_list[-1] : flow_info['is_last_link'] = True
else : flow_info['is_last_link'] = False
request = scrapy.Request(url=link, callback=self.parse_item_with_stop_date)
request.meta['item'] = flow_info
yield request
else:
next_page = response.xpath('//ul[@class="pagination"]/li[3]/a/@href').extract_first()
if next_page is not None:
flow_info['to_next_page'] = False
request = scrapy.Request(url=next_page, callback=self.parse_with_stop_date)
request.meta['item'] = flow_info
yield request
def parse_item(self, response):
flow_info = response.meta['item']
item = NoticiasItem()
text = ''
date_str = response.xpath('//div[@class="balazo"]').extract_first()
if date_str.find('<br>') > -1 : date_str = date_str[date_str.find('<br>'):]
date_str = remove_tags(date_str)
date_lst = date_str.split('/')
date_lst[1] = self.month_parser[date_lst[1]]
date_lst = map(int, date_lst)
news_date = datetime(date_lst[2], date_lst[1], date_lst[0], tzinfo=self.tz).isoformat("T")
topic = response.css('div.seccion > h3.left > a').extract_first()
if topic is not None : topic = remove_tags(topic)
if topic is not None:
if topic == "Letras en su Tinta":
title = ''
lines = response.css('div.sumario > p').extract()
for line in lines:
if lines.index(line) != len(lines)-1 : title += remove_tags(line) + ". "
else : title += remove_tags(line) + "."
else:
title = response.css('div.titulo > h1').extract_first()
if title is not None : title = remove_tags(title)
else:
title = None
for p in response.css('div.contenido > p').extract():
text += remove_tags(p) + "\n"
## News item info ##
item['date'] = news_date
item['topic'] = topic
item['title'] = title
item['text'] = text.strip()
item['url'] = response.url
yield item
def parse_item_with_stop_date(self, response):
date_str = response.xpath('//div[@class="balazo"]').extract_first()
if date_str.find('<br>') > -1 : date_str = date_str[date_str.find('<br>'):]
date_str = remove_tags(date_str)
date_lst = date_str.split('/')
date_lst[1] = self.month_parser[date_lst[1]]
news_date = "-".join(date_lst)
news_date = datetime.strptime(news_date, '%d-%m-%Y').date()
if news_date >= self.stopDate:
flow_info = response.meta['item']
item = NoticiasItem()
text = ''
date_lst = map(int, date_lst)
news_date = datetime(date_lst[2], date_lst[1], date_lst[0], tzinfo=self.tz).isoformat("T")
topic = response.css('div.seccion > h3.left > a').extract_first()
if topic is not None : topic = remove_tags(topic)
if topic is not None:
if topic == "Letras en su Tinta":
title = ''
lines = response.css('div.sumario > p').extract()
for line in lines:
if lines.index(line) != len(lines)-1 : title += remove_tags(line) + ". "
else : title += remove_tags(line) + "."
else:
title = response.css('div.titulo > h1').extract_first()
if title is not None : title = remove_tags(title)
else:
title = None
for p in response.css('div.contenido > p').extract():
text += remove_tags(p) + "\n"
## News item info ##
item['date'] = news_date
item['topic'] = topic
item['title'] = title
item['text'] = text.strip()
item['url'] = response.url
yield item
if flow_info['is_last_link']:
flow_info['to_next_page'] = True
request = scrapy.Request(url=flow_info['return_url'], callback=self.parse_with_stop_date, dont_filter=True)
request.meta['item'] = flow_info
yield request
descarga_por_dia/cuartoPoder/cuartoPoder/items.py
View file @ fa10af53
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
# Define here the models for your scraped items # Define here the models for your scraped items
# #
# See documentation in: # See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html # https://doc.scrapy.org/en/latest/topics/items.html
import scrapy import scrapy
......
descarga_por_dia/cuartoPoder/cuartoPoder/pipelines.py
View file @ fa10af53
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
# Define your item pipelines here # Define your item pipelines here
# #
# Don't forget to add your pipeline to the ITEM_PIPELINES setting # Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json import json
from collections import OrderedDict from collections import OrderedDict
......
descarga_por_dia/cuartoPoder/cuartoPoder/spiders/noticias.py
View file @ fa10af53
...@@ -39,8 +39,8 @@ class ImportantData(scrapy.Item): ...@@ -39,8 +39,8 @@ class ImportantData(scrapy.Item):
""" """
Useful data for the flow of the implementation Useful data for the flow of the implementation
""" """
CONTINUE = scrapy.Field() to_next_page = scrapy.Field()
page = scrapy.Field() next_page = scrapy.Field()
...@@ -62,8 +62,8 @@ class QuotesSpider(scrapy.Spider): ...@@ -62,8 +62,8 @@ class QuotesSpider(scrapy.Spider):
self.second_URL = self.baseURL + "/XStatic/cuartopoder/template/cargaBloque.aspx?strControl=ctrlArchivoResultadosPaginadoListado&" self.second_URL = self.baseURL + "/XStatic/cuartopoder/template/cargaBloque.aspx?strControl=ctrlArchivoResultadosPaginadoListado&"
flow_info = ImportantData() flow_info = ImportantData()
flow_info['CONTINUE'] = False flow_info['to_next_page'] = False
flow_info['page'] = 2 flow_info['next_page'] = 2
request = scrapy.Request(url=first_URL, callback=self.parse) request = scrapy.Request(url=first_URL, callback=self.parse)
request.meta['item'] = flow_info request.meta['item'] = flow_info
...@@ -74,20 +74,19 @@ class QuotesSpider(scrapy.Spider): ...@@ -74,20 +74,19 @@ class QuotesSpider(scrapy.Spider):
def parse(self, response): def parse(self, response):
flow_info = response.meta['item'] flow_info = response.meta['item']
CONTINUE = flow_info['CONTINUE']
for link in response.css('ul.news-list').xpath('./li/h5/a/@href').extract(): for link in response.css('ul.news-list').xpath('./li/h5/a/@href').extract():
CONTINUE = True to_next_page = True
news_link = self.baseURL + link news_link = self.baseURL + link
yield scrapy.Request(url=news_link, callback=self.parse_item) yield scrapy.Request(url=news_link, callback=self.parse_item)
if CONTINUE: if flow_info['to_next_page']:
page = flow_info['page'] page = flow_info['next_page']
page_URL = self.second_URL + "p={3}&eids=&fd={1}-{2}-{0}&fh={1}-{2}-{0}&id=portada".format(self.year, self.month.zfill(2), self.day.zfill(2), str(page)) page_URL = self.second_URL + "p={3}&eids=&fd={1}-{2}-{0}&fh={1}-{2}-{0}&id=portada".format(self.year, self.month.zfill(2), self.day.zfill(2), str(page))
flow_info['CONTINUE'] = False flow_info['to_next_page'] = False
flow_info['page'] += 1 flow_info['next_page'] += 1
request = scrapy.Request(url=page_URL, callback=self.parse) request = scrapy.Request(url=page_URL, callback=self.parse)
request.meta['item'] = flow_info request.meta['item'] = flow_info
...@@ -101,8 +100,7 @@ class QuotesSpider(scrapy.Spider): ...@@ -101,8 +100,7 @@ class QuotesSpider(scrapy.Spider):
text = '' text = ''
topic = response.css('div.big-title').xpath('./h2/a/span').extract_first() topic = response.css('div.big-title').xpath('./h2/a/span').extract_first()
if topic is not None: if topic is not None : topic = remove_tags(topic)
topic = remove_tags(topic)
for p in response.css('div.post-content').css('p').extract(): for p in response.css('div.post-content').css('p').extract():
p = remove_tags(p) p = remove_tags(p)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment