Skip to content

crawlersNoticias
crawlersNoticias
Commit 834d071c authored by Renán Sosa Guillen's avatar Renán Sosa Guillen
Browse files
Options

merge foraneos

parents a760e68d 9a8630fb
Show whitespace changes
Inline Side-by-side
Showing with 73 additions and 32 deletions
README.md
View file @ 834d071c
......@@ -473,6 +473,16 @@ Adicionalmente se cuenta con los siguientes medios extranjeros:
scrapy crawl noticias --nolog -s filename=2018-02-23.json -a year=2018 -a month=2 -a day=23
```
No se encontró notas antes del 2013.10.04.
* [El Heraldo, Honduras](http://www.elheraldo.hn)
Uso:
```bash
cd descarga_hacia_atras/foraneos/heraldoHn
scrapy crawl noticias --nolog -s filename=noticias.json // obtiene todas las posibles
scrapy crawl noticias --nolog -s filename=noticias.json -a year=2018 -a month=3 -a day=9 //obtiene hasta una fecha dada
```
No se encontró notas antes del ----.--.--.
* [La Prensa Gráfica, El Salvador](https://www.laprensagrafica.com)
Uso:
......
crawler_data.json
View file @ 834d071c
......@@ -36,6 +36,7 @@
{"nombre": "Yucatan al Minuto", "crawler": "descarga_hacia_atras/yucatanAlMinuto", "desde": "17-01-2017", "url": "http://www.yucatanalminuto.com"},
{"nombre": "Yucatan en Corto", "crawler": "descarga_por_dia/yucatanEnCorto", "desde": "02-04-2011", "url": "http://www.yucatanencorto.com/noticias"},
{"nombre": "Diario Co Latino", "crawler": "descarga_por_dia/foraneos/diarioCoLatino", "desde": "04-10-2013", "url": "https://www.diariocolatino.com"},
{"nombre": "El Heraldo Hn", "crawler": "descarga_hacia_atras/foraneos/heraldoHn", "url": "http://www.elheraldo.hn"},
{"nombre": "La Prensa Grafica", "crawler": "descarga_hacia_atras/foraneos/prensaGrafica", "desde": "05-09-2017", "url": "https://www.laprensagrafica.com"},
{"nombre": "The San Pedro Sun", "crawler": "descarga_por_dia/foraneos/sanPedroSun", "desde": "21-07-2008", "url": "https://www.sanpedrosun.com"},
{"nombre": "Tiempo Digital Hn", "crawler": "descarga_por_dia/foraneos/tiempoDigitalHn", "desde": "17-04-2015", "url": "https://tiempo.hn"},
......
descarga_hacia_atras/foraneos/heraldoHn/heraldoHn/spiders/noticias.py
View file @ 834d071c
......@@ -21,6 +21,8 @@ TAG_RE = re.compile(r'<[^>]+>')
def remove_tags(text):
return TAG_RE.sub('', text)
LOC = re.compile(r'[A-Z].*?, ?[A-Z].+?\.')
class ImportantData(scrapy.Item):
CONTINUE_SEARCHING = scrapy.Field()
......@@ -45,15 +47,15 @@ class QuotesSpider(scrapy.Spider):
baseURL = "http://www.elheraldo.hn/"
sectionList = ["economia", "mundo", "tecnologia", "cine", "cultura", "turismo",
"honduras", "sucesos", "espectaculos", "deportes"]
# sectionList = ["economia"]
sectionList = ["tegucigalpa", "economia", "mundo", "revistas/crimenes",
"pais", "sucesos", "deportes", "entretenimiento"]
# sectionList = ["tegucigalpa"]
if self.stopDate is None:
for s in sectionList:
info = ImportantData()
info['page'] = 1
request = scrapy.Request(url=baseURL + s, callback=self.parse)
request = scrapy.Request(url=baseURL + s + "/", callback=self.parse)
request.meta['item'] = info
yield request
......@@ -62,27 +64,34 @@ class QuotesSpider(scrapy.Spider):
info = ImportantData()
info['page'] = 1
info['CONTINUE_SEARCHING'] = False
request = scrapy.Request(url=baseURL + s, callback=self.parse_with_stop_date)
request = scrapy.Request(url=baseURL + s + "/", callback=self.parse_with_stop_date)
request.meta['item'] = info
yield request
def parse(self, response):
print response.url
searchData = response.meta['item']
CONTINUE_SEARCHING = True
linkSet = set()
if searchData['page'] == 1:
searchData['section_url'] = response.url + "/"
entrySet = set(response.css('article.entry').css('div.content').css('a::attr(href)').extract())
entrySet.remove(searchData['section_url'])
linkSet = set(response.css('article.grid').css('div.content').css('a::attr(href)').extract())
searchData['section_url'] = response.url
linkSet = linkSet.union(set(response.xpath('//article[@id="destacada"]/a/@href').extract()))
linkSet = linkSet.union(set(response.xpath('//aside[@id="mini_sidebar"]/article/a/@href').extract()))
linkSet = linkSet.union(set(response.xpath('//section[@id="principal"]/article/a/@href').extract()))
linkSet = linkSet.union(set(response.xpath('//aside[@id="otras_noticias"]/article/a/@href').extract()))
linkSet = linkSet.union(set(response.xpath('//div[@class="contenedor"]/article/a/@href').extract()))
linkSet = linkSet.union(set(response.xpath('//article[@class="nobordes"]/div/a/@href').extract()))
linkSet.remove(searchData['section_url'])
linkSet.union(entrySet)
else:
linkSet = set(response.css('article.grid').css('div.content').css('a::attr(href)').extract())
linkSet = linkSet.union(set(response.xpath('//div[@class="contenedor"]/article/a/@href').extract()))
linkSet = linkSet.union(set(response.xpath('//article[@class="nobordes"]/div/a/@href').extract()))
try:
linkSet.remove(searchData['section_url'])
except KeyError:
pass
if len(linkSet) <= 0:
CONTINUE_SEARCHING = False
for link in linkSet:
......@@ -103,21 +112,27 @@ class QuotesSpider(scrapy.Spider):
if not CONTINUE_SEARCHING:
if searchData['page'] == 1:
searchData['section_url'] = response.url + "/"
entrySet = set(response.css('article.entry').css('div.content').css('a::attr(href)').extract())
entrySet.remove(searchData['section_url'])
linkSet = set(response.css('article.grid').css('div.content').css('a::attr(href)').extract())
linkSet.remove(searchData['section_url'])
linkSet.union(entrySet)
linkList = list(linkSet)
searchData['section_url'] = response.url
linkList = response.xpath('//article[@id="destacada"]/a/@href').extract()
linkList.extend(response.xpath('//aside[@id="mini_sidebar"]/article/a/@href').extract())
linkList.extend(response.xpath('//section[@id="principal"]/article/a/@href').extract())
linkList.extend(response.xpath('//aside[@id="otras_noticias"]/article/a/@href').extract())
linkList.extend(response.xpath('//div[@class="contenedor"]/article/a/@href').extract())
linkList.extend(response.xpath('//article[@class="nobordes"]/div/a/@href').extract())
linkList.remove(searchData['section_url'])
else:
linkSet = set(response.css('article.grid').css('div.content').css('a::attr(href)').extract())
linkList = response.xpath('//div[@class="contenedor"]/article/a/@href').extract()
linkList.extend(response.xpath('//article[@class="nobordes"]/div/a/@href').extract())
try:
linkSet.remove(searchData['section_url'])
linkList = list(linkSet)
linkList.remove(searchData['section_url'])
except KeyError:
linkList = []
pass
newsList = []
for link in linkList:
if not link in newsList:
newsList.append(link)
for link in linkList:
info = ImportantData()
......@@ -142,10 +157,11 @@ class QuotesSpider(scrapy.Spider):
def parse_item(self, response):
item = NoticiasItem()
d = response.xpath('//time/text()').extract_first()
res = remove_tags(response.xpath('//script[@type="application/ld+json"]').extract_first())
newsData = json.loads(res)
item['date'] = newsData['datePublished'][:-1]
item['date'] = datetime.strptime(d, '%d.%m.%Y').isoformat("T")
item['title'] = newsData['headline']
try:
......@@ -154,7 +170,15 @@ class QuotesSpider(scrapy.Spider):
topic = None
item['topic'] = topic
item['text'] = newsData['articleBody']
text = newsData['articleBody']
if text.find(u'\u00a0') >= 0:
loc = text[:text.find(u'\u00a0')] + "."
m = LOC.match(loc)
if m:
item['location'] = m.group(0)
text = text[text.find(u'\u00a0') + 1:]
item['text'] = text
item['url'] = response.url
......@@ -162,18 +186,16 @@ class QuotesSpider(scrapy.Spider):
def parse_item_with_stop_date(self, response):
res = remove_tags(response.xpath('//script[@type="application/ld+json"]').extract_first())
newsData = json.loads(res)
d = newsData['datePublished']
d = d[:d.find("T")]
dt = datetime.strptime(d, '%Y-%m-%d').date()
d = response.xpath('//time/text()').extract_first()
dt = datetime.strptime(d, '%d.%m.%Y').date()
if dt >= self.stopDate:
info = response.meta['item']
item = NoticiasItem()
text = ''
res = remove_tags(response.xpath('//script[@type="application/ld+json"]').extract_first())
newsData = json.loads(res)
item['date'] = newsData['datePublished'][:-1]
item['date'] = datetime.strptime(d, '%d.%m.%Y').isoformat("T")
item['title'] = newsData['headline']
try:
......@@ -182,7 +204,15 @@ class QuotesSpider(scrapy.Spider):
topic = None
item['topic'] = topic
item['text'] = newsData['articleBody']
text = newsData['articleBody']
if text.find(u'\u00a0') >= 0:
loc = text[:text.find(u'\u00a0')] + "."
m = LOC.match(loc)
if m:
item['location'] = m.group(0)
text = text[text.find(u'\u00a0')+1:]
item['text'] = text
item['url'] = response.url
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment