Skip to content

crawlersNoticias
crawlersNoticias
Commit 30b928b1 authored by Renán Sosa Guillen's avatar Renán Sosa Guillen
Browse files
Options

deleted

parent 39fc5ca3
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 610 deletions
descarga_hacia_atras/foraneos/elSalvador/elSalvador/items.py deleted 100644 → 0
View file @ 39fc5ca3
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class NoticiasItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
title = scrapy.Field()
text = scrapy.Field()
date = scrapy.Field()
location = scrapy.Field()
author = scrapy.Field()
topic = scrapy.Field()
url = scrapy.Field()
descarga_hacia_atras/foraneos/elSalvador/elSalvador/middlewares.py deleted 100644 → 0
View file @ 39fc5ca3
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
class ElsalvadorSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class ElsalvadorDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
descarga_hacia_atras/foraneos/elSalvador/elSalvador/pipelines.py deleted 100644 → 0
View file @ 39fc5ca3
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
from collections import OrderedDict
class JsonWriterPipeline(object):
def __init__(self, filename):
self.filename = filename
@classmethod
def from_crawler(cls, crawler):
# Here you get whatever value was passed through the "filename" command line parameter
settings = crawler.settings
filename = settings.get('filename')
# Instantiate the pipeline with the file name
return cls(filename)
def open_spider(self, spider):
self.counter = 0
self.file = open(self.filename, 'w')
self.file.write("[")
def close_spider(self, spider):
self.file.write("]")
self.file.close()
def process_item(self, item, spider):
# print("this is my item", item)
row = []
try:
row.append(("date", item['date']))
except:
pass
try:
row.append(("topic", item['topic']))
except:
pass
try:
row.append(("title", item['title']))
except:
pass
try:
row.append(("author", item['author']))
except:
pass
try:
row.append(("location", item['location']))
except:
pass
try:
row.append(("text", item['text']))
except:
pass
try:
row.append(("url", item['url']))
except:
pass
line = OrderedDict(row)
self.counter += 1
if self.counter == 1:
self.file.write(json.dumps(line))
elif self.counter > 1:
self.file.write(",\n" + json.dumps(line))
return item
descarga_hacia_atras/foraneos/elSalvador/elSalvador/settings.py deleted 100644 → 0
View file @ 39fc5ca3
# -*- coding: utf-8 -*-
# Scrapy settings for elSalvador project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'elSalvador'
SPIDER_MODULES = ['elSalvador.spiders']
NEWSPIDER_MODULE = 'elSalvador.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'elSalvador (+http://www.yourdomain.com)'
# Obey robots.txt rules
# ROBOTSTXT_OBEY = True
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'elSalvador.middlewares.ElsalvadorSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'elSalvador.middlewares.ElsalvadorDownloaderMiddleware': 543,
#}
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'elSalvador.pipelines.JsonWriterPipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
descarga_hacia_atras/foraneos/elSalvador/elSalvador/spiders/__init__.py deleted 100644 → 0
View file @ 39fc5ca3
# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.
descarga_hacia_atras/foraneos/elSalvador/elSalvador/spiders/noticias.py deleted 100644 → 0
View file @ 39fc5ca3
# -*- coding: utf-8 -*-
import scrapy, re, json, ast
from scrapy.selector import Selector
from datetime import datetime, date
from elSalvador.items import NoticiasItem
"""
MEDIO:
El Salvador
USO:
// Si se quiere obtener todas las noticias desde las más actuales hasta las más antiguas. //
scrapy crawl noticias --nolog -s filename=noticias.json
-------------------------------------------------------------------------------------------------
// Si se quiere obtener todas las noticias desde las más actuales hasta una fecha específica. //
scrapy crawl noticias --nolog -s filename=noticias.json -a year=2018 -a month=3 -a day=11
-------------------------------------------------------------------------------------------------
Después será necesario hacer uso del archivo parse_date_files.py para que las noticias contenidas
en noticias.json sean separadas en archivos por fecha.
"""
TAG_RE = re.compile(r'<[^>]+>')
def remove_tags(text):
return TAG_RE.sub('', text)
# LOC = re.compile(r'[A-Z].*?, ?[A-Z].+?\.')
class ImportantData(scrapy.Item):
CONTINUE_SEARCHING = scrapy.Field()
LAST_LINK = scrapy.Field()
page = scrapy.Field()
section_url = scrapy.Field()
url = scrapy.Field()
class QuotesSpider(scrapy.Spider):
name = "noticias"
def start_requests(self):
year = getattr(self, "year", None)
month = getattr(self, "month", None)
day = getattr(self, "day", None)
if year is not None and month is not None and day is not None:
self.stopDate = date(int(year), int(month), int(day))
else:
self.stopDate = None
baseURL = "http://www.elsalvador.com/category/noticias/"
# sectionList = []
sectionList = ["internacional"]
if self.stopDate is None:
for s in sectionList:
info = ImportantData()
info['page'] = 1
request = scrapy.Request(url=baseURL + s + "/", callback=self.parse)
request.meta['item'] = info
yield request
else:
for s in sectionList:
info = ImportantData()
info['page'] = 0
info['CONTINUE_SEARCHING'] = False
request = scrapy.Request(url=baseURL + s + "/", callback=self.parse_with_stop_date)
request.meta['item'] = info
yield request
# for s in sectionList:
# yield scrapy.Request(url=baseURL + s + "/", callback=self.parse)
def parse(self, response):
# print response.body
# print "----------------------------------------------------\n\n"
# print response.url
# searchData = response.meta['item']
# CONTINUE_SEARCHING = True
# linkSet = set()
# if searchData['page'] == 1:
# searchData['section_url'] = response.url
# linkSet = linkSet.union(set(response.xpath('//article[@id="destacada"]/a/@href').extract()))
# linkSet = linkSet.union(set(response.xpath('//aside[@id="mini_sidebar"]/article/a/@href').extract()))
# linkSet = linkSet.union(set(response.xpath('//section[@id="principal"]/article/a/@href').extract()))
# linkSet = linkSet.union(set(response.xpath('//aside[@id="otras_noticias"]/article/a/@href').extract()))
# linkSet = linkSet.union(set(response.xpath('//div[@class="contenedor"]/article/a/@href').extract()))
# linkSet = linkSet.union(set(response.xpath('//article[@class="nobordes"]/div/a/@href').extract()))
# linkSet.remove(searchData['section_url'])
#
# else:
# linkSet = linkSet.union(set(response.xpath('//div[@class="contenedor"]/article/a/@href').extract()))
# linkSet = linkSet.union(set(response.xpath('//article[@class="nobordes"]/div/a/@href').extract()))
# try:
# linkSet.remove(searchData['section_url'])
# except KeyError:
# pass
# if len(linkSet) <= 0:
# CONTINUE_SEARCHING = False
#
# for link in linkSet:
# yield scrapy.Request(url=link, callback=self.parse_item)
#
# if CONTINUE_SEARCHING:
# searchData['page'] += 1
# page = searchData['page']
# url = searchData['section_url']
# request = scrapy.Request(url=url + "?page=" + str(page), callback=self.parse)
# request.meta['item'] = searchData
# yield request
linkList = response.xpath('//div[@id="main"]').css('h2.large-title').xpath('./a/@href').extract()
linkList.extend(response.xpath('//div[@class="container even"]').css('h2.large-title').xpath('./a/@href').extract())
for link in linkList:
print link
# url = "http://www.elsalvador.com//wp-json/LoadMore/Category/posts"
# url_peticion = "/category/noticias/internacional/"
# frmdata = {'pppDestacado': "5", 'pppNoDestacado': "4", 'slug': "internacional", 'paged': "4526", 'category_name': "Internacional", 'url_peticion': url_peticion}
#
# yield scrapy.http.FormRequest(url=url, formdata=frmdata, callback=self.after_post)
# def after_post(self, response):
# searchData = response.meta['item']
# # from scrapy.shell import inspect_response
# # print "This is response: "
# unescaped = ast.literal_eval(response.body.strip())
# body = Selector(text=unescaped)
# # inspect_response(response, self)
# newsList = []
# linksObtained = body.xpath('//div[@class="row news"]').css('div.subsection').css('h2').xpath('./a/@href').extract()
# for link in linksObtained:
# link = link.replace('\\', '')
# if not link in newsList:
# newsList.append(link)
#
# # print len(newsList) checar length de newList para determinar el paro
# if len(newsList) > 0:
# for link in newsList:
# info = ImportantData()
# info['url'] = searchData['url']
# info['page'] = searchData['page']
# info['section_url'] = searchData['section_url']
# if link == linkList[-1]: info['LAST_LINK'] = True
# else: info['LAST_LINK'] = False
# reqst = scrapy.Request(url=link, callback=self.parse_item_with_stop_date)
# reqst.meta['item'] = info
# yield reqst
def parse_with_stop_date(self, response):
searchData = response.meta['item']
CONTINUE_SEARCHING = searchData['CONTINUE_SEARCHING']
if not CONTINUE_SEARCHING:
if searchData['page'] == 0:
searchData['section_url'] = response.url
newsList = response.xpath('//div[@id="main"]').css('h2.large-title').xpath('./a/@href').extract()
# newsList.extend(response.xpath('//div[@class="container even"]').css('h2.large-title').xpath('./a/@href').extract())
else:
unescaped = ast.literal_eval(response.body.strip())
body = Selector(text=unescaped)
newsList = []
for link in body.xpath('//div[@class="row news"]').css('div.subsection').css('h2').xpath('./a/@href').extract():
link = link.replace('\\', '')
if not link in newsList:
newsList.append(link)
if len(newsList) > 0:
for link in newsList:
info = ImportantData()
# info['url'] = response.url
info['page'] = searchData['page']
info['section_url'] = searchData['section_url']
if link == newsList[-1]: info['LAST_LINK'] = True
else: info['LAST_LINK'] = False
reqst = scrapy.Request(url=link, callback=self.parse_item_with_stop_date)
reqst.meta['item'] = info
yield reqst
# if searchData['page'] == 1:
# searchData['section_url'] = response.url
# linkList = response.xpath('//article[@id="destacada"]/a/@href').extract()
# linkList.extend(response.xpath('//aside[@id="mini_sidebar"]/article/a/@href').extract())
# linkList.extend(response.xpath('//section[@id="principal"]/article/a/@href').extract())
# linkList.extend(response.xpath('//aside[@id="otras_noticias"]/article/a/@href').extract())
# linkList.extend(response.xpath('//div[@class="contenedor"]/article/a/@href').extract())
# linkList.extend(response.xpath('//article[@class="nobordes"]/div/a/@href').extract())
# linkList.remove(searchData['section_url'])
#
# else:
# linkList = response.xpath('//div[@class="contenedor"]/article/a/@href').extract()
# linkList.extend(response.xpath('//article[@class="nobordes"]/div/a/@href').extract())
# try:
# linkList.remove(searchData['section_url'])
# except KeyError:
# pass
#
# newsList = []
# for link in linkList:
# if not link in newsList:
# newsList.append(link)
#
# for link in newsList:
# info = ImportantData()
# info['url'] = response.url
# info['page'] = searchData['page']
# info['section_url'] = searchData['section_url']
# if link == linkList[-1]: info['LAST_LINK'] = True
# else: info['LAST_LINK'] = False
# reqst = scrapy.Request(url=link, callback=self.parse_item_with_stop_date)
# reqst.meta['item'] = info
# yield reqst
else:
searchData['CONTINUE_SEARCHING'] = False
searchData['page'] += 1
page = str(searchData['page'])
url = "http://www.elsalvador.com//wp-json/LoadMore/Category/posts"
url_peticion = "/category/noticias/internacional/"
frmdata = {'pppDestacado': "5", 'pppNoDestacado': "4", 'slug': "internacional", 'paged': page,
'category_name': "Internacional", 'url_peticion': url_peticion}
request = scrapy.http.FormRequest(url=url, formdata=frmdata, callback=self.parse_with_stop_date)
request.meta['item'] = searchData
yield request
# searchData['CONTINUE_SEARCHING'] = False
# searchData['page'] += 1
# page = searchData['page']
# url = searchData['section_url']
# request = scrapy.Request(url=url + "?page=" + str(page), callback=self.parse_with_stop_date)
# request.meta['item'] = searchData
# yield request
def parse_item(self, response):
item = NoticiasItem()
d = response.xpath('//time/text()').extract_first()
res = remove_tags(response.xpath('//script[@type="application/ld+json"]').extract_first())
newsData = json.loads(res)
item['date'] = datetime.strptime(d, '%d.%m.%Y').isoformat("T")
item['title'] = newsData['headline']
try:
topic = newsData['articleSection']
except:
topic = None
item['topic'] = topic
text = newsData['articleBody']
if text.find(u'\u00a0') >= 0:
loc = text[:text.find(u'\u00a0')] + "."
m = LOC.match(loc)
if m:
item['location'] = m.group(0)
text = text[text.find(u'\u00a0') + 1:]
item['text'] = text
item['url'] = response.url
yield item
def parse_item_with_stop_date(self, response):
d = response.xpath('//meta[@property="article:published_time"]/@content').extract_first()
dt = datetime.strptime(d[:10], '%Y-%m-%d').date()
if dt >= self.stopDate:
info = response.meta['item']
item = NoticiasItem()
res = remove_tags(response.xpath('//script[@type="application/ld+json"]').extract_first())
newsData = json.loads(res)
item['date'] = datetime.strptime(d, '%d.%m.%Y').isoformat("T")
item['title'] = newsData['headline']
try:
topic = newsData['articleSection']
except:
topic = None
item['topic'] = topic
text = newsData['articleBody']
if text.find(u'\u00a0') >= 0:
loc = text[:text.find(u'\u00a0')] + "."
m = LOC.match(loc)
if m:
item['location'] = m.group(0)
text = text[text.find(u'\u00a0')+1:]
item['text'] = text
item['url'] = response.url
yield item
if info['LAST_LINK']:
info['CONTINUE_SEARCHING'] = True
request = scrapy.Request(url=info['section_url'], callback=self.parse_with_stop_date, dont_filter=True)
request.meta['item'] = info
yield request
descarga_hacia_atras/foraneos/elSalvador/scrapy.cfg deleted 100644 → 0
View file @ 39fc5ca3
# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs.io/en/latest/deploy.html
[settings]
default = elSalvador.settings
[deploy]
#url = http://localhost:6800/
project = elSalvador
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment