Commit 5953d008 authored by Renán Sosa Guillen's avatar Renán Sosa Guillen
Browse files

crawlers

parent ba65cc37
Loading
Loading
Loading
Loading
+8 −2
Changes for descarga_por_dia/alChile/alChile/items.py: 8 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -8,7 +8,13 @@
import scrapy


class AlchileItem(scrapy.Item):
class NoticiasItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass
    title = scrapy.Field()
    text = scrapy.Field()
    date = scrapy.Field()
    location = scrapy.Field()
    author = scrapy.Field()
    topic = scrapy.Field()
    url = scrapy.Field()
+65 −1
Changes for descarga_por_dia/alChile/alChile/pipelines.py: 65 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -5,7 +5,71 @@
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import json
from collections import OrderedDict


class JsonWriterPipeline(object):

    def __init__(self, filename):
        self.filename = filename

    @classmethod
    def from_crawler(cls, crawler):
        # Here you get whatever value was passed through the "filename" command line parameter
        settings = crawler.settings
        filename = settings.get('filename')

        # Instantiate the pipeline with the file name
        return cls(filename)

    def open_spider(self, spider):
        self.counter = 0
        self.file = open(self.filename, 'w')
        self.file.write("[")

    def close_spider(self, spider):
        self.file.write("]")
        self.file.close()

class AlchilePipeline(object):
    def process_item(self, item, spider):
        # print("this is my item", item)
        row = []
        try:
            row.append(("date", item['date']))
        except:
            pass
        try:
            row.append(("topic", item['topic']))
        except:
            pass
        try:
            row.append(("title", item['title']))
        except:
            pass
        try:
            row.append(("author", item['author']))
        except:
            pass
        try:
            row.append(("location", item['location']))
        except:
            pass
        try:
            row.append(("text", item['text']))
        except:
            pass
        try:
            row.append(("url", item['url']))
        except:
            pass

        line = OrderedDict(row)

        self.counter += 1
        if self.counter == 1:
            self.file.write(json.dumps(line))
        elif self.counter > 1:
            self.file.write(",\n" + json.dumps(line))

        return item
+6 −6
Changes for descarga_por_dia/alChile/alChile/settings.py: 6 added lines, 6 removed lines.
Original line number Diff line number Diff line
@@ -19,7 +19,7 @@ NEWSPIDER_MODULE = 'alChile.spiders'
#USER_AGENT = 'alChile (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = True
# ROBOTSTXT_OBEY = True

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
@@ -27,13 +27,13 @@ ROBOTSTXT_OBEY = True
# Configure a delay for requests for the same website (default: 0)
# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
DOWNLOAD_DELAY = 0.5
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
@@ -64,9 +64,9 @@ ROBOTSTXT_OBEY = True

# Configure item pipelines
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
#    'alChile.pipelines.AlchilePipeline': 300,
#}
ITEM_PIPELINES = {
   'alChile.pipelines.JsonWriterPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See http://doc.scrapy.org/en/latest/topics/autothrottle.html
+12 −22
Changes for descarga_por_dia/alChile/alChile/spiders/noticias.py: 12 added lines, 22 removed lines.
Original line number Diff line number Diff line
import scrapy, re
from alChile.items import NoticiasItem

"""
scrapy crawl noticias -t json --nolog -o noticias.json -a year=2017 -a month=3 -a day=22
MEDIO:
Al Chile, Yucatan
USO:
scrapy crawl noticias --nolog -s filename=2017-03-22.json -a year=2017 -a month=3 -a day=22
"""

TAG_RE = re.compile(r'<[^>]+>')
@@ -9,16 +13,6 @@ def remove_tags(text):
    return TAG_RE.sub('', text)


class NoticiasItem(scrapy.Item):
    title = scrapy.Field()
    text = scrapy.Field()
    date = scrapy.Field()
    location = scrapy.Field()
    author = scrapy.Field()
    topic = scrapy.Field()
    url = scrapy.Field()


class QuotesSpider(scrapy.Spider):
    name = "noticias"

@@ -26,26 +20,22 @@ class QuotesSpider(scrapy.Spider):
        year = getattr(self, 'year', None)
        month = getattr(self, 'month', None)
        day = getattr(self, 'day', None)
        
        self.baseURL = 'http://alchile.com.mx/' + year + '/' + month + '/' + day

        yield scrapy.Request(url=self.baseURL, callback=self.parse)


    def parse(self, response):
        pagination = response.css('div.page-nav').css('a.last::attr(href)').extract()
        if ( len(pagination) > 0 ):
            pagination = pagination[0].strip('/')
            pages = int(pagination[pagination.rfind('/')+1:])

            for page in range(0,pages):
                if ( page == 0 ):
        yield scrapy.Request(url=response.url, callback=self.parse_page, dont_filter=True)

                else:
                    yield scrapy.Request(url=self.baseURL+"/page/"+str(page+1), callback=self.parse_page)
        pagination = response.css('div.page-nav').css('a::attr(href)').extract()
        if len(pagination) > 0:
            pagination = pagination[-2].strip('/')
            pages = int(pagination[pagination.rfind('/')+1:])

        else:
            yield scrapy.Request(url=response.url, callback=self.parse_page, dont_filter=True)
            for page in range(1,pages):
                yield scrapy.Request(url=self.baseURL+"/page/"+str(page+1), callback=self.parse_page)


    def parse_page(self, response):
Loading