Skip to content

crawlersNoticias
crawlersNoticias
Commit 5c3ec077 authored by Renán Sosa Guillen's avatar Renán Sosa Guillen
Browse files
Options

crawl rss

parent ca85aaa9
Hide whitespace changes
Inline Side-by-side
Showing with 92 additions and 6 deletions
crawler_script/crawl_rss.py
View file @ 5c3ec077
......@@ -4,10 +4,12 @@ import sys
import json
import os
import datetime
from collections import OrderedDict
today = datetime.datetime.now()
baseDir = "/home/geoint/virtualHDD/m3/noticias/"
scrapyDir = "/home/geoint/crawlersNoticias/"
row = {}
with open(sys.argv[1]) as data_file:
siteList = json.load(data_file)
os.chdir(baseDir)
......@@ -84,16 +86,58 @@ with open(sys.argv[1]) as data_file:
for line in master:
counter += 1
if media == 'elFinanciero':
row = OrderedDict([
('date', line['date']),
('topic', line['topic']),
('title', line['title']),
('author', line['author']),
('url', line['url']),
('text', line['text'])
])
elif media == 'elUniversal':
row = OrderedDict([
('date', line['date']),
('topic', line['topic']),
('title', line['title']),
('author', line['author']),
('location', line['location']),
('url', line['url']),
('text', line['text'])
])
if counter == 1:
infile3.write(json.dumps(line))
infile3.write(json.dumps(row))
elif counter > 1:
infile3.write(',\n' + json.dumps(line))
infile3.write(',\n' + json.dumps(row))
for line in slave:
if not line['url'] in urlSet:
lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d')
if lineDate == currentDate:
infile3.write(',\n' + json.dumps(line))
if media == 'elFinanciero':
row = OrderedDict([
('date', line['date']),
('topic', line['topic']),
('title', line['title']),
('author', line['author']),
('url', line['url']),
('text', line['text'])
])
elif media == 'elUniversal':
row = OrderedDict([
('date', line['date']),
('topic', line['topic']),
('title', line['title']),
('author', line['author']),
('location', line['location']),
('url', line['url']),
('text', line['text'])
])
infile3.write(',\n' + json.dumps(row))
elif (currentDate - lineDate).days == 1:
YESTERDAY = True
......@@ -119,15 +163,57 @@ with open(sys.argv[1]) as data_file:
for line in master:
counter += 1
if media == 'elFinanciero':
row = OrderedDict([
('date', line['date']),
('topic', line['topic']),
('title', line['title']),
('author', line['author']),
('url', line['url']),
('text', line['text'])
])
elif media == 'elUniversal':
row = OrderedDict([
('date', line['date']),
('topic', line['topic']),
('title', line['title']),
('author', line['author']),
('location', line['location']),
('url', line['url']),
('text', line['text'])
])
if counter == 1:
infile3.write(json.dumps(line))
infile3.write(json.dumps(row))
elif counter > 1:
infile3.write(',\n' + json.dumps(line))
infile3.write(',\n' + json.dumps(row))
for line in slave:
lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d')
if not line['url'] in urlSet and lineDate == currentDate:
infile3.write(',\n' + json.dumps(line))
if media == 'elFinanciero':
row = OrderedDict([
('date', line['date']),
('topic', line['topic']),
('title', line['title']),
('author', line['author']),
('url', line['url']),
('text', line['text'])
])
elif media == 'elUniversal':
row = OrderedDict([
('date', line['date']),
('topic', line['topic']),
('title', line['title']),
('author', line['author']),
('location', line['location']),
('url', line['url']),
('text', line['text'])
])
infile3.write(',\n' + json.dumps(row))
infile3.write(']')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment