Skip to content

crawlersNoticias
crawlersNoticias
Commit 20632ff9 authored by Renán Sosa Guillen's avatar Renán Sosa Guillen
Browse files
Options

crawl rss

parent 5c3ec077
Hide whitespace changes
Inline Side-by-side
Showing with 41 additions and 32 deletions
crawler_script/crawl_rss.py
View file @ 20632ff9
...@@ -10,6 +10,7 @@ today = datetime.datetime.now() ...@@ -10,6 +10,7 @@ today = datetime.datetime.now()
baseDir = "/home/geoint/virtualHDD/m3/noticias/" baseDir = "/home/geoint/virtualHDD/m3/noticias/"
scrapyDir = "/home/geoint/crawlersNoticias/" scrapyDir = "/home/geoint/crawlersNoticias/"
row = {} row = {}
urlSet = set()
with open(sys.argv[1]) as data_file: with open(sys.argv[1]) as data_file:
siteList = json.load(data_file) siteList = json.load(data_file)
os.chdir(baseDir) os.chdir(baseDir)
...@@ -77,45 +78,50 @@ with open(sys.argv[1]) as data_file: ...@@ -77,45 +78,50 @@ with open(sys.argv[1]) as data_file:
f2 = filename f2 = filename
f3 = baseDir + media + '/' + filename f3 = baseDir + media + '/' + filename
with open(f1) as infile1, open(f2) as infile2, open(f3, 'a') as infile3: with open(f1) as infile1, open(f2) as infile2, open(f3, 'a') as infile3:
master = json.load(infile1)
slave = json.load(infile2)
urlSet = set([line['url'] for line in master])
counter = 0 counter = 0
slave = json.load(infile2)
infile3.write('[') infile3.write('[')
for line in master: try:
counter += 1 master = json.load(infile1)
if media == 'elFinanciero': urlSet = set([line['url'] for line in master])
row = OrderedDict([
('date', line['date']), for line in master:
('topic', line['topic']), counter += 1
('title', line['title']),
('author', line['author']), if media == 'elFinanciero':
('url', line['url']), row = OrderedDict([
('text', line['text']) ('date', line['date']),
]) ('topic', line['topic']),
elif media == 'elUniversal': ('title', line['title']),
row = OrderedDict([ ('author', line['author']),
('date', line['date']), ('url', line['url']),
('topic', line['topic']), ('text', line['text'])
('title', line['title']), ])
('author', line['author']), elif media == 'elUniversal':
('location', line['location']), row = OrderedDict([
('url', line['url']), ('date', line['date']),
('text', line['text']) ('topic', line['topic']),
]) ('title', line['title']),
('author', line['author']),
if counter == 1: ('location', line['location']),
infile3.write(json.dumps(row)) ('url', line['url']),
elif counter > 1: ('text', line['text'])
infile3.write(',\n' + json.dumps(row)) ])
if counter == 1:
infile3.write(json.dumps(row))
elif counter > 1:
infile3.write(',\n' + json.dumps(row))
except IOError:
pass
for line in slave: for line in slave:
if not line['url'] in urlSet: if not line['url'] in urlSet:
lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d') lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d')
if lineDate == currentDate: if lineDate == currentDate:
counter += 1
if media == 'elFinanciero': if media == 'elFinanciero':
row = OrderedDict([ row = OrderedDict([
...@@ -137,7 +143,10 @@ with open(sys.argv[1]) as data_file: ...@@ -137,7 +143,10 @@ with open(sys.argv[1]) as data_file:
('text', line['text']) ('text', line['text'])
]) ])
infile3.write(',\n' + json.dumps(row)) if counter == 1:
infile3.write(json.dumps(row))
elif counter > 1:
infile3.write(',\n' + json.dumps(row))
elif (currentDate - lineDate).days == 1: elif (currentDate - lineDate).days == 1:
YESTERDAY = True YESTERDAY = True
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment