Skip to content

crawlersNoticias
crawlersNoticias
Commit 2c81f7be authored by Renán Sosa Guillen's avatar Renán Sosa Guillen
Browse files
Options

crawl rss

parent 82c3a777
Hide whitespace changes
Inline Side-by-side
Showing with 26 additions and 24 deletions
crawler_script/crawl_rss.py
View file @ 2c81f7be
......@@ -87,31 +87,33 @@ with open(sys.argv[1]) as data_file:
for line in master:
counter += 1
lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d')
if lineDate == currentDate:
if media == 'elFinanciero':
row = OrderedDict([
('date', line['date']),
('topic', line['topic']),
('title', line['title']),
('author', line['author']),
('url', line['url']),
('text', line['text'])
])
elif media == 'elUniversal':
row = OrderedDict([
('date', line['date']),
('topic', line['topic']),
('title', line['title']),
('author', line['author']),
('location', line['location']),
('url', line['url']),
('text', line['text'])
])
if media == 'elFinanciero':
row = OrderedDict([
('date', line['date']),
('topic', line['topic']),
('title', line['title']),
('author', line['author']),
('url', line['url']),
('text', line['text'])
])
elif media == 'elUniversal':
row = OrderedDict([
('date', line['date']),
('topic', line['topic']),
('title', line['title']),
('author', line['author']),
('location', line['location']),
('url', line['url']),
('text', line['text'])
])
if counter == 1:
infile3.write(json.dumps(row))
elif counter > 1:
infile3.write(',\n' + json.dumps(row))
if counter == 1:
infile3.write(json.dumps(row))
elif counter > 1:
infile3.write(',\n' + json.dumps(row))
for line in slave:
if not line['url'] in urlSet:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment