Skip to content

crawlersNoticias
crawlersNoticias
Commit 10c46676 authored by Renán Sosa Guillen's avatar Renán Sosa Guillen
Browse files
Options

crawl rss

parent 835f6586
Show whitespace changes
Inline Side-by-side
Showing with 3 additions and 3 deletions
crawler_script/crawl_rss.py
View file @ 10c46676
...@@ -90,7 +90,7 @@ with open(sys.argv[1]) as data_file: ...@@ -90,7 +90,7 @@ with open(sys.argv[1]) as data_file:
if lineDate == currentDate: if lineDate == currentDate:
counter += 1 counter += 1
if media == 'elFinanciero': if media == 'elFinanciero' or media == 'solDeMex':
row = OrderedDict([ row = OrderedDict([
('date', line['date']), ('date', line['date']),
('topic', line['topic']), ('topic', line['topic']),
...@@ -169,7 +169,7 @@ with open(sys.argv[1]) as data_file: ...@@ -169,7 +169,7 @@ with open(sys.argv[1]) as data_file:
for line in master: for line in master:
counter += 1 counter += 1
if media == 'elFinanciero': if media == 'elFinanciero' or media == 'solDeMex':
row = OrderedDict([ row = OrderedDict([
('date', line['date']), ('date', line['date']),
('topic', line['topic']), ('topic', line['topic']),
...@@ -198,7 +198,7 @@ with open(sys.argv[1]) as data_file: ...@@ -198,7 +198,7 @@ with open(sys.argv[1]) as data_file:
lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d') lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d')
if not line['url'] in urlSet and lineDate == currentDate: if not line['url'] in urlSet and lineDate == currentDate:
if media == 'elFinanciero': if media == 'elFinanciero' or media == 'solDeMex':
row = OrderedDict([ row = OrderedDict([
('date', line['date']), ('date', line['date']),
('topic', line['topic']), ('topic', line['topic']),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment