crawl rss

crawler_script/crawl_rss.py

@@ -10,6 +10,7 @@ today = datetime.datetime.now()
 baseDir = "/home/geoint/virtualHDD/m3/noticias/"
 scrapyDir = "/home/geoint/crawlersNoticias/"
 row = {}
+urlSet = set()
 with open(sys.argv[1]) as data_file:
     siteList = json.load(data_file)
     os.chdir(baseDir)
@@ -77,45 +78,50 @@ with open(sys.argv[1]) as data_file:
                     f2 = filename
                     f3 = baseDir + media + '/' + filename
                     with open(f1) as infile1, open(f2) as infile2, open(f3, 'a') as infile3:
-                        master = json.load(infile1)
-                        slave = json.load(infile2)
-
-                        urlSet = set([line['url'] for line in master])
                         counter = 0
+                        slave = json.load(infile2)
                         infile3.write('[')
 
-                        for line in master:
-                            counter += 1
-
-                            if media == 'elFinanciero':
-                                row = OrderedDict([
-                                    ('date', line['date']),
-                                    ('topic', line['topic']),
-                                    ('title', line['title']),
-                                    ('author', line['author']),
-                                    ('url', line['url']),
-                                    ('text', line['text'])
-                                ])
-                            elif media == 'elUniversal':
-                                row = OrderedDict([
-                                    ('date', line['date']),
-                                    ('topic', line['topic']),
-                                    ('title', line['title']),
-                                    ('author', line['author']),
-                                    ('location', line['location']),
-                                    ('url', line['url']),
-                                    ('text', line['text'])
-                                ])
-
-                            if counter == 1:
-                                infile3.write(json.dumps(row))
-                            elif counter > 1:
-                                infile3.write(',\n' + json.dumps(row))
+                        try:
+                            master = json.load(infile1)
+
+                            urlSet = set([line['url'] for line in master])
+
+                            for line in master:
+                                counter += 1
+
+                                if media == 'elFinanciero':
+                                    row = OrderedDict([
+                                        ('date', line['date']),
+                                        ('topic', line['topic']),
+                                        ('title', line['title']),
+                                        ('author', line['author']),
+                                        ('url', line['url']),
+                                        ('text', line['text'])
+                                    ])
+                                elif media == 'elUniversal':
+                                    row = OrderedDict([
+                                        ('date', line['date']),
+                                        ('topic', line['topic']),
+                                        ('title', line['title']),
+                                        ('author', line['author']),
+                                        ('location', line['location']),
+                                        ('url', line['url']),
+                                        ('text', line['text'])
+                                    ])
+
+                                if counter == 1:
+                                    infile3.write(json.dumps(row))
+                                elif counter > 1:
+                                    infile3.write(',\n' + json.dumps(row))
+                        except IOError:
+                            pass
 
                         for line in slave:
                             if not line['url'] in urlSet:
                                 lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d')
                                 if lineDate == currentDate:
+                                    counter += 1
 
                                     if media == 'elFinanciero':
                                         row = OrderedDict([
@@ -137,7 +143,10 @@ with open(sys.argv[1]) as data_file:
                                             ('text', line['text'])
                                         ])
 
-                                    infile3.write(',\n' + json.dumps(row))
+                                    if counter == 1:
+                                        infile3.write(json.dumps(row))
+                                    elif counter > 1:
+                                        infile3.write(',\n' + json.dumps(row))
                                 elif (currentDate - lineDate).days == 1:
                                     YESTERDAY = True