crawl rss

crawler_script/crawl_rss.py

@@ -10,6 +10,7 @@ today = datetime.datetime.now()
 baseDir = "/home/geoint/virtualHDD/m3/noticias/"
 scrapyDir = "/home/geoint/crawlersNoticias/"
 row = {}
+urlSet = set()
 with open(sys.argv[1]) as data_file:
     siteList = json.load(data_file)
     os.chdir(baseDir)
@@ -77,12 +78,14 @@ with open(sys.argv[1]) as data_file:
                     f2 = filename
                     f3 = baseDir + media + '/' + filename
                     with open(f1) as infile1, open(f2) as infile2, open(f3, 'a') as infile3:
-                        master = json.load(infile1)
+                        counter = 0
                         slave = json.load(infile2)
+                        infile3.write('[')
+
+                        try:
+                            master = json.load(infile1)
 
                             urlSet = set([line['url'] for line in master])
-                        counter = 0
-                        infile3.write('[')
 
                             for line in master:
                                 counter += 1
@@ -111,11 +114,14 @@ with open(sys.argv[1]) as data_file:
                                     infile3.write(json.dumps(row))
                                 elif counter > 1:
                                     infile3.write(',\n' + json.dumps(row))
+                        except IOError:
+                            pass
 
                         for line in slave:
                             if not line['url'] in urlSet:
                                 lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d')
                                 if lineDate == currentDate:
+                                    counter += 1
 
                                     if media == 'elFinanciero':
                                         row = OrderedDict([
@@ -137,6 +143,9 @@ with open(sys.argv[1]) as data_file:
                                             ('text', line['text'])
                                         ])
 
+                                    if counter == 1:
+                                        infile3.write(json.dumps(row))
+                                    elif counter > 1:
                                         infile3.write(',\n' + json.dumps(row))
                                 elif (currentDate - lineDate).days == 1:
                                     YESTERDAY = True