crawl rss

crawler_script/crawl_rss.py

@@ -10,7 +10,6 @@ today = datetime.datetime.now()
 baseDir = "/home/geoint/virtualHDD/m3/noticias/"
 scrapyDir = "/home/geoint/crawlersNoticias/"
 row = {}
-urlSet = set()
 with open(sys.argv[1]) as data_file:
     siteList = json.load(data_file)
     os.chdir(baseDir)
@@ -77,15 +76,14 @@ with open(sys.argv[1]) as data_file:
                     f1 = mydir + '/' + filename
                     f2 = filename
                     f3 = baseDir + media + '/' + filename
-                    with open(f1) as infile1, open(f2) as infile2, open(f3, 'a') as infile3:
-                        counter = 0
-                        slave = json.load(infile2)
-                        infile3.write('[')
-
-                        try:
+                    try:
+                        with open(f1) as infile1, open(f2) as infile2, open(f3, 'a') as infile3:
                             master = json.load(infile1)
+                            slave = json.load(infile2)
 
                             urlSet = set([line['url'] for line in master])
+                            counter = 0
+                            infile3.write('[')
 
                             for line in master:
                                 counter += 1
@@ -114,46 +112,42 @@ with open(sys.argv[1]) as data_file:
                                     infile3.write(json.dumps(row))
                                 elif counter > 1:
                                     infile3.write(',\n' + json.dumps(row))
-                        except IOError:
-                            pass
-
-                        for line in slave:
-                            if not line['url'] in urlSet:
-                                lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d')
-                                if lineDate == currentDate:
-                                    counter += 1
 
-                                    if media == 'elFinanciero':
-                                        row = OrderedDict([
-                                            ('date', line['date']),
-                                            ('topic', line['topic']),
-                                            ('title', line['title']),
-                                            ('author', line['author']),
-                                            ('url', line['url']),
-                                            ('text', line['text'])
-                                        ])
-                                    elif media == 'elUniversal':
-                                        row = OrderedDict([
-                                            ('date', line['date']),
-                                            ('topic', line['topic']),
-                                            ('title', line['title']),
-                                            ('author', line['author']),
-                                            ('location', line['location']),
-                                            ('url', line['url']),
-                                            ('text', line['text'])
-                                        ])
+                            for line in slave:
+                                if not line['url'] in urlSet:
+                                    lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d')
+                                    if lineDate == currentDate:
+
+                                        if media == 'elFinanciero':
+                                            row = OrderedDict([
+                                                ('date', line['date']),
+                                                ('topic', line['topic']),
+                                                ('title', line['title']),
+                                                ('author', line['author']),
+                                                ('url', line['url']),
+                                                ('text', line['text'])
+                                            ])
+                                        elif media == 'elUniversal':
+                                            row = OrderedDict([
+                                                ('date', line['date']),
+                                                ('topic', line['topic']),
+                                                ('title', line['title']),
+                                                ('author', line['author']),
+                                                ('location', line['location']),
+                                                ('url', line['url']),
+                                                ('text', line['text'])
+                                            ])
 
-                                    if counter == 1:
-                                        infile3.write(json.dumps(row))
-                                    elif counter > 1:
                                         infile3.write(',\n' + json.dumps(row))
-                                elif (currentDate - lineDate).days == 1:
-                                    YESTERDAY = True
+                                    elif (currentDate - lineDate).days == 1:
+                                        YESTERDAY = True
 
-                        infile3.write(']')
+                            infile3.write(']')
 
-                    os.system("mv " + f3 + " " + mydir)
-                    # os.system("rm " + f2)
+                        os.system("mv " + f3 + " " + mydir)
+                        # os.system("rm " + f2)
+                    except:
+                        os.system("cp " + f2 + " " + mydir)
 
                     if YESTERDAY:
                         currentDate -= datetime.timedelta(days=1)