crawl rss

crawler_script/crawl_rss.py

@@ -24,13 +24,20 @@ with open(sys.argv[1]) as data_file:
         except:
             print "ok"
         os.chdir(media)
+        CORRECT_YEAR = False
 
+        while not CORRECT_YEAR:
             lstYears = os.listdir(".")
             lstYears.sort()
             year = desde.year
             if len(lstYears) > 0:
-            year = int(lstYears[len(lstYears) - 1])
+                element = lstYears[len(lstYears) - 1]
+                if element[-4:] == 'json':
+                    os.system('rm ' + element)
+                else:
+                    CORRECT_YEAR = True
                 
+        year = int(element)
         for y in range(year, today.year + 1):
             print y
             try:
@@ -59,6 +66,7 @@ with open(sys.argv[1]) as data_file:
                 day = 1
 
             for d in range(day, ((datetime.date(y, 12, 31) - datetime.date(y, 1, 1)).days + 1 if today.year != y else today.timetuple().tm_yday) + 1):
+                YESTERDAY = False
                 filename = currentDate.strftime('%Y-%m-%d') + ".json"
                 scrapycommand = "scrapy crawl noticias -t json --nolog -o " + filename
                 mydir = os.getcwd()
@@ -71,8 +79,6 @@ with open(sys.argv[1]) as data_file:
                 if fileSize <= 3:
                     os.system('rm ' + filename)
                 else:
-                    YESTERDAY = False
-
                     f1 = mydir + '/' + filename
                     f2 = filename
                     f3 = baseDir + media + '/' + filename