crawl all

crawler_script/crawl_all.py

@@ -23,40 +23,41 @@ with open(sys.argv[1]) as data_file:
 
 		lstYears = os.listdir(".")
 		lstYears.sort()
-		year  = desde.year
-		if len(lstYears)>0:
+		year = desde.year
+		if len(lstYears) > 0:
 			year = int(lstYears[len(lstYears)-1])
+		
 		for y in range(year, today.year+1):
 			print y
 			try:
-                        	os.makedirs(str(y))
-	                except:
-                        	print "ok"
+				os.makedirs(str(y))
+			except:
+				print "ok"
 
 			os.chdir(str(y))
 #			print os.getcwd()
 
- 			lstDays = os.listdir(".")
+			lstDays = os.listdir(".")
 			lstDays = [l for l in lstDays if not l.startswith('.')]
-                	lstDays.sort()
-                	print lstDays
+			lstDays.sort()
+			print lstDays
 
 			day = desde.timetuple().tm_yday
 			print day
 			currentDate = desde
-                	if len(lstDays)>0:
+			if len(lstDays) > 0:
 				strDate = lstDays[len(lstDays)-1]
 				strDate = strDate[:strDate.find(".")]				
-				currentDate =  datetime.datetime.strptime(strDate, '%Y-%m-%d')
-	                        day = currentDate.timetuple().tm_yday
-			elif y!=desde.year:
+				currentDate = datetime.datetime.strptime(strDate, '%Y-%m-%d')
+				day = currentDate.timetuple().tm_yday
+			elif y != desde.year:
 				currentDate = datetime.datetime.strptime(str(y)+"-01-01", '%Y-%m-%d')
 				day = 1
 
-			for d in range(day, (365 if today.year!=y else today.timetuple().tm_yday)+1):
+			for d in range(day, ((datetime.date(y,12,31)-datetime.date(y,1,1)).days + 1 if today.year!=y else today.timetuple().tm_yday)+1):
 				filename = currentDate.strftime('%Y-%m-%d')+".json"
 				scrapycommand = "scrapy crawl noticias -t json -o " + filename + " -a year="+str(currentDate.year)+ " -a month="+str(currentDate.month)+" -a day="+str(currentDate.day)
-				mydir =  os.getcwd()
+				mydir = os.getcwd()
 				print mydir
 				os.chdir(scrapyDir+s['crawler'])
 				print scrapycommand