script

crawler_script/crawl_all.py

@@ -53,7 +53,7 @@ with open(sys.argv[1]) as data_file:
 				currentDate = datetime.datetime.strptime(str(y)+"-01-01", '%Y-%m-%d')
 				day = 1
 
-			for d in range(day, 365+1):
+			for d in range(day, (365 if today.year!=y else today.timetuple().tm_yday)+1):
 				filename = currentDate.strftime('%Y-%m-%d')+".json"
 				scrapycommand = "scrapy crawl noticias -t json -o " + filename + " -a year="+str(currentDate.year)+ " -a month="+str(currentDate.month)+" -a day="+str(currentDate.day)
 				mydir =  os.getcwd()

crawler_script/crawler_data.json

-[
-{"nombre": "alChile", "crawler": "sitios_yucatan/alChile"},
-{"nombre": "desdeElBalcon", "crawler": "sitios_yucatan/desdeElBalcon"},
-{"nombre": "diarioYucatan", "crawler": "sitios_yucatan/diarioYucatan"},
-{"nombre": "grilloPorteno", "crawler": "sitios_yucatan/grilloPorteno"},
-{"nombre": "laJornadaMaya", "crawler": "sitios_yucatan/alChile"},
-{"nombre": "laVerdadYuc", "crawler": "sitios_yucatan/laVerdadYuc"},
-{"nombre": "lectorMX", "crawler": "sitios_yucatan/lectorMX"},
-{"nombre": "miPuntoDeVista", "crawler": "sitios_yucatan/miPuntoDeVista"},
-{"nombre": "notirivas", "crawler": "sitios_yucatan/notirivas"},
-{"nombre": "notisureste", "crawler": "sitios_yucatan/notisureste"},
-{"nombre": "puntoMedio", "crawler": "sitios_yucatan/puntoMedio"},
-{"nombre": "sona893", "crawler": "sitios_yucatan/sona893"},
-{"nombre": "yucatanALaMano", "crawler": "sitios_yucatan/yucatanALaMano"},
-{"nombre": "yucatanAlMinuto", "crawler": "sitios_yucatan/yucatanAlMinuto"},
-{"nombre": "yucatanEnCorto", "crawler": "sitios_yucatan/yucatanEnCorto"},
-{"nombre": "diarioYaqui", "crawler": "otros_sitios/diarioYaqui"},
-{"nombre": "laJornada", "crawler": "otros_sitios/laJornada"},
-{"nombre": "laJornadaAgs", "crawler": "otros_sitios/laJornadaAgs"},
-{"nombre": "laJornadaBC", "crawler": "otros_sitios/laJornadaBC"},
-{"nombre": "laJornadaGro", "crawler": "otros_sitios/laJornadaGro"},
-{"nombre": "laJornadaOte", "crawler": "otros_sitios/laJornadaOte"},
-{"nombre": "laJornadaSanLuis", "crawler": "otros_sitios/laJornadaSanLuis"},
-{"nombre": "laJornadaVer", "crawler": "otros_sitios/laJornadaVer"},
-{"nombre": "laJornadaZac", "crawler": "otros_sitios/laJornadaZac"}
-]
\ No newline at end of file