crawl all

crawler_script/crawl_all.py

@@ -7,10 +7,8 @@ import datetime
 
 
 today = datetime.datetime.now()
-# baseDir = "/home/geoint/virtualHDD/m3/noticias/"
-# scrapyDir = "/home/geoint/crawlersNoticias/"
-baseDir = "/home/cna_service/prueba/"
-scrapyDir = "/home/cna_service/crawler/crawlersNoticias/"
+baseDir = "/home/geoint/virtualHDD/m3/noticias/"
+scrapyDir = "/home/geoint/crawlersNoticias/"
 with open(sys.argv[1]) as data_file:    
     siteList = json.load(data_file)
     os.chdir(baseDir)
@@ -60,7 +58,6 @@ with open(sys.argv[1]) as data_file:
 
             for d in range(day, ((datetime.date(y,12,31)-datetime.date(y,1,1)).days + 1 if today.year!=y else today.timetuple().tm_yday)+1):
                 filename = currentDate.strftime('%Y-%m-%d')+".json"
-                # scrapycommand = "scrapy crawl noticias -t json --nolog -o " + filename + " -a year="+str(currentDate.year)+ " -a month="+str(currentDate.month)+" -a day="+str(currentDate.day)
                 scrapycommand = "scrapy crawl noticias --nolog -s filename=" + filename + " -a year="+str(currentDate.year)+ " -a month="+str(currentDate.month)+" -a day="+str(currentDate.day)
                 mydir = os.getcwd()
                 print mydir
@@ -77,5 +74,5 @@ with open(sys.argv[1]) as data_file:
             os.chdir("..")
 
         os.chdir("..")
-print today.year
+# print today.year
 #	scrapy crawl noticias -t json -o $y-$m-$d.json -a year=$y -a month=$m -a day=$d      # ejecucion del crawler correspondiente segun el sitio
\ No newline at end of file