Commit 3a860c4d authored by Renán Sosa Guillen's avatar Renán Sosa Guillen

crawl all

parent d45d61be
...@@ -62,7 +62,9 @@ with open(sys.argv[1]) as data_file: ...@@ -62,7 +62,9 @@ with open(sys.argv[1]) as data_file:
os.chdir(scrapyDir+s['crawler']) os.chdir(scrapyDir+s['crawler'])
print scrapycommand print scrapycommand
os.system(scrapycommand) os.system(scrapycommand)
os.system("mv " + filename + " " + mydir) fileSize = os.stat(filename).st_size
if fileSize <= 3: os.system('rm '+filename)
else: os.system("mv " + filename + " " + mydir)
os.chdir(mydir) os.chdir(mydir)
currentDate = currentDate + datetime.timedelta(days=1) currentDate = currentDate + datetime.timedelta(days=1)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment