crawl all

3a860c4d · Renán Sosa Guillen · d45d61be · 3a860c4d
Commit 3a860c4d authored Oct 05, 2017 by Renán Sosa Guillen
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 1 deletion

crawl_all.py crawler_script/crawl_all.py +3 -1

No files found.
--- a/crawler_script/crawl_all.py
+++ b/crawler_script/crawl_all.py
@@ -62,7 +62,9 @@ with open(sys.argv[1]) as data_file:
 				os.chdir(scrapyDir+s['crawler'])
 				print scrapycommand
 				os.system(scrapycommand)
-				os.system("mv " + filename + " " + mydir)
+				fileSize = os.stat(filename).st_size
+				if fileSize <= 3: os.system('rm '+filename)
+				else: os.system("mv " + filename + " " + mydir)
 				os.chdir(mydir)
 				currentDate = currentDate + datetime.timedelta(days=1)