crawl rss

crawler_script/crawl_rss.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+import sys
+import json
+import os
+import datetime
+
+today = datetime.datetime.now()
+baseDir = "/home/geoint/virtualHDD/m3/noticias/"
+scrapyDir = "/home/geoint/crawlersNoticias/"
+with open(sys.argv[1]) as data_file:
+    siteList = json.load(data_file)
+    os.chdir(baseDir)
+
+    for s in siteList:
+        # desde = datetime.datetime.strptime(s['desde'], '%d-%m-%Y')
+        desde = today
+        print str(s['nombre'] + ", desde:" + desde.strftime("%Y-%m-%d"))
+        media = s['crawler'][s['crawler'].find("/") + 1:]
+        try:
+            os.makedirs(media)
+        except:
+            print "ok"
+        os.chdir(media)
+
+        lstYears = os.listdir(".")
+        lstYears.sort()
+        year = desde.year
+        if len(lstYears) > 0:
+            year = int(lstYears[len(lstYears) - 1])
+
+        for y in range(year, today.year + 1):
+            print y
+            try:
+                os.makedirs(str(y))
+            except:
+                print "ok"
+
+            os.chdir(str(y))
+            #			print os.getcwd()
+
+            lstDays = os.listdir(".")
+            lstDays = [l for l in lstDays if not l.startswith('.')]
+            lstDays.sort()
+            print lstDays
+
+            day = desde.timetuple().tm_yday
+            print day
+            currentDate = desde
+            if len(lstDays) > 0:
+                strDate = lstDays[len(lstDays) - 1]
+                strDate = strDate[:strDate.find(".")]
+                currentDate = datetime.datetime.strptime(strDate, '%Y-%m-%d')
+                day = currentDate.timetuple().tm_yday
+            elif y != desde.year:
+                currentDate = datetime.datetime.strptime(str(y) + "-01-01", '%Y-%m-%d')
+                day = 1
+
+            for d in range(day, ((datetime.date(y, 12, 31) - datetime.date(y, 1, 1)).days + 1 if today.year != y else today.timetuple().tm_yday) + 1):
+                filename = currentDate.strftime('%Y-%m-%d') + ".json"
+                scrapycommand = "scrapy crawl noticias -t json --nolog -o " + filename
+                mydir = os.getcwd()
+                print mydir
+                os.chdir(scrapyDir + s['crawler'])
+                print media
+                print scrapycommand
+                os.system(scrapycommand)
+                fileSize = os.stat(filename).st_size
+                if fileSize <= 3:
+                    os.system('rm ' + filename)
+                else:
+                    YESTERDAY = False
+
+                    f1 = mydir + '/' + filename
+                    f2 = filename
+                    f3 = baseDir + media + '/' + filename
+                    with open(f1) as infile1, open(f2) as infile2, open(f3, 'a') as infile3:
+                        master = json.load(infile1)
+                        slave = json.load(infile2)
+
+                        urlSet = set([line['url'] for line in master])
+                        counter = 0
+                        infile3.write('[')
+
+                        for line in master:
+                            counter += 1
+                            if counter == 1:
+                                infile3.write(json.dumps(line))
+                            elif counter > 1:
+                                infile3.write(',\n' + json.dumps(line))
+
+                        for line in slave:
+                            if not line['url'] in urlSet:
+                                lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d')
+                                if lineDate == currentDate:
+                                    infile3.write(',\n' + json.dumps(line))
+                                elif (currentDate - lineDate).days == 1:
+                                    YESTERDAY = True
+
+                        infile3.write(']')
+
+                    os.system("mv " + f3 + " " + mydir)
+                    # os.system("rm " + f2)
+
+                    if YESTERDAY:
+                        currentDate -= datetime.timedelta(days=1)
+                        filenameYesterday = currentDate.strftime('%Y-%m-%d') + ".json"
+
+                        f1 = mydir + '/' + filenameYesterday
+                        f2 = filename
+                        f3 = baseDir + media + '/' + filenameYesterday
+                        with open(f1) as infile1, open(f2) as infile2, open(f3, 'a') as infile3:
+                            master = json.load(infile1)
+                            slave = json.load(infile2)
+
+                            urlSet = set([line['url'] for line in master])
+                            counter = 0
+                            infile3.write('[')
+
+                            for line in master:
+                                counter += 1
+                                if counter == 1:
+                                    infile3.write(json.dumps(line))
+                                elif counter > 1:
+                                    infile3.write(',\n' + json.dumps(line))
+
+                            for line in slave:
+                                lineDate = datetime.datetime.strptime(line['date'][:10], '%Y-%m-%d')
+                                if not line['url'] in urlSet and lineDate == currentDate:
+                                    infile3.write(',\n' + json.dumps(line))
+
+                            infile3.write(']')
+
+                        os.system("mv " + f3 + " " + mydir)
+                    os.system("rm " + f2)
+
+                os.chdir(mydir)
+                if YESTERDAY:
+                    currentDate += datetime.timedelta(days=2)
+                else:
+                    currentDate += datetime.timedelta(days=1)
+
+            os.chdir("..")
+
+        os.chdir("..")
+print today.year
+#	scrapy crawl noticias -t json -o $y-$m-$d.json -a year=$y -a month=$m -a day=$d      # ejecucion del crawler correspondiente segun el sitio
\ No newline at end of file