version ordenada

parse_date_files2.py

+import json, os, sys
+from datetime import datetime
+from collections import OrderedDict
+
+"""
+Uso:
+python parse_date_files.py <nombre_del_crawler>
+
+Ej.
+python parse_date_files.py laJornadaBC2
+"""
+
+def dictRowGenerator(line):
+    row = []
+
+    try:
+        row.append(("date", line['date']))
+    except:
+        pass
+    try:
+        row.append(("topic", line['topic']))
+    except:
+        pass
+    try:
+        row.append(("title", line['title']))
+    except:
+        pass
+    try:
+        row.append(("author", line['author']))
+    except:
+        pass
+    try:
+        row.append(("location", line['location']))
+    except:
+        pass
+    try:
+        row.append(("url", line['url']))
+    except:
+        pass
+    try:
+        row.append(("text", line['text']))
+    except:
+        pass
+
+    return row
+
+
+download_type = 'descarga_hacia_atras'
+this_file_path = os.path.dirname(os.path.realpath(__file__))
+json_file_path = this_file_path+'/'+download_type+'/'+sys.argv[1]
+destination_path = this_file_path+'/'+sys.argv[1]
+
+json_file = json.loads(open(json_file_path+'/noticias.json').read())
+date_set = set()
+urlSet = set()
+
+for news in json_file:
+    if news['date'] is not None:
+        news_date = news['date'][:news['date'].rfind('T')]
+        if len(news_date) > 10:
+            news_date = news['date'][:news['date'].rfind(' ')]
+
+        if not news_date in date_set:
+            date_set.add(news_date)
+            print(news_date)
+
+            try:
+                export_file = open(destination_path+'/'+news_date[:4]+'/'+news_date+'.json', 'a')
+            except:
+                os.makedirs(destination_path+'/'+news_date[:4])
+                export_file = open(destination_path+'/'+news_date[:4]+'/'+news_date+'.json', 'a')
+
+            counter = 0
+            export_file.write("[")
+
+            for line in json_file:
+                if line['date'] is not None:
+                    line_date = line['date'][:line['date'].rfind('T')]
+                    if len(line_date) > 10:
+                        line_date = line['date'][:line['date'].rfind(' ')]
+
+                    if not line['url'] in urlSet and line_date == news_date:
+                        urlSet.add(line['url'])
+                        counter += 1
+
+                        auxRow = dictRowGenerator(line)
+                        row = OrderedDict(auxRow)
+                        if counter == 1:
+                            export_file.write(json.dumps(row))
+                        elif counter > 1:
+                            export_file.write(",\n" + json.dumps(row))
+
+            export_file.write("]")
+            export_file.close()
+