bulk upload

catalog/management/commands/loadNews.py

@@ -79,64 +79,64 @@ class Command(BaseCommand):
 						fileDate = datetime.datetime.strptime(f[:f.find(".")], "%Y-%m-%d").date()
 
 #						if fileDate >= lastDate.date():
-							with open(f) as data_file:
-								try:
-									print(f)
-									data = json.load(data_file)
-									objectsList = []
-									for d in data:
-										newsDate = dateutil.parser.parse(d['date'])
-										#check for repeted news
+						with open(f) as data_file:
+							try:
+								print(f)
+								data = json.load(data_file)
+								objectsList = []
+								for d in data:
+									newsDate = dateutil.parser.parse(d['date'])
+									#check for repeted news
 #										if News.objects.all().filter(Q(publisher=publisher.id)&Q(title=d['title'])&Q(date__gte=newsDate)).count() == 0:
 
-											if d['title'] == None:
-												d['title'] = "Sin Titulo"
-											if len(d['title']) >= 512:
-												d['title'] = d['title'][:500]
-											print (p + " "+  str(newsDate) +  ": " + d['title'])
-											news = News()
-											news.publisher = publisher
-											news.title = d['title']
-											news.text = d['text']
-											news.url = d['url']
-											news.date = newsDate
+									if d['title'] == None:
+										d['title'] = "Sin Titulo"
+									if len(d['title']) >= 512:
+										d['title'] = d['title'][:500]
+									print (p + " "+  str(newsDate) +  ": " + d['title'])
+									news = News()
+									news.publisher = publisher
+									news.title = d['title']
+									news.text = d['text']
+									news.url = d['url']
+									news.date = newsDate
 
 
 #											print ("topic type:",  type(d['topic']))
 #											print (d['topic'])
-											topicstr
-											if "topic" not in d or d['topic'] == "" or d['topic'] == None or d['topic'] ==[]:
-												topicstr = ["Sin Tema"]
+									topicstr
+									if "topic" not in d or d['topic'] == "" or d['topic'] == None or d['topic'] ==[]:
+										topicstr = ["Sin Tema"]
 
-											if type(d['topic'])==list:
-												topicstr = d['topic']
+									if type(d['topic'])==list:
+										topicstr = d['topic']
 #												for t in d['topic']:
 ##													topic, created = Topic.objects.all().get_or_create(name=t)
 #													topic, created = Topic.objects.get_or_create(name=t)
 #													news.topic.add(topic)
-											else:
+									else:
 #												topic, created = Topic.objects.get_or_create(name=d['topic'])
 #												news.topic.add(topic)
-												topicstr = [d['topic']]
-											
-											news.topic_raw = json.domps(topicst)
+										topicstr = [d['topic']]
+									
+									news.topic_raw = json.domps(topicst)
 #											news.save()
-											objectsList.append(news)
+									objectsList.append(news)
+							
+							from itertools import islice
+							batch_size = 100
+							while True:
+								batch = list(islice(objectsList, batch_size))
+								if not batch:
+									break
+								News.objects.bulk_create(batch, batch_size)
+
+							except ValueError as e:
+								print (e)
+								print ("Error: " + os.getcwd()+"/"+f)
+								print (type(d['topic']))
 								
-								from itertools import islice
-								batch_size = 100
-								while True:
-									batch = list(islice(objectsList, batch_size))
-									if not batch:
-										break
-									News.objects.bulk_create(batch, batch_size)
-
-								except ValueError as e:
-									print (e)
-									print ("Error: " + os.getcwd()+"/"+f)
-									print (type(d['topic']))
-									
-								data_file.close()
+							data_file.close()
 
 					os.chdir("..")
 			os.chdir("..")