bulk upload

catalog/management/commands/loadNews.py

@@ -53,20 +53,20 @@ class Command(BaseCommand):
 				publisher=publisher[0]
 				
 			#----------------------------------------------------------------
-#			news = News.objects.all().filter(publisher=publisher.id).order_by("-date")
-#			minYear = 0
-#			lastDate = datetime.datetime(1950,1,1)
+			news = News.objects.all().filter(publisher=publisher.id).order_by("-date")
+			minYear = 0
+			lastDate = datetime.datetime(1950,1,1)
 
-#			if news.count()>0:
-#				minYear = news[0].date.year
-#				lastDate = news[0].date
+			if news.count()>0:
+				minYear = news[0].date.year
+				lastDate = news[0].date
 			
 			# Year list from directory structure
 			yearList = sorted([ int(y) for y in os.listdir('.') if os.path.isdir(y) ])
 
 			print (yearList)
 			for y in sorted(yearList):
-#				if y >=minYear:
+				if y >=minYear:
 					os.chdir(str(y))
 					print (os.getcwd())
 
@@ -75,8 +75,10 @@ class Command(BaseCommand):
 					
 					for f in filesList:
 						fileDate = datetime.datetime.strptime(f[:f.find(".")], "%Y-%m-%d").date()
-
-#						if fileDate >= lastDate.date():
+						print(fileDate)
+						if fileDate >= lastDate.date():
+							#delete news form the most recent day in the database so it can upload the day again
+							News.objects.filter(date__gte=fileDate).delete()
 							with open(f) as data_file:
 								try:
 									print(f)
@@ -85,7 +87,7 @@ class Command(BaseCommand):
 									for d in data:
 										newsDate = dateutil.parser.parse(d['date'])
 										#check for repeted news
-#										if News.objects.all().filter(Q(publisher=publisher.id)&Q(title=d['title'])&Q(date__gte=newsDate)).count() == 0:
+	#										if News.objects.all().filter(Q(publisher=publisher.id)&Q(title=d['title'])&Q(date__gte=newsDate)).count() == 0:
 
 										if d['title'] == None:
 											d['title'] = "Sin Titulo"
@@ -100,24 +102,24 @@ class Command(BaseCommand):
 										news.date = newsDate
 
 
-#											print ("topic type:",  type(d['topic']))
-#											print (d['topic'])
+	#											print ("topic type:",  type(d['topic']))
+	#											print (d['topic'])
 										if "topic" not in d or d['topic'] == "" or d['topic'] == None or d['topic'] ==[]:
 											topicstr = ["Sin Tema"]
 
 										if type(d['topic'])==list:
 											topicstr = d['topic']
-#												for t in d['topic']:
-##													topic, created = Topic.objects.all().get_or_create(name=t)
-#													topic, created = Topic.objects.get_or_create(name=t)
-#													news.topic.add(topic)
+	#												for t in d['topic']:
+	##													topic, created = Topic.objects.all().get_or_create(name=t)
+	#													topic, created = Topic.objects.get_or_create(name=t)
+	#													news.topic.add(topic)
 										else:
-#												topic, created = Topic.objects.get_or_create(name=d['topic'])
-#												news.topic.add(topic)
+	#												topic, created = Topic.objects.get_or_create(name=d['topic'])
+	#												news.topic.add(topic)
 											topicstr = [d['topic']]
 										
 										news.topic_raw = json.dumps(topicstr)
-#									news.save()
+	#									news.save()
 										objectsList.append(news)
 									
 									News.objects.bulk_create(objectsList)