Batch upload

catalog/management/commands/loadNews.py

@@ -83,6 +83,7 @@ class Command(BaseCommand):
 								try:
 									print(f)
 									data = json.load(data_file)
+									objectsList = []
 									for d in data:
 										newsDate = dateutil.parser.parse(d['date'])
 										#check for repeted news
@@ -119,7 +120,16 @@ class Command(BaseCommand):
 												topicstr = [d['topic']]
 											
 											news.topic_raw = json.domps(topicst)
-											news.save()
+#											news.save()
+											objectsList.append(news)
+								
+								from itertools import islice
+								batch_size = 100
+								while True:
+									batch = list(islice(objectsList, batch_size))
+									if not batch:
+										break
+									News.objects.bulk_create(batch, batch_size)
 
 								except ValueError as e:
 									print (e)

catalog/models.py

@@ -50,11 +50,13 @@ class News(models.Model):
 	title = models.CharField(max_length=512)
 	publisher = models.ForeignKey(Publisher, on_delete=models.CASCADE, default='')
 	topic = models.ManyToManyField(Topic, verbose_name='Topic(s)', blank=True)
+	topic_raw = models.CharField(max_length=512, default='')
 	date = models.DateTimeField(verbose_name='Publication Date')
 	text = TextField(verbose_name='News Text', default='', blank=True)
 	url = models.URLField(verbose_name='URL', max_length=512)
 	search_vector = SearchVectorField(null=True)
 	objects = PostManager()
+	
 	def save(self, *args, **kwargs):
 		super(News, self).save(*args, **kwargs)
 		if 'update_fields' not in kwargs or 'search_vector' not in kwargs['update_fields']: