from django.core.management.base import BaseCommand, CommandError
from catalog.models import News, Publisher, Topic, audioTime
from django.db.models import Q
import os
import glob
import json
import datetime
from django.utils import timezone
import dateutil.parser
import itertools
from datetime import date
from itertools import islice
updateRadioStations = False
recordingsDir = "/home/geoint/M3_NFS/recordings/"


class Command(BaseCommand):
	help = 'Update database'


	def add_arguments(self, parser):
		parser.add_argument('basedir', nargs=1, type=str)

	def handle(self, *args, **options):
		
		#Load news
		os.chdir(options['basedir'][0])
		#Publisher list form directory structure
		publisherList = [ i for i in os.listdir(options['basedir'][0]) if os.path.isdir(i) ]
		for p in publisherList:
			print (p)
			curentdir = options['basedir'][0]+p+"/"
			os.chdir(curentdir)
			publisher = Publisher.objects.all().filter(shortName=p)
#			publisher = Publisher.objects.filter(shortName=p).getFir
			print(os.getcwd())
			if publisher.count()<=0:
				if os.path.isfile("settings.json") :
					print(p, "do not exsist, crating publisher")
					with open('settings.json', 'r') as f:
						cfgfile = json.load(f)
					newPublisher = Publisher()
					newPublisher.shortName=p
					newPublisher.name=cfgfile["name"]
					newPublisher.crawler=cfgfile["crawler"]
					newPublisher.url=cfgfile["url"]
					newPublisher.type="texto"
					newPublisher.save()
					publisher = Publisher.objects.filter(shortName=p)[0]
				else:
					print("settings.json not found")
					continue
			else:
				publisher=publisher[0]
				
			#----------------------------------------------------------------
			news = News.objects.all().filter(publisher=publisher.id).order_by("-date")
			minYear = 0
			lastDate = datetime.datetime(1950,1,1)
			if news.count()>0:
				minYear = news[0].date.year
				lastDate = news[0].date
				print("LAST DATE ", lastDate)
			# Year list from directory structure
			yearList = sorted([ int(y) for y in os.listdir('.') if os.path.isdir(y) ])

			print (yearList)
			for y in sorted(yearList):
				if y >=minYear:
					os.chdir(str(y))
					print (os.getcwd())

					filesList = sorted(glob.glob("*.json"))
					print (filesList)
					
					for f in filesList:
						fileDate = datetime.datetime.strptime(f[:f.find(".")], "%Y-%m-%d").date()
						if fileDate >= lastDate.date():
							print("FILE DATE ", fileDate)
							#delete news form the most recent day in the database so it can upload the day again
							News.objects.filter(Q(publisher=publisher.id) & Q(date__gte=fileDate)).delete()
							with open(f) as data_file:
								try:
									print(f)
									data = json.load(data_file)
									objectsList = []
									for d in data:
										newsDate = dateutil.parser.parse(d['date'])
										#check for repeted news
	#										if News.objects.all().filter(Q(publisher=publisher.id)&Q(title=d['title'])&Q(date__gte=newsDate)).count() == 0:

										if d['title'] == None:
											d['title'] = "Sin Titulo"
										if len(d['title']) >= 512:
											d['title'] = d['title'][:500]
										print (p + " "+  str(newsDate) +  ": " + d['title'])
										news = News()
										news.publisher = publisher
										news.title = d['title']
										news.text = d['text']
										news.url = d['url']
										news.date = newsDate


	#											print ("topic type:",  type(d['topic']))
	#											print (d['topic'])
										if "topic" not in d or d['topic'] == "" or d['topic'] == None or d['topic'] ==[]:
											topicstr = ["Sin Tema"]

										elif type(d['topic'])==list:
											topicstr = d['topic']
	#												for t in d['topic']:
	##													topic, created = Topic.objects.all().get_or_create(name=t)
	#													topic, created = Topic.objects.get_or_create(name=t)
	#													news.topic.add(topic)
										else:
	#												topic, created = Topic.objects.get_or_create(name=d['topic'])
	#												news.topic.add(topic)
											topicstr = [d['topic']]
											
										if len(json.dumps(topicstr)) >= 1024:
											print(topicstr)
											news.topic_raw = json.dumps(topicstr[:4])
										else:
											news.topic_raw = json.dumps(topicstr)
	#									news.save()
										objectsList.append(news)
									
									News.objects.bulk_create(objectsList)
									print(fileDate, " SAVED")
								except ValueError as e:
									print (e)
									print ("Error: " + os.getcwd()+"/"+f)
									print (type(d['topic']))
									
								data_file.close()

					os.chdir("..")
			os.chdir("..")


