update

catalog/management/commands/updateDB.py

@@ -27,8 +27,14 @@ class Command(BaseCommand):
 			audioTime.objects.all().delete()
 			publishers  = Publisher.objects.all().filter(type="audio")
 			for p in publishers:
-				files =  [files for r, d, files in os.walk(recordingsDir+p.shortName)]
-				files =  list(itertools.chain.from_iterable(files))
+				files =  [files for r, d, files in os.walk(recordingsDir+p.shortName) if files]
+				files =  sorted(list(itertools.chain.from_iterable(files)))
+				sortedFiles = files
+				#print(files)
+#				f1 = files[0]
+#				startDate = f1[:f1.find(".flac")]
+#				f2 = files[len(files)-1]
+#				endDate = f2[:f2.find(".flac")]
 				minutes = len(files)
 				sortedFiles = sorted([(f[:f.find(".flac")]) for f in files if f.find(".flac")>0 and f.count(".")==1])
 				print(p.shortName+": "+ str(minutes))
@@ -37,12 +43,18 @@ class Command(BaseCommand):
 					print("len timestamp: " + str(len(sortedFiles[0]))) 
 
 					since = datetime.datetime.utcfromtimestamp(int(sortedFiles[0]))
-					audioTime.objects.update_or_create(publisher=p, defaults={'minutes': minutes, "startDate": since},)
+					print("startDate: ", since) 
+					print(sortedFiles[len(sortedFiles)-1])
+					endDate = datetime.datetime.utcfromtimestamp(int(sortedFiles[len(sortedFiles)-1]))
+					print("endDate: ", endDate) 
+
+					audioTime.objects.update_or_create(publisher=p, defaults={'minutes': minutes, "startDate": since, "endDate":endDate},)
 		#load news
 		os.chdir(options['basedir'][0])
 		publisherList = os.listdir(options['basedir'][0])
 		for p in publisherList:
 			print (p)
+
 			os.chdir(p)
 			publisher  = Publisher.objects.all().filter(shortName=p)[0]
 			news = News.objects.all().filter(publisher=publisher.id).order_by("-date")

catalog/migrations/0007_audiotime_enddate.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-01-24 19:37
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('catalog', '0006_audiotime'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='audiotime',
+            name='endDate',
+            field=models.DateTimeField(default=django.utils.timezone.now, verbose_name='End Date'),
+            preserve_default=False,
+        ),
+    ]

catalog/models.py

@@ -96,5 +96,6 @@ def save_user_profile(sender, instance, **kwargs):
 class audioTime(models.Model):
         publisher = models.ForeignKey(Publisher, on_delete=models.CASCADE)
         startDate = models.DateTimeField(verbose_name='Start Date')
+        endDate = models.DateTimeField(verbose_name='End Date')
         minutes =  models.BigIntegerField(default=0)
 

catalog/static/js/data.js

-m3Data={"users": 1, "publishers": 62, "text": 62, "audio": 0, "documents": 3707164, "searches": 56, "audioTime": 0};
\ No newline at end of file
+m3Data={"users": 17, "publishers": 80, "text": 62, "audio": 17, "documents": 3743812, "searches": 43, "audioTime": 5182281};
\ No newline at end of file

catalog/static/new/css/status.css

+video { 
+	position: fixed;
+	top: 50%;
+	left: 50%;
+	min-width: 100%;
+	min-height: 100%;
+	width: auto;
+	height: auto;
+	z-index: -100;
+	transform: translateX(-50%) translateY(-50%);
+	background-size: cover;
+	transition: 1s opacity;
+}
+

catalog/static/new/css/style.css

@@ -3,23 +3,31 @@ form label
 {
 	color:GhostWhite ;
 }
-.news-text
+@media (min-device-width: 800px)
 {
-	font-size: 1.5em;
-	line-height:200%;
-	column-count:2;
-	column-gap:4em;
-	margin:3em;
-}
-.news-title
-{
-	font-size:2.1em;
-	font-family: 'Libre Franklin', sans-serif;
-	font-weight:900;
+	.news-text
+	{
+		font-size: 1.5em;
+		line-height:200%;
+		column-count:2;
+		column-gap:4em;
+		margin:3em;
+	}
+	.news-title
+	{
+		font-size:2.1em;
+		font-family: 'Libre Franklin', sans-serif;
+		font-weight:900;
 
+	}	
 }
 .progress-description a
 {
 	color: GhostWhite;
 
 }
+.CropLongTexts {
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+}

catalog/templates/new/status.html

+es<!DOCTYPE html>
+{% load static %}
+{% load humanize %}
+
+
+<html lang="es">
+<head>
+	<meta charset="utf-8">
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<title>Monitoreo de Multiples Medios</title>
+	<meta name="description" content="M³: Monitoreo de Multiples Medios">
+	<meta name="author" content="Mario Chirinos Colunga - CentroGeo">
+	<meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport">
+	<link rel="icon" href="/static/images/m3.png">
+	
+	<!-- Bootstrap 3.3.7 -->
+	<link rel="stylesheet" href="{% static 'new/bootstrap-3.3.7-dist/css/bootstrap.min.css' %}">
+	<!-- Font Awesome -->
+	<link rel="stylesheet" href="{% static 'new/font-awesome/css/font-awesome.min.css' %}">
+	<!-- IonIcons -->
+	<link rel="stylesheet" href="{% static 'new/Ionicons/css/ionicons.min.css' %}">
+	<!-- Theme style -->
+	<link rel="stylesheet" href="{% static 'new/adminLTE-dist/css/AdminLTE.min.css' %}">
+	<!-- AdminLTE Skins. Choose a skin from the css/skins
+       folder instead of downloading all of them to reduce the load. -->
+	<link rel="stylesheet" href="{% static 'new/adminLTE-dist/css/skins/_all-skins.min.css' %}">
+	<!-- Google Font -->
+	<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Source+Sans+Pro:300,400,600,700,300italic,400italic,600italic">
+	<link href="https://fonts.googleapis.com/css?family=Libre+Franklin" rel="stylesheet">
+	<link rel="stylesheet" href="{% static 'new/css/style.css' %}">
+
+
+
+	{% block headMedia %}
+		<link rel="stylesheet" href="{% static 'new/css/status.css'%}" type="text/css">
+
+	{% endblock %}
+
+
+</head>
+<body class="sidebar-mini skin-red" style="height: auto; min-height: 100%;">
+	<div class="visible-lg-block visible-md-block">
+		<video  id="bgvid" playsinline autoplay muted loop poster="{% static 'images/poster.jpg'%}" >
+			<!-- WCAG general accessibility recommendation is that media such as background video play through only once. Loop turned on for the purposes of illustration; if removed, the end of the video will fade in the same way created by pressing the "Pause" button  -->
+			<source src="{% static 'images/background.mp4'%}" type='video/mp4; codecs="avc1.42E01E, mp4a.40.2"' />
+			<source src="{% static 'images/background.ogv'%}"  type='video/ogg; codecs="theora, vorbis"' />
+			<source src="{% static 'images/background.webm'%}" type='video/webm; codecs="vp8, vorbis"' />
+
+		</video>
+	</div>
+
+	<div class="container" style="height: auto; min-height: 100%;">
+		<div class="row">
+
+
+			<div class="col-8 align-self-center">
+			
+<div class="box">
+	<div class="box-header">
+		<h3 class="box-title">Medios en Texto</h3>
+
+		<div class="box-tools">
+		</div>
+	</div><!-- /.box-header -->
+	<div class="box-body table-responsive no-padding">
+		<table class="table table-hover">
+			<tbody>
+				<tr>
+					<th>Medio</th>
+					<th>Documentos</th>
+					<th>Desde</th>
+					<th>Ultima Noticia</th>
+					<th>Estado</th>
+				</tr>
+				{% for k, v in textPublishers.items %}
+					<tr>
+						<td>{{k}}</td><td>{{v.docs|intcomma}}</td><td>{{v.startDate|date:"M d, Y"}}</td><td>{{v.endDate|date:"M d, Y"}}</td>
+						<td>
+							{% if v.status > 1 %}
+								<span class="label label-danger">{{v.status}} dias</span>
+							{% else %}
+								<span class="label label-success">OK</span>
+							{% endif %}
+						</td>
+					</tr>
+				{% endfor %}
+			</tbody>
+		</table>
+	</div><!-- /.box-body -->
+</div>
+
+			</div>
+		</div>
+                <div class="row">
+
+
+                        <div class="col-8 align-self-center">
+
+<div class="box">
+        <div class="box-header">
+                <h3 class="box-title">Medios en Audio</h3>
+
+                <div class="box-tools">
+                </div>
+        </div><!-- /.box-header -->
+        <div class="box-body table-responsive no-padding">
+                <table class="table table-hover">
+                        <tbody>
+                                <tr>
+                                        <th>Medio</th>
+                                        <th>Horas</th>
+                                        <th>Desde</th>
+                                        <th>Hasta</th>
+                                        <th>Estado</th>
+                                </tr>
+                                {% for k, v in audioPublishers.items %}
+                                        <tr>
+                                                <td>{{k}}</td><td>{{v.time|intcomma}}</td><td>{{v.startDate|date:"M d, Y"}}</td><td>{{v.endDate|date:"M d, Y"}}</td>
+                                                <td>
+                                                        {% if v.status > 1 %}
+                                                                <span class="label label-danger">{{v.status}} dias</span>
+                                                        {% else %}
+                                                                <span class="label label-success">OK</span>
+                                                        {% endif %}
+                                                </td>
+                                        </tr>
+                                {% endfor %}
+                        </tbody>
+                </table>
+        </div><!-- /.box-body -->
+</div>
+
+                        </div>
+                </div>
+	<footer class="">
+	</footer>	
+	</div><!-- ./wrapper -->
+
+	<!-- jQuery 3 -->
+	<script src="{% static 'new/js/jquery-3.3.1.min.js' %}"></script>
+
+	<!-- Bootstrap 3.3.7 -->
+	<script src="{% static 'new/bootstrap-3.3.7-dist/js/bootstrap.min.js' %}"></script>
+	<!-- AdminLTE App -->
+	<script src="{% static 'new/adminLTE-dist/js/adminlte.min.js' %}"></script>
+
+        {% block scripts %}{% endblock %}
+
+</body>
+</html>
+

catalog/urls.py

@@ -6,7 +6,7 @@ from django.views.i18n import javascript_catalog
 
 urlpatterns = [
     url(r'^$', views.index, name='index'),
-    url(r'^test/$', views.indexTemp, name='indexTemp'),
+    url(r'^status/$', views.status, name='status'),
 
     url(r'^js/jsi18n/$', javascript_catalog, name='jsi18n'),
 
@@ -21,6 +21,7 @@ urlpatterns = [
     url(r'^ws/download/$', views.wsDownloadNews, name='ws-download-news'),
     url(r'^ws/searches/$', views.wsSearchList, name='ws-search-list'),
     url(r'^ws/playlist/$', views.wsAudioList, name='ws-audio-list'),
+    url(r'^ws/wordcloud/(?P<newsId>\w+)/$', views.wsWordCloud, name='ws-wordcloud'),
 
     url(r'^audio/$', views.audioList, name='audio'),
     url(r'^audio/(?P<publisher>\w+)$', views.audioPublisher, name='audio'),

catalog/views.py

@@ -30,6 +30,7 @@ import urllib
 import json
 #import StringIO #python2
 from io import StringIO #python3
+from io import BytesIO #python3
 
 import time
 import zipfile
@@ -41,6 +42,10 @@ from quantiphy import Quantity
 from numpy import array_equal
 import os
 import shutil
+
+import wordcloud
+from stop_words import get_stop_words
+
 #-------------------------------------------------------------------------------
 def getAudiosList2Convert(request, streamPublisher, initTimestamp, endTimestamp, days):
 
@@ -152,10 +157,10 @@ def settingsView(request):
 
 	print( request.user.profile.subscriptions.all() )
 
-	return render(request,'userprofile.html',{"form":form})
+	return render(request,'new/userProfile.html',{"form":form})
 #-------------------------------------------------------------------------------
 def settingsSearches(request):
-        return render(request,'searches.html',{})
+        return render(request,'new/searches.html',{})
 #-------------------------------------------------------------------------------
 def getNewsByRequest(request):
 	print ("getNewsByRequest\n\n\n")
@@ -187,8 +192,33 @@ def getNewsByRequest(request):
 
 	return News.objects.filter(myQuery)
 #-------------------------------------------------------------------------------
-def indexTemp(request):
-        return render(request,'new/index.html',{})
+def status(request):
+	tPublishers = Publisher.objects.filter(type="texto")
+	tdic = {}
+	
+
+#= {p.name: {"docs": News.objects.filter(publisher=p).count(), "endDate":News.objects.filter(publisher=p).order_by('-date')[0].date} for p in tPublishers}
+	for p in tPublishers:
+		status = {}
+		status["docs"] = News.objects.filter(publisher=p).count()
+		status["startDate"]=News.objects.filter(publisher=p).order_by('date')[0].date
+		status["endDate"]=News.objects.filter(publisher=p).order_by('-date')[0].date
+		status["status"]=(datetime.datetime.now(datetime.timezone.utc)-status["endDate"]).days
+		tdic[p.name]=status
+
+	aPublishers = Publisher.objects.filter(type="audio")
+	adic={}
+	
+	for p in aPublishers:
+		status = {}
+		status["time"] =  math.floor(audioTime.objects.filter(publisher=p)[0].minutes/60)
+		status["startDate"] = audioTime.objects.filter(publisher=p)[0].startDate
+		status["endDate"] = audioTime.objects.filter(publisher=p)[0].endDate
+		status["status"]=(datetime.datetime.now(datetime.timezone.utc)-status["endDate"]).days
+
+		adic[p.name]=status
+
+	return render(request,'new/status.html',{"textPublishers":tdic, "audioPublishers":adic})
 #-------------------------------------------------------------------------------
 def index(request):
 	"""
@@ -237,18 +267,24 @@ def index(request):
 	donutChart = [{"label": Publisher.objects.get(id=q['publisher']).name, "value":q["count"], "url":urlDict[q['publisher']]} for q in queryset]
 
 
-	publishers = Publisher.objects.filter( Q(id__in = news.values('publisher').distinct()))
+#	publishers = Publisher.objects.filter( Q(id__in = news.values('publisher').distinct()))
+
 
+#	queryset = news.values("publisher").annotate(day=TruncMonth('date') )
 
-	queryset = news.values("publisher").annotate(day=TruncMonth('date') )
 
+#	dateList =[q['day'].strftime("%Y-%m-%d") for q in queryset.values("day").distinct().order_by("-day")]
+#	dateDicts = {p.shortName:{ n['day'].strftime("%Y-%m-%d"):n['count'] for n in queryset.filter(publisher=p.id).values("day").distinct().order_by("-day").annotate(count = Count('publisher')) } for p in publishers}
 
-	dateList =[q['day'].strftime("%Y-%m-%d") for q in queryset.values("day").distinct().order_by("-day")]
-	dateDicts = {p.shortName:{ n['day'].strftime("%Y-%m-%d"):n['count'] for n in queryset.filter(publisher=p.id).values("day").distinct().order_by("-day").annotate(count = Count('publisher')) } for p in publishers}
+#	histogramsData = [ {  "date":d, "data":{ k:(dateDicts[k][d] if d in dateDicts[k] else 0) for k in dateDicts.keys()}  }  for d in dateList] 
+#	histogramsLabels = [p.shortName for p in publishers]
 
-	histogramsData = [ {  "date":d, "data":{ k:(dateDicts[k][d] if d in dateDicts[k] else 0) for k in dateDicts.keys()}  }  for d in dateList] 
-	histogramsLabels = [p.shortName for p in publishers]
 
+	dateCount = news.values("publisher").annotate(day=TruncMonth('date') ).values("day").order_by("day").annotate(count=Count("day"))
+
+
+	histogramsData  = [ { "date":d["day"].strftime("%Y-%m-%d"), "data":{ "total": d["count"]}} for d in dateCount ]
+	histogramsLabels = ["total"]
 #	return render(request,'index.html',{"form":form, "info":info, "donutChart":{}, "histogram": {}, "histogramsLabels":{}})
 	return render(request,'new/index.html',{"form":form, "info":info, "donutChart":donutChart, "histogram": histogramsData, "histogramsLabels":histogramsLabels})
 #-------------------------------------------------------------------------------
@@ -315,13 +351,36 @@ def wsAudioList(request):
 #-------------------------------------------------------------------------------
 def wsSearchList(request):
 
-	searches = Search.objects.filter(user=request.user)
+	columns = ['text', 'startDate', 'endDate', "publishers"]
+	order = dict()
+	order["asc"]=""
+	order["desc"]="-"
+
+	orderBy = columns[ int(request.GET["order[0][column]"]) ]
+	direction = order[request.GET["order[0][dir]"]]
+
+
+	searches = Search.objects.filter(user=request.user).order_by(direction+orderBy)
+	if 'search[value]' in request.GET and request.GET['search[value]'] != "":
+		search = request.GET['search[value]']
+		searches = searches.filter(text__icontains=search)
 
 	data = dict()
 	data['data']=[[s.text, "*" if s is None else s.startDate.strftime('%Y-%m-%d'), "*" if s is None else s.endDate.strftime('%Y-%m-%d'), ','.join([ sub.shortName for sub in s.publishers.all()])] for s in searches]
 	data['recordsTotal'] = searches.count()
 	data['recordsFiltered'] = searches.count()
 
+
+	paginator = Paginator(searches, request.GET['length'])
+	page = (int(request.GET['start'])/int(request.GET['length']))+1
+	try:
+		searches = paginator.page(page)
+	except PageNotAnInteger:
+		searches = paginator.page(1)
+	except EmptyPage:
+		searches = paginator.page(paginator.num_pages)
+	data['data']=[[s.text, "*" if s is None else s.startDate.strftime('%Y-%m-%d'), "*" if s is None else s.endDate.strftime('%Y-%m-%d'), ','.join([ sub.shortName for sub in s.publishers.all()])] for s in searches]
+
 	return HttpResponse(json.dumps(data), content_type="application/json")
 
 #-------------------------------------------------------------------------------
@@ -398,7 +457,7 @@ def wsGraphs(request):
 def wsDownloadNews(request):
 	news = getNewsByRequest(request)
 
-	s = StringIO.StringIO()		# Open StringIO to grab in-memory ZIP contents	
+	s = BytesIO() #StringIO()		# Open StringIO to grab in-memory ZIP contents	
 	zf = zipfile.ZipFile(s, "w")	# The zip compressor
 	zf.writestr("setting.txt", json.dumps(request.GET))
 	data = news2JSON(news);
@@ -407,11 +466,13 @@ def wsDownloadNews(request):
 		zf.writestr("data.json", json.dumps(data))
 
 	if request.GET['format']=="CSV":
-		csvString = StringIO.StringIO()	
+		csvString = StringIO()	
 		writer = csv.writer(csvString,quoting=csv.QUOTE_ALL)#, fieldnames=[k for k in data[0]])
 #		writer.writeheader()
 		for d in data:
-			row = [ d[k].encode('utf-8') for k in d ]
+#			row = [ d[k].encode('utf-8') for k in d ]
+			row = [ d[k] for k in d ]
+
 			writer.writerow( row )
 		zf.writestr("data.csv", csvString.getvalue())
 
@@ -420,6 +481,16 @@ def wsDownloadNews(request):
 	response = HttpResponse(s.getvalue(), content_type="application/x-zip-compressed")
 	response['Content-Disposition'] = 'attachment; filename="news_'+ str(int(time.time()))+'.zip"'
 	return response
+#-------------------------------------------------------------------------------
+def wsWordCloud(request, newsId):
+	text = News.objects.filter(id=newsId)[0].text
+	wc = wordcloud.WordCloud(stopwords={ w for w in get_stop_words('es')}, background_color="Gainsboro", width=512,height=512,mode="RGBA",  colormap="tab20")
+	wc.generate(text)
+	image = wc.to_image()
+
+	response = HttpResponse(content_type="image/jpeg")
+	image.save(response, "PNG")
+	return response
 
 #-------------------------------------------------------------------------------
 def audioList(request):