Skip to content

m3_webInterface
m3_webInterface
Commit f38c125c authored by Mario Chirinos Colunga's avatar Mario Chirinos Colunga 💬
Browse files
Options

update

parent 9ebc6af2
Hide whitespace changes
Inline Side-by-side
Showing with 311 additions and 31 deletions
catalog/management/commands/updateDB.py
View file @ f38c125c
......@@ -27,8 +27,14 @@ class Command(BaseCommand):
audioTime.objects.all().delete()
publishers = Publisher.objects.all().filter(type="audio")
for p in publishers:
files = [files for r, d, files in os.walk(recordingsDir+p.shortName)]
files = list(itertools.chain.from_iterable(files))
files = [files for r, d, files in os.walk(recordingsDir+p.shortName) if files]
files = sorted(list(itertools.chain.from_iterable(files)))
sortedFiles = files
#print(files)
# f1 = files[0]
# startDate = f1[:f1.find(".flac")]
# f2 = files[len(files)-1]
# endDate = f2[:f2.find(".flac")]
minutes = len(files)
sortedFiles = sorted([(f[:f.find(".flac")]) for f in files if f.find(".flac")>0 and f.count(".")==1])
print(p.shortName+": "+ str(minutes))
......@@ -37,12 +43,18 @@ class Command(BaseCommand):
print("len timestamp: " + str(len(sortedFiles[0])))
since = datetime.datetime.utcfromtimestamp(int(sortedFiles[0]))
audioTime.objects.update_or_create(publisher=p, defaults={'minutes': minutes, "startDate": since},)
print("startDate: ", since)
print(sortedFiles[len(sortedFiles)-1])
endDate = datetime.datetime.utcfromtimestamp(int(sortedFiles[len(sortedFiles)-1]))
print("endDate: ", endDate)
audioTime.objects.update_or_create(publisher=p, defaults={'minutes': minutes, "startDate": since, "endDate":endDate},)
#load news
os.chdir(options['basedir'][0])
publisherList = os.listdir(options['basedir'][0])
for p in publisherList:
print (p)
os.chdir(p)
publisher = Publisher.objects.all().filter(shortName=p)[0]
news = News.objects.all().filter(publisher=publisher.id).order_by("-date")
......
catalog/migrations/0007_audiotime_enddate.py 0 → 100644
View file @ f38c125c
# -*- coding: utf-8 -*-
# Generated by Django 1.11.18 on 2019-01-24 19:37
from __future__ import unicode_literals
from django.db import migrations, models
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [
('catalog', '0006_audiotime'),
]
operations = [
migrations.AddField(
model_name='audiotime',
name='endDate',
field=models.DateTimeField(default=django.utils.timezone.now, verbose_name='End Date'),
preserve_default=False,
),
]
catalog/models.py
View file @ f38c125c
......@@ -96,5 +96,6 @@ def save_user_profile(sender, instance, **kwargs):
class audioTime(models.Model):
publisher = models.ForeignKey(Publisher, on_delete=models.CASCADE)
startDate = models.DateTimeField(verbose_name='Start Date')
endDate = models.DateTimeField(verbose_name='End Date')
minutes = models.BigIntegerField(default=0)
catalog/static/js/data.js
View file @ f38c125c
m3Data={"users": 1, "publishers": 62, "text": 62, "audio": 0, "documents": 3707164, "searches": 56, "audioTime": 0};
\ No newline at end of file
m3Data={"users": 17, "publishers": 80, "text": 62, "audio": 17, "documents": 3743812, "searches": 43, "audioTime": 5182281};
\ No newline at end of file
catalog/static/new/css/status.css 0 → 100644
View file @ f38c125c
video {
position: fixed;
top: 50%;
left: 50%;
min-width: 100%;
min-height: 100%;
width: auto;
height: auto;
z-index: -100;
transform: translateX(-50%) translateY(-50%);
background-size: cover;
transition: 1s opacity;
}
catalog/static/new/css/style.css
View file @ f38c125c
......@@ -3,23 +3,31 @@ form label
{
color:GhostWhite ;
}
.news-text
@media (min-device-width: 800px)
{
font-size: 1.5em;
line-height:200%;
column-count:2;
column-gap:4em;
margin:3em;
}
.news-title
{
font-size:2.1em;
font-family: 'Libre Franklin', sans-serif;
font-weight:900;
.news-text
{
font-size: 1.5em;
line-height:200%;
column-count:2;
column-gap:4em;
margin:3em;
}
.news-title
{
font-size:2.1em;
font-family: 'Libre Franklin', sans-serif;
font-weight:900;
}
}
.progress-description a
{
color: GhostWhite;
}
.CropLongTexts {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
catalog/templates/new/status.html 0 → 100644
View file @ f38c125c
es<!DOCTYPE html>
{% load static %}
{% load humanize %}
<html lang="es">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<title>Monitoreo de Multiples Medios</title>
<meta name="description" content="M³: Monitoreo de Multiples Medios">
<meta name="author" content="Mario Chirinos Colunga - CentroGeo">
<meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport">
<link rel="icon" href="/static/images/m3.png">
<!-- Bootstrap 3.3.7 -->
<link rel="stylesheet" href="{% static 'new/bootstrap-3.3.7-dist/css/bootstrap.min.css' %}">
<!-- Font Awesome -->
<link rel="stylesheet" href="{% static 'new/font-awesome/css/font-awesome.min.css' %}">
<!-- IonIcons -->
<link rel="stylesheet" href="{% static 'new/Ionicons/css/ionicons.min.css' %}">
<!-- Theme style -->
<link rel="stylesheet" href="{% static 'new/adminLTE-dist/css/AdminLTE.min.css' %}">
<!-- AdminLTE Skins. Choose a skin from the css/skins
folder instead of downloading all of them to reduce the load. -->
<link rel="stylesheet" href="{% static 'new/adminLTE-dist/css/skins/_all-skins.min.css' %}">
<!-- Google Font -->
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Source+Sans+Pro:300,400,600,700,300italic,400italic,600italic">
<link href="https://fonts.googleapis.com/css?family=Libre+Franklin" rel="stylesheet">
<link rel="stylesheet" href="{% static 'new/css/style.css' %}">
{% block headMedia %}
<link rel="stylesheet" href="{% static 'new/css/status.css'%}" type="text/css">
{% endblock %}
</head>
<body class="sidebar-mini skin-red" style="height: auto; min-height: 100%;">
<div class="visible-lg-block visible-md-block">
<video id="bgvid" playsinline autoplay muted loop poster="{% static 'images/poster.jpg'%}" >
<!-- WCAG general accessibility recommendation is that media such as background video play through only once. Loop turned on for the purposes of illustration; if removed, the end of the video will fade in the same way created by pressing the "Pause" button -->
<source src="{% static 'images/background.mp4'%}" type='video/mp4; codecs="avc1.42E01E, mp4a.40.2"' />
<source src="{% static 'images/background.ogv'%}" type='video/ogg; codecs="theora, vorbis"' />
<source src="{% static 'images/background.webm'%}" type='video/webm; codecs="vp8, vorbis"' />
</video>
</div>
<div class="container" style="height: auto; min-height: 100%;">
<div class="row">
<div class="col-8 align-self-center">
<div class="box">
<div class="box-header">
<h3 class="box-title">Medios en Texto</h3>
<div class="box-tools">
</div>
</div><!-- /.box-header -->
<div class="box-body table-responsive no-padding">
<table class="table table-hover">
<tbody>
<tr>
<th>Medio</th>
<th>Documentos</th>
<th>Desde</th>
<th>Ultima Noticia</th>
<th>Estado</th>
</tr>
{% for k, v in textPublishers.items %}
<tr>
<td>{{k}}</td><td>{{v.docs|intcomma}}</td><td>{{v.startDate|date:"M d, Y"}}</td><td>{{v.endDate|date:"M d, Y"}}</td>
<td>
{% if v.status > 1 %}
<span class="label label-danger">{{v.status}} dias</span>
{% else %}
<span class="label label-success">OK</span>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div><!-- /.box-body -->
</div>
</div>
</div>
<div class="row">
<div class="col-8 align-self-center">
<div class="box">
<div class="box-header">
<h3 class="box-title">Medios en Audio</h3>
<div class="box-tools">
</div>
</div><!-- /.box-header -->
<div class="box-body table-responsive no-padding">
<table class="table table-hover">
<tbody>
<tr>
<th>Medio</th>
<th>Horas</th>
<th>Desde</th>
<th>Hasta</th>
<th>Estado</th>
</tr>
{% for k, v in audioPublishers.items %}
<tr>
<td>{{k}}</td><td>{{v.time|intcomma}}</td><td>{{v.startDate|date:"M d, Y"}}</td><td>{{v.endDate|date:"M d, Y"}}</td>
<td>
{% if v.status > 1 %}
<span class="label label-danger">{{v.status}} dias</span>
{% else %}
<span class="label label-success">OK</span>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div><!-- /.box-body -->
</div>
</div>
</div>
<footer class="">
</footer>
</div><!-- ./wrapper -->
<!-- jQuery 3 -->
<script src="{% static 'new/js/jquery-3.3.1.min.js' %}"></script>
<!-- Bootstrap 3.3.7 -->
<script src="{% static 'new/bootstrap-3.3.7-dist/js/bootstrap.min.js' %}"></script>
<!-- AdminLTE App -->
<script src="{% static 'new/adminLTE-dist/js/adminlte.min.js' %}"></script>
{% block scripts %}{% endblock %}
</body>
</html>
catalog/urls.py
View file @ f38c125c
......@@ -6,7 +6,7 @@ from django.views.i18n import javascript_catalog
urlpatterns = [
url(r'^$', views.index, name='index'),
url(r'^test/$', views.indexTemp, name='indexTemp'),
url(r'^status/$', views.status, name='status'),
url(r'^js/jsi18n/$', javascript_catalog, name='jsi18n'),
......@@ -21,6 +21,7 @@ urlpatterns = [
url(r'^ws/download/$', views.wsDownloadNews, name='ws-download-news'),
url(r'^ws/searches/$', views.wsSearchList, name='ws-search-list'),
url(r'^ws/playlist/$', views.wsAudioList, name='ws-audio-list'),
url(r'^ws/wordcloud/(?P<newsId>\w+)/$', views.wsWordCloud, name='ws-wordcloud'),
url(r'^audio/$', views.audioList, name='audio'),
url(r'^audio/(?P<publisher>\w+)$', views.audioPublisher, name='audio'),
......
catalog/views.py
View file @ f38c125c
......@@ -30,6 +30,7 @@ import urllib
import json
#import StringIO #python2
from io import StringIO #python3
from io import BytesIO #python3
import time
import zipfile
......@@ -41,6 +42,10 @@ from quantiphy import Quantity
from numpy import array_equal
import os
import shutil
import wordcloud
from stop_words import get_stop_words
#-------------------------------------------------------------------------------
def getAudiosList2Convert(request, streamPublisher, initTimestamp, endTimestamp, days):
......@@ -152,10 +157,10 @@ def settingsView(request):
print( request.user.profile.subscriptions.all() )
return render(request,'userprofile.html',{"form":form})
return render(request,'new/userProfile.html',{"form":form})
#-------------------------------------------------------------------------------
def settingsSearches(request):
return render(request,'searches.html',{})
return render(request,'new/searches.html',{})
#-------------------------------------------------------------------------------
def getNewsByRequest(request):
print ("getNewsByRequest\n\n\n")
......@@ -187,8 +192,33 @@ def getNewsByRequest(request):
return News.objects.filter(myQuery)
#-------------------------------------------------------------------------------
def indexTemp(request):
return render(request,'new/index.html',{})
def status(request):
tPublishers = Publisher.objects.filter(type="texto")
tdic = {}
#= {p.name: {"docs": News.objects.filter(publisher=p).count(), "endDate":News.objects.filter(publisher=p).order_by('-date')[0].date} for p in tPublishers}
for p in tPublishers:
status = {}
status["docs"] = News.objects.filter(publisher=p).count()
status["startDate"]=News.objects.filter(publisher=p).order_by('date')[0].date
status["endDate"]=News.objects.filter(publisher=p).order_by('-date')[0].date
status["status"]=(datetime.datetime.now(datetime.timezone.utc)-status["endDate"]).days
tdic[p.name]=status
aPublishers = Publisher.objects.filter(type="audio")
adic={}
for p in aPublishers:
status = {}
status["time"] = math.floor(audioTime.objects.filter(publisher=p)[0].minutes/60)
status["startDate"] = audioTime.objects.filter(publisher=p)[0].startDate
status["endDate"] = audioTime.objects.filter(publisher=p)[0].endDate
status["status"]=(datetime.datetime.now(datetime.timezone.utc)-status["endDate"]).days
adic[p.name]=status
return render(request,'new/status.html',{"textPublishers":tdic, "audioPublishers":adic})
#-------------------------------------------------------------------------------
def index(request):
"""
......@@ -237,18 +267,24 @@ def index(request):
donutChart = [{"label": Publisher.objects.get(id=q['publisher']).name, "value":q["count"], "url":urlDict[q['publisher']]} for q in queryset]
publishers = Publisher.objects.filter( Q(id__in = news.values('publisher').distinct()))
# publishers = Publisher.objects.filter( Q(id__in = news.values('publisher').distinct()))
# queryset = news.values("publisher").annotate(day=TruncMonth('date') )
queryset = news.values("publisher").annotate(day=TruncMonth('date') )
# dateList =[q['day'].strftime("%Y-%m-%d") for q in queryset.values("day").distinct().order_by("-day")]
# dateDicts = {p.shortName:{ n['day'].strftime("%Y-%m-%d"):n['count'] for n in queryset.filter(publisher=p.id).values("day").distinct().order_by("-day").annotate(count = Count('publisher')) } for p in publishers}
dateList =[q['day'].strftime("%Y-%m-%d") for q in queryset.values("day").distinct().order_by("-day")]
dateDicts = {p.shortName:{ n['day'].strftime("%Y-%m-%d"):n['count'] for n in queryset.filter(publisher=p.id).values("day").distinct().order_by("-day").annotate(count = Count('publisher')) } for p in publishers}
# histogramsData = [ { "date":d, "data":{ k:(dateDicts[k][d] if d in dateDicts[k] else 0) for k in dateDicts.keys()} } for d in dateList]
# histogramsLabels = [p.shortName for p in publishers]
histogramsData = [ { "date":d, "data":{ k:(dateDicts[k][d] if d in dateDicts[k] else 0) for k in dateDicts.keys()} } for d in dateList]
histogramsLabels = [p.shortName for p in publishers]
dateCount = news.values("publisher").annotate(day=TruncMonth('date') ).values("day").order_by("day").annotate(count=Count("day"))
histogramsData = [ { "date":d["day"].strftime("%Y-%m-%d"), "data":{ "total": d["count"]}} for d in dateCount ]
histogramsLabels = ["total"]
# return render(request,'index.html',{"form":form, "info":info, "donutChart":{}, "histogram": {}, "histogramsLabels":{}})
return render(request,'new/index.html',{"form":form, "info":info, "donutChart":donutChart, "histogram": histogramsData, "histogramsLabels":histogramsLabels})
#-------------------------------------------------------------------------------
......@@ -315,13 +351,36 @@ def wsAudioList(request):
#-------------------------------------------------------------------------------
def wsSearchList(request):
searches = Search.objects.filter(user=request.user)
columns = ['text', 'startDate', 'endDate', "publishers"]
order = dict()
order["asc"]=""
order["desc"]="-"
orderBy = columns[ int(request.GET["order[0][column]"]) ]
direction = order[request.GET["order[0][dir]"]]
searches = Search.objects.filter(user=request.user).order_by(direction+orderBy)
if 'search[value]' in request.GET and request.GET['search[value]'] != "":
search = request.GET['search[value]']
searches = searches.filter(text__icontains=search)
data = dict()
data['data']=[[s.text, "*" if s is None else s.startDate.strftime('%Y-%m-%d'), "*" if s is None else s.endDate.strftime('%Y-%m-%d'), ','.join([ sub.shortName for sub in s.publishers.all()])] for s in searches]
data['recordsTotal'] = searches.count()
data['recordsFiltered'] = searches.count()
paginator = Paginator(searches, request.GET['length'])
page = (int(request.GET['start'])/int(request.GET['length']))+1
try:
searches = paginator.page(page)
except PageNotAnInteger:
searches = paginator.page(1)
except EmptyPage:
searches = paginator.page(paginator.num_pages)
data['data']=[[s.text, "*" if s is None else s.startDate.strftime('%Y-%m-%d'), "*" if s is None else s.endDate.strftime('%Y-%m-%d'), ','.join([ sub.shortName for sub in s.publishers.all()])] for s in searches]
return HttpResponse(json.dumps(data), content_type="application/json")
#-------------------------------------------------------------------------------
......@@ -398,7 +457,7 @@ def wsGraphs(request):
def wsDownloadNews(request):
news = getNewsByRequest(request)
s = StringIO.StringIO() # Open StringIO to grab in-memory ZIP contents
s = BytesIO() #StringIO() # Open StringIO to grab in-memory ZIP contents
zf = zipfile.ZipFile(s, "w") # The zip compressor
zf.writestr("setting.txt", json.dumps(request.GET))
data = news2JSON(news);
......@@ -407,11 +466,13 @@ def wsDownloadNews(request):
zf.writestr("data.json", json.dumps(data))
if request.GET['format']=="CSV":
csvString = StringIO.StringIO()
csvString = StringIO()
writer = csv.writer(csvString,quoting=csv.QUOTE_ALL)#, fieldnames=[k for k in data[0]])
# writer.writeheader()
for d in data:
row = [ d[k].encode('utf-8') for k in d ]
# row = [ d[k].encode('utf-8') for k in d ]
row = [ d[k] for k in d ]
writer.writerow( row )
zf.writestr("data.csv", csvString.getvalue())
......@@ -420,6 +481,16 @@ def wsDownloadNews(request):
response = HttpResponse(s.getvalue(), content_type="application/x-zip-compressed")
response['Content-Disposition'] = 'attachment; filename="news_'+ str(int(time.time()))+'.zip"'
return response
#-------------------------------------------------------------------------------
def wsWordCloud(request, newsId):
text = News.objects.filter(id=newsId)[0].text
wc = wordcloud.WordCloud(stopwords={ w for w in get_stop_words('es')}, background_color="Gainsboro", width=512,height=512,mode="RGBA", colormap="tab20")
wc.generate(text)
image = wc.to_image()
response = HttpResponse(content_type="image/jpeg")
image.save(response, "PNG")
return response
#-------------------------------------------------------------------------------
def audioList(request):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment