Commit 8732738a authored by Mario Chirinos Colunga's avatar Mario Chirinos Colunga 💬

mario

parent 930cb6b4
No preview for this file type
......@@ -3,7 +3,7 @@ from django.contrib import admin
# Register your models here.
#from catalog.models import Publisher
from catalog.models import Publisher
#admin.site.register(Publisher)
admin.site.register(Publisher)
No preview for this file type
No preview for this file type
......@@ -9,9 +9,21 @@ class SearchForm(forms.Form):
startDate = forms.DateField(label="Fecha de Inicio", widget=forms.DateInput(attrs={'type':'date', 'class':"form-control"}), required=False)
endDate = forms.DateField(label="Fecha de Fin", widget=forms.DateInput(attrs={'type':'date', 'class':"form-control"}), required=False)
# news = News.objects.all()
# publishersList = Publisher.objects.all().filter( Q(id__in = news.values('publisher').distinct()))
# choice = [ (r.id,r.name) for r in publishersList ]
# publishers = forms.MultipleChoiceField(label="Fuentes", widget=forms.SelectMultiple(attrs={'class':"form-control"}), choices=choice, required=False)
publishers = forms.MultipleChoiceField(label="Fuentes", widget=forms.SelectMultiple(attrs={'class':"form-control"}), choices=[], required=False)
text = forms. CharField(label="Palabras Clave", widget=forms.TextInput(attrs={'class':"form-control", "placeholder":"Palabras clave"}), required=False)
def __init__(self, *args, **kwargs):
super(SearchForm, self).__init__(*args, **kwargs)
news = News.objects.all()
publishersList = Publisher.objects.all().filter( Q(id__in = news.values('publisher').distinct()))
choice = [ (r.id,r.name) for r in publishersList ]
publishers = forms.MultipleChoiceField(label="Fuentes", widget=forms.SelectMultiple(attrs={'class':"form-control"}), choices=choice, required=False)
text = forms. CharField(label="Palabras Clave", widget=forms.TextInput(attrs={'class':"form-control", "placeholder":"Palabras clave"}), required=False)
self.fields['publishers'] = forms.MultipleChoiceField(label="Fuentes", widget=forms.SelectMultiple(attrs={'class':"form-control"}), choices=choice, required=False)
No preview for this file type
......@@ -5,6 +5,7 @@ import os
import json
import datetime
from django.utils import timezone
import dateutil.parser
class Command(BaseCommand):
help = 'Update database'
......@@ -46,16 +47,16 @@ class Command(BaseCommand):
try:
data = json.load(data_file)
for d in data:
if d['date'].find("+") >=0:
date =d['date'][:d['date'].find("+")].replace("T", " ")
newsDate = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
else:
date = d['date']
newsDate = datetime.datetime.strptime(date, "%Y-%m-%d")
# if d['date'].find("+") >=0:
# date =d['date'][:d['date'].find("+")].replace("T", " ")
# newsDate = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
# else:
# date = d['date']
# newsDate = datetime.datetime.strptime(date, "%Y-%m-%d")
newsDate = dateutil.parser.parse(d['date'])
newsDate = timezone.make_aware(newsDate, timezone.get_current_timezone())
# newsDate = timezone.make_aware(newsDate, timezone.get_current_timezone())
# print "newsDate:"
# print newsDate
......
from django.core.management.base import BaseCommand, CommandError
from django.contrib.postgres.search import SearchVector, SearchQuery, SearchRank
from django.contrib.postgres.aggregates import StringAgg
#from polls.models import Question as Poll
#import json
#from pprint import pprint
from catalog.models import News
class Command(BaseCommand):
help = 'Update search vectors'
# def add_arguments(self, parser):
# parser.add_argument('filename', nargs=1, type=str)
def handle(self, *args, **options):
vector=SearchVector('text')
for news in News.objects.annotate(document=vector):
news.search_vector = news.document
news.save(update_fields=['search_vector'])
No preview for this file type
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-14 04:50
# Generated by Django 1.10.7 on 2017-08-15 20:59
from __future__ import unicode_literals
from django.db import migrations, models
......@@ -18,6 +18,15 @@ class Migration(migrations.Migration):
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=128)),
('shortName', models.CharField(default=b'', max_length=32)),
('crawler', models.CharField(default=b'', max_length=32)),
('type', models.CharField(default=b'', max_length=32)),
('url', models.URLField(default=b'')),
],
options={
'ordering': ('name',),
'verbose_name': 'Medio',
'verbose_name_plural': 'Medios',
},
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-14 04:53
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('catalog', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='publisher',
name='shortName',
field=models.CharField(default=b'', max_length=16),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-14 04:58
# Generated by Django 1.10.7 on 2017-08-15 20:59
from __future__ import unicode_literals
from django.db import migrations, models
......@@ -8,7 +8,7 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('catalog', '0005_auto_20170814_0456'),
('catalog', '0001_initial'),
]
operations = [
......@@ -16,7 +16,12 @@ class Migration(migrations.Migration):
name='Topic',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=64, verbose_name=b'Tema')),
('name', models.CharField(max_length=256, verbose_name=b'Tema')),
],
options={
'ordering': ('name',),
'verbose_name': 'Tema',
'verbose_name_plural': 'Temas',
},
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-14 04:54
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('catalog', '0002_publisher_shortname'),
]
operations = [
migrations.AddField(
model_name='publisher',
name='crawler',
field=models.CharField(default=b'', max_length=16),
),
migrations.AddField(
model_name='publisher',
name='type',
field=models.CharField(default=b'', max_length=16),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-14 05:01
# Generated by Django 1.10.7 on 2017-08-15 20:59
from __future__ import unicode_literals
from django.db import migrations, models
......@@ -9,7 +9,7 @@ import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('catalog', '0007_auto_20170814_0459'),
('catalog', '0002_topic'),
]
operations = [
......@@ -17,12 +17,17 @@ class Migration(migrations.Migration):
name='News',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('title', models.CharField(max_length=256)),
('title', models.CharField(max_length=512)),
('date', models.DateTimeField(verbose_name=b'Publication Date')),
('text', models.TextField(blank=True, default=b'', verbose_name=b'News Text')),
('url', models.URLField(verbose_name=b'URL')),
('url', models.URLField(max_length=512, verbose_name=b'URL')),
('publisher', models.ForeignKey(default=b'', on_delete=django.db.models.deletion.CASCADE, to='catalog.Publisher')),
('topic', models.ManyToManyField(blank=True, to='catalog.Topic', verbose_name=b'Topic(s)')),
],
options={
'ordering': ('date',),
'verbose_name': 'Noticia',
'verbose_name_plural': 'Noticias',
},
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-14 05:02
# Generated by Django 1.10.7 on 2017-08-16 03:52
from __future__ import unicode_literals
import django.contrib.postgres.search
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('catalog', '0008_news'),
('catalog', '0003_news'),
]
operations = [
migrations.AlterModelOptions(
name='news',
options={'ordering': ('date',), 'verbose_name': 'Noticia', 'verbose_name_plural': 'Noticias'},
migrations.AddField(
model_name='news',
name='search_vector',
field=django.contrib.postgres.search.SearchVectorField(null=True),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-14 04:55
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('catalog', '0003_auto_20170814_0454'),
]
operations = [
migrations.AddField(
model_name='publisher',
name='url',
field=models.URLField(default=b''),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-14 04:56
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('catalog', '0004_publisher_url'),
]
operations = [
migrations.AlterModelOptions(
name='publisher',
options={'ordering': ('name',), 'verbose_name': 'Medio', 'verbose_name_plural': 'Medios'},
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-14 04:59
# Generated by Django 1.11.4 on 2017-08-19 20:50
from __future__ import unicode_literals
import django.contrib.postgres.indexes
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('catalog', '0006_topic'),
('catalog', '0004_news_search_vector'),
]
operations = [
migrations.AlterModelOptions(
name='topic',
options={'ordering': ('name',), 'verbose_name': 'Tema', 'verbose_name_plural': 'Temas'},
migrations.AddIndex(
model_name='news',
index=django.contrib.postgres.indexes.GinIndex(fields=[b'search_vector'], name='catalog_new_search__62ed1c_gin'),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-14 05:15
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('catalog', '0009_auto_20170814_0502'),
]
operations = [
migrations.AlterField(
model_name='publisher',
name='crawler',
field=models.CharField(default=b'', max_length=32),
),
migrations.AlterField(
model_name='publisher',
name='shortName',
field=models.CharField(default=b'', max_length=32),
),
migrations.AlterField(
model_name='publisher',
name='type',
field=models.CharField(default=b'', max_length=32),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-14 07:32
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('catalog', '0010_auto_20170814_0515'),
]
operations = [
migrations.AlterField(
model_name='news',
name='url',
field=models.URLField(max_length=512, verbose_name=b'URL'),
),
migrations.AlterField(
model_name='topic',
name='name',
field=models.CharField(max_length=256, verbose_name=b'Tema'),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-14 16:20
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('catalog', '0011_auto_20170814_0732'),
]
operations = [
migrations.AlterField(
model_name='news',
name='title',
field=models.CharField(max_length=512),
),
]
from django.db import models
from django.db.models.fields import TextField
from django.contrib.postgres.search import SearchVectorField, SearchVector
from django.contrib.postgres.indexes import GinIndex
# Create your models here.
class Publisher(models.Model):
name = models.CharField(max_length=128)
shortName = models.CharField(max_length=32, default="")
......@@ -31,6 +31,10 @@ class Topic(models.Model):
verbose_name_plural = 'Temas'
ordering = ('name',)
class PostManager(models.Manager):
def with_documents(self):
vector = SearchVector('text')
return self.get_queryset().annotate(document=vector)
class News(models.Model):
title = models.CharField(max_length=512)
......@@ -39,6 +43,15 @@ class News(models.Model):
date = models.DateTimeField(verbose_name='Publication Date')
text = TextField(verbose_name='News Text', default='', blank=True)
url = models.URLField(verbose_name='URL', max_length=512)
search_vector = SearchVectorField(null=True)
objects = PostManager()
def save(self, *args, **kwargs):
super(News, self).save(*args, **kwargs)
if 'update_fields' not in kwargs or 'search_vector' not in kwargs['update_fields']:
instance = self._meta.default_manager.with_documents().get(pk=self.pk)
instance.search_vector = instance.document
instance.save(update_fields=['search_vector'])
def __str__(self):
return self.title
......@@ -47,4 +60,4 @@ class News(models.Model):
verbose_name = 'Noticia'
verbose_name_plural = 'Noticias'
ordering = ('date',)
indexes = [ GinIndex(fields=['search_vector']) ]
from django.db import models
from django.db.models.fields import TextField
# Create your models here.
class Publisher(models.Model):
name = models.CharField(max_length=128)
shortName = models.CharField(max_length=16)
crawler = models.CharField(max_length=16, default="")
type = models.CharField(max_length=16, default='')
url = models.URLField(verbose_name='URL')
def __str__(self):
return self.name
class Meta:
verbose_name = 'Medio'
verbose_name_plural = 'Medios'
ordering = ('name',)
class Topic(models.Model):
name = models.CharField(verbose_name='Tema', max_length=100)
def __str__(self):
return self.topic_name
class Meta:
verbose_name = 'Tema'
verbose_name_plural = 'Temas'
ordering = ('name',)
class News(models.Model):
title = models.CharField(max_length=500)
publisher = models.ForeignKey(Publisher, on_delete=models.CASCADE, default='')
topic = models.ManyToManyField(Topic, verbose_name='Topic(s)', blank=True)
date = models.DateTimeField(verbose_name='Publication Date')
text = TextField(verbose_name='News Text', default='', blank=True)
url = models.URLField(verbose_name='URL')
def __str__(self):
return self.title
class Meta:
verbose_name = 'Noticia'
verbose_name_plural = 'Noticias'
ordering = ('date',)
from django.db import models
from django.db.models.fields import TextField
# Create your models here.
No preview for this file type
......@@ -42,13 +42,28 @@
<li>
<button type="submit" class="btn btn-default" style="width:100%;">Buscar</button>
</li>
</form>
</ul>
<!-- /.nav-second-level -->
</li>
<li>
<a href="#"><i class="fa fa-download fa-fw"></i>Descargar<span class="fa arrow"></span></a>
<ul class="nav nav-second-level" aria-expanded="true">
<form role="form">
<li>
<a href="#">JSON</a>
</li>
<li>
<a href="#">CSV</a>
</li>
</form>
</ul>
</li>
</ul>
</div>
......
{% extends "base_generic.html" %}
{% load app_filters %}
{% block dash-title %}<a href="{{news.publisher.url}}" target="blank">{{news.publisher.name}}</a>{% endblock %}
......@@ -13,7 +15,7 @@
<!-- .panel-heading -->
<div class="panel-body">
<div class="panel-group" id="accordion" style="text-align: justify; text-justify: auto;">
<p>{{news.text}}</p>
<p>{{news.text|highlight:request.GET.text}}</p>
</div>
</div>
<!-- .panel-body -->
......
from django import template
import re
from django.utils.safestring import mark_safe
register = template.Library()
@register.filter
def highlight(text, word):
pattern = re.compile('(%s)' % word, re.IGNORECASE)
result = pattern.sub(r'<b>\1</b>',text)
return mark_safe(result)
......@@ -7,9 +7,11 @@ urlpatterns = [
url(r'^$', views.index, name='index'),
url(r'^publishers/$', views.publisherList, name='publishers-list'),
url(r'^publishers/(?P<type>\w+)$', views.publisherList, name='publishers-list'),
url(r'^news/details/(?P<newsId>\w+)$', views.newsDetails, name='news-details'),
url(r'^news/details/(?P<newsId>\w+)/$', views.newsDetails, name='news-details'),
url(r'^news/$', views.newsList, name='news-list'),
url(r'^ws/news/$', views.wsNewsList, name='ws-news-list'),
url(r'^ws/graphs/$', views.wsGraphs, name='ws-graphs'),
url(r'^ws/download/$', views.wsDownloadNews, name='ws-download-news'),
]
from django.conf.urls import url
from . import views
urlpatterns = [
url(r'^$', views.index, name='index'),
url(r'^publishers/$', views.publisherList, name='publishers-list'),
url(r'^publishers/(?P<type>\w+)$', views.publisherList, name='publishers-list'),
url(r'^news/details/(?P<newsId>\w+)$', views.newsDetails, name='news-details'),
url(r'^news/$', views.newsList, name='news-list'),
url(r'^ws/news/$', views.wsNewsList, name='ws-news-list'),
]
from django.conf.urls import url
from . import views
urlpatterns = [
# url(r'^$', views.index, name='index'),
# url(r'^publishers/$', views.publisherList, name='publishers-list'),
# url(r'^publishers/(?P<type>\w+)$', views.publisherList, name='publishers-list'),
# url(r'^news/details/(?P<newsId>\w+)$', views.newsDetails, name='news-details'),
# url(r'^news/$', views.newsList, name='news-list'),
# url(r'^ws/news/$', views.wsNewsList, name='ws-news-list'),
]
No preview for this file type
......@@ -39,8 +39,8 @@ def getNewsByRequest(request):
myQuery &= Q(date__lte=request.GET['endDate'])
if "text" in request.GET and request.GET['text']!="None" and request.GET['text']!="":
myQuery &= Q(text__icontains=request.GET['text'])
# myQuery &= Q(text__search=request.GET['text'])
myQuery &= Q(search_vector=request.GET['text'])
print myQuery
......@@ -165,7 +165,7 @@ def wsNewsList(request):#, publisherShortName="all"):
for n in news:
row = list()
row.append('<a href="'+reverse('news-details', args =(n.id,) )+'">'+ n.title+ '</a>')
row.append('<a href="'+reverse('news-details', args =(n.id,) )+ "?"+ request.GET.urlencode() +'">'+ n.title+ '</a>')
row.append('<a href="'+n.url+'" target="blank">'+n.publisher.shortName+'</a>')
row.append(n.date.strftime("%Y-%m-%d"))
data['data'].append(row)
......@@ -176,5 +176,20 @@ def wsNewsList(request):#, publisherShortName="all"):
def newsDetails(request, newsId):
form = SearchForm(request.GET)
news = News.objects.get(id=newsId)
# news.text = news.text.replace(request.GET['text'], "<b>"+request.GET['text']+"</b>")
return render(request,'newsDetails.html',{"form":form,"news":news})
#-------------------------------------------------------------------------------
def wsGraphs(request):
data=dict()
if request.GET['type']=="histogram":
data = 0;
return HttpResponse(json.dumps(data), content_type="application/json")
#-------------------------------------------------------------------------------
def wsDownloadNews(request):
data=dict()
if request.GET['format']=="JSON":
data = 0;
return HttpResponse(json.dumps(data), content_type="application/json")
No preview for this file type
......@@ -39,6 +39,7 @@ INSTALLED_APPS = [
'django.contrib.staticfiles',
'django.contrib.humanize',
'catalog.apps.CatalogConfig',
'django.contrib.postgres',
]
......
"""catedrasCONACYT URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: url(r'^$', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.conf.urls import url, include
2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib import admin
from django.conf.urls import include
from django.views.generic import RedirectView
from django.conf.urls.static import static
from django.conf import settings
urlpatterns = [
url(r'^admin/', admin.site.urls),
url(r'^catalog/', include('catalog.urls')),
url(r'^$', RedirectView.as_view(url='/catalog/', permanent=True)),
url(r'^accounts/', include('django.contrib.auth.urls')),
] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
"""catedrasCONACYT URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: url(r'^$', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.conf.urls import url, include
2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib import admin
from django.conf.urls import include
from django.views.generic import RedirectView
from django.conf.urls.static import static
from django.conf import settings
urlpatterns = [
# url(r'^admin/', admin.site.urls),
# url(r'^catalog/', include('catalog.urls')),
# url(r'^$', RedirectView.as_view(url='/catalog/', permanent=True)),
# url(r'^accounts/', include('django.contrib.auth.urls')),
] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
No preview for this file type
No preview for this file type
sudo su - postgres
psql
CREATE USER geoint WITH PASSWORD 'gepoint';
CREATE DATABASE m3db;
ALTER ROLE geoint SET client_encoding TO 'utf8';
ALTER ROLE geoint SET default_transaction_isolation TO 'read committed';
ALTER ROLE geoint SET timezone TO 'UTC';
GRANT ALL PRIVILEGES ON DATABASE m3db TO geoint;
DROP DATABASE [ IF EXISTS ] name
find . -path "*/migrations/*.py" -not -name "__init__.py" -delete
find . -path "*/migrations/*.pyc" -delete
cp m3_webInterface/urls.py_nourls m3_webInterface/urls.py
cp catalog/urls.py_nourls catalog/urls.py
cp catalog/models.py_nomodels catalog/models.py
cp m3_webInterface/urls.py_ m3_webInterface/urls.py
cp catalog/urls.py_ catalog/urls.py
cp catalog/models.py_ catalog/models.py
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment