crawlers

descarga_por_dia/paisElSalvador/diarioCoLatino/diarioCoLatino/pipelines.py

@@ -55,6 +55,14 @@ class JsonWriterPipeline(object):
             row.append(("location", item['location']))
         except:
             pass
+        try:
+            row.append(("twitter", item['twitter']))
+        except:
+            pass
+        try:
+            row.append(("email", item['email']))
+        except:
+            pass
         try:
             row.append(("text", item['text']))
         except:

descarga_por_dia/paisElSalvador/diarioCoLatino/diarioCoLatino/spiders/noticias.py

@@ -20,9 +20,9 @@ def remove_tags(text):
 # TAG2_RE = re.compile(r'\ntransition_[^\]]+\]')
 # TAG3_RE = re.compile(r'\[[^\]]+[\]\n]')
 AUTH_RE = re.compile(r'\nPor.+?\n')
-TW_RE = re.compile(r'(\n(\| )?Twitter:\s+)?(@[\w.%+-]+.)', re.I)
-LOC_RE = re.compile(r'\n.*?\/(PL|AFP|DPA)', re.I)
-EM_RE = re.compile(r'(\n(Email|Correo electr.{1,3}nico|Comentarios?):\s)?[\w.-]+@[\w-]+(\.[a-zA-Z]{2,6}){1,2}\s?')
+TW_RE = re.compile(r'\n((\| )?Twitter:\s+)?@[\w.%+-]+.\n', re.I)
+LOC_RE = re.compile(r'\n.*?\/(PL|AFP|DPA|SIGNIS ALC)\n', re.I)
+EM_RE = re.compile(r'\n((Email|Correo electr.{1,3}nico|Comentarios?):\s)?[\w.-]+@[\w-]+(\.[a-zA-Z]{2,6}){1,2}\s?\n')
 #** correo pasarlo dos veces seguidas
 #Frases a quitar: '\nFotografías\n', '\nDiario Co Latino\n'
 class QuotesSpider(scrapy.Spider):
@@ -74,7 +74,7 @@ class QuotesSpider(scrapy.Spider):
 
         text = "\n" + text
         """ Obtiene autor """
-        res = AUTH_RE.search(text)
+        res = AUTH_RE.match(text)
         if res:
             m = res.group(0)
             item['author'] = m[m.find('Por')+len('Por'):].strip()
@@ -90,30 +90,39 @@ class QuotesSpider(scrapy.Spider):
             text = "\n" + text
 
         """ Obtiene lugar """
-        res = LOC_RE.search(text)
+        res = LOC_RE.match(text)
         if res:
             m = res.group(0)
             if m[m.find('/') + 1:].strip().lower() != 'dpa':
                 item['location'] = m[:m.find('/')].strip()
                 text = text[text.find(m) + len(m):].strip()
                 text = "\n" + text
+            else:
+                text = text[text.find(m) + len(m):].strip()
+                text = "\n" + text
 
         """ Elimina correo """
         res = EM_RE.search(text)
         if res:
             m = res.group(0)
             item['email'] = m.strip()
-            text = text[text.find(m) + len(m):].strip()
+            # text = text[text.find(m) + len(m):].strip()
+            text = text.replace(m, '').strip()
             text = "\n" + text
 
         res = EM_RE.search(text)
         if res:
             m = res.group(0)
             item['email'] = m.strip()
-            text = text[text.find(m) + len(m):].strip()
+            # text = text[text.find(m) + len(m):].strip()
+            text = text.replace(m, '').strip()
             text = "\n" + text
 
-        text[text.find("\nDiario Co Latino\n") + len("\nDiario Co Latino\n")]
+        text = text.replace("\n@Diario Co Latino\n", '').strip()
+        text = "\n" + text
+        text = text.replace("\nDiario Co Latino\n", '').strip()
+        text = "\n" + text
+        text = text.replace("\nCo Latino\n", '').strip()
 
         # result = LOC_RE.search(text)
         # if result: