L1C products population

catalog/management/commands/populate_products_l1c.py

@@ -5,11 +5,7 @@
 
     Usage:
     $ cd GeoInt_SIDT
-    $ python manage.py populate_products_l1c --file_path [path_where_files_are_located]
-
-    Example:
-    $ cd GeoInt_SIDT
-    $ python manage.py populate_products_l1c --file_path /home/geointdev/sidt-env/sidt-zip/L1C/
+    $ python manage.py populate_products_l1c
 """
 
 
@@ -17,7 +13,7 @@ from django.core.management.base import BaseCommand, CommandError
 from sentinelsat.sentinel import SentinelAPI
 from catalog.models import Product_l1c
 import xml.etree.ElementTree as ET
-import os, requests, json
+import os, sys, requests, json, paramiko
 
 
 usr = "emmhp"
@@ -26,16 +22,16 @@ URL_Sentinel = 'https://scihub.copernicus.eu/dhus'
 
 apiSentinel = SentinelAPI(usr, pwd, URL_Sentinel)
 
+SSH_ADDRESS  = "192.168.1.59"
+SSH_USERNAME = "geoint"
+SSH_COMMAND  = "cd NAS/sentinelImages/L1C/; pwd"
 
-class Command(BaseCommand):
-    help = "Populates DB with downloaded L1C products."
+ssh = paramiko.SSHClient()
+ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-    def add_arguments(self, parser):
-        ## positional arguments
-        # parser.add_argument('file_path', nargs=1, type=str)
-        parser.add_argument('--file_path', dest='file_path')
-        parser.add_argument('--products_list', dest='products_list')
 
+class Command(BaseCommand):
+    help = "Populates DB with downloaded L1C products."
 
     def generate_json(self, product_name):
         """
@@ -44,13 +40,12 @@ class Command(BaseCommand):
         uri       = "https://scihub.copernicus.eu/dhus/odata/v1/Products?$filter=Name eq '" + product_name + "'"
         r         = requests.get(uri, auth=(usr, pwd))
         root      = ET.fromstring(r.content)
-        print root
+        # print root
         # base_link = root[5][0].text
         try:
             uuid      = root[5][11][0].text
         except:
-            print root
-            print "\n"
+            pass
 
         data_dict = apiSentinel.get_product_odata(uuid, full=True)
 
@@ -58,37 +53,49 @@ class Command(BaseCommand):
 
 
     def handle(self, *args, **options):
-        if options['file_path'] and not options['products_list']:
-            path      = options['file_path']
-            file_list = os.listdir(path)
-        
-        elif options['file_path'] and options['products_list']:
-            path      = options['file_path']
-            file_list = options['products_list']
-        
-        file_list.sort()
-        
-        for file in file_list:
-            file_name      = file.replace(".zip", '')
-            PRODUCT_EXISTS = Product_l1c.objects.filter(identifier=file_name).exists()
-
-            if not PRODUCT_EXISTS:
-                try: 
-                    data_dict = self.generate_json(file_name)
-                    # print json.dumps(data_dict, indent=3, sort_keys=True, default=str)
-
-                    product_l1c = Product_l1c(
-                        uuid       = data_dict['id'],
-                        identifier = data_dict['Identifier'],
-                        file_path  = path,
-                        json       = json.dumps(data_dict, indent=3, sort_keys=True, ensure_ascii=True, default=str)
-                    )
-
-                    product_l1c.save()
-                    print "Id: " + data_dict['id']
+        ssh_stdin = ssh_stdout = ssh_stderr = None
+
+        try:
+            ssh.connect(SSH_ADDRESS, username=SSH_USERNAME)
+            ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(SSH_COMMAND)
+            path = ssh_stdout.read().replace("\n", '')
+
+            sftp = ssh.open_sftp()  ## Creates SFTPClient() object
+            sftp.chdir(path)
+            file_list = sftp.listdir()
+
+            for file in file_list:
+                file_name      = file.replace(".zip", '')
+                PRODUCT_EXISTS = Product_l1c.objects.filter(identifier=file_name).exists()
+
+                if not PRODUCT_EXISTS:
+                    try: 
+                        data_dict = self.generate_json(file_name)
+                        # print json.dumps(data_dict, indent=3, sort_keys=True, default=str)
+
+                        product_l1c = Product_l1c(
+                            uuid       = data_dict['id'],
+                            identifier = data_dict['Identifier'],
+                            file_path  = path,
+                            json       = json.dumps(data_dict, indent=3, sort_keys=True, ensure_ascii=True, default=str)
+                        )
+
+                        product_l1c.save()
+                        print "NEW product with UUID: " + data_dict['id']
+                    
+                    except:
+                        print "ERROR with file " + file_name
                 
-                except:
-                    print "Error con el archivo " + file_name
-            
-            else:
-                print "Product " + file_name + " is already in DB."
+                # else:
+                #     print "Product " + file_name + " is ALREADY in DB."
+        
+        except Exception as e:
+            sys.stderr.write("SSH connection error: {0}".format(e))
+
+        if ssh_stdout:
+            sys.stdout.write(ssh_stdout.read())
+        if ssh_stderr:
+            sys.stderr.write(ssh_stderr.read())
+
+        sftp.close()
+        ssh.close()