Commit 5adbe140 authored by Renán Sosa Guillen's avatar Renán Sosa Guillen

L1C products population

parent 96c36358
...@@ -5,11 +5,7 @@ ...@@ -5,11 +5,7 @@
Usage: Usage:
$ cd GeoInt_SIDT $ cd GeoInt_SIDT
$ python manage.py populate_products_l1c --file_path [path_where_files_are_located] $ python manage.py populate_products_l1c
Example:
$ cd GeoInt_SIDT
$ python manage.py populate_products_l1c --file_path /home/geointdev/sidt-env/sidt-zip/L1C/
""" """
...@@ -17,7 +13,7 @@ from django.core.management.base import BaseCommand, CommandError ...@@ -17,7 +13,7 @@ from django.core.management.base import BaseCommand, CommandError
from sentinelsat.sentinel import SentinelAPI from sentinelsat.sentinel import SentinelAPI
from catalog.models import Product_l1c from catalog.models import Product_l1c
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
import os, requests, json import os, sys, requests, json, paramiko
usr = "emmhp" usr = "emmhp"
...@@ -26,16 +22,16 @@ URL_Sentinel = 'https://scihub.copernicus.eu/dhus' ...@@ -26,16 +22,16 @@ URL_Sentinel = 'https://scihub.copernicus.eu/dhus'
apiSentinel = SentinelAPI(usr, pwd, URL_Sentinel) apiSentinel = SentinelAPI(usr, pwd, URL_Sentinel)
SSH_ADDRESS = "192.168.1.59"
SSH_USERNAME = "geoint"
SSH_COMMAND = "cd NAS/sentinelImages/L1C/; pwd"
class Command(BaseCommand): ssh = paramiko.SSHClient()
help = "Populates DB with downloaded L1C products." ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
def add_arguments(self, parser):
## positional arguments
# parser.add_argument('file_path', nargs=1, type=str)
parser.add_argument('--file_path', dest='file_path')
parser.add_argument('--products_list', dest='products_list')
class Command(BaseCommand):
help = "Populates DB with downloaded L1C products."
def generate_json(self, product_name): def generate_json(self, product_name):
""" """
...@@ -44,13 +40,12 @@ class Command(BaseCommand): ...@@ -44,13 +40,12 @@ class Command(BaseCommand):
uri = "https://scihub.copernicus.eu/dhus/odata/v1/Products?$filter=Name eq '" + product_name + "'" uri = "https://scihub.copernicus.eu/dhus/odata/v1/Products?$filter=Name eq '" + product_name + "'"
r = requests.get(uri, auth=(usr, pwd)) r = requests.get(uri, auth=(usr, pwd))
root = ET.fromstring(r.content) root = ET.fromstring(r.content)
print root # print root
# base_link = root[5][0].text # base_link = root[5][0].text
try: try:
uuid = root[5][11][0].text uuid = root[5][11][0].text
except: except:
print root pass
print "\n"
data_dict = apiSentinel.get_product_odata(uuid, full=True) data_dict = apiSentinel.get_product_odata(uuid, full=True)
...@@ -58,37 +53,49 @@ class Command(BaseCommand): ...@@ -58,37 +53,49 @@ class Command(BaseCommand):
def handle(self, *args, **options): def handle(self, *args, **options):
if options['file_path'] and not options['products_list']: ssh_stdin = ssh_stdout = ssh_stderr = None
path = options['file_path']
file_list = os.listdir(path) try:
ssh.connect(SSH_ADDRESS, username=SSH_USERNAME)
elif options['file_path'] and options['products_list']: ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(SSH_COMMAND)
path = options['file_path'] path = ssh_stdout.read().replace("\n", '')
file_list = options['products_list']
sftp = ssh.open_sftp() ## Creates SFTPClient() object
file_list.sort() sftp.chdir(path)
file_list = sftp.listdir()
for file in file_list:
file_name = file.replace(".zip", '') for file in file_list:
PRODUCT_EXISTS = Product_l1c.objects.filter(identifier=file_name).exists() file_name = file.replace(".zip", '')
PRODUCT_EXISTS = Product_l1c.objects.filter(identifier=file_name).exists()
if not PRODUCT_EXISTS:
try: if not PRODUCT_EXISTS:
data_dict = self.generate_json(file_name) try:
# print json.dumps(data_dict, indent=3, sort_keys=True, default=str) data_dict = self.generate_json(file_name)
# print json.dumps(data_dict, indent=3, sort_keys=True, default=str)
product_l1c = Product_l1c(
uuid = data_dict['id'], product_l1c = Product_l1c(
identifier = data_dict['Identifier'], uuid = data_dict['id'],
file_path = path, identifier = data_dict['Identifier'],
json = json.dumps(data_dict, indent=3, sort_keys=True, ensure_ascii=True, default=str) file_path = path,
) json = json.dumps(data_dict, indent=3, sort_keys=True, ensure_ascii=True, default=str)
)
product_l1c.save()
print "Id: " + data_dict['id'] product_l1c.save()
print "NEW product with UUID: " + data_dict['id']
except:
print "ERROR with file " + file_name
except: # else:
print "Error con el archivo " + file_name # print "Product " + file_name + " is ALREADY in DB."
else: except Exception as e:
print "Product " + file_name + " is already in DB." sys.stderr.write("SSH connection error: {0}".format(e))
if ssh_stdout:
sys.stdout.write(ssh_stdout.read())
if ssh_stderr:
sys.stderr.write(ssh_stderr.read())
sftp.close()
ssh.close()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment