Skip to content

GeoSentinel
GeoSentinel
Commit 7bf86fe7 authored by Mario Chirinos's avatar Mario Chirinos
Browse files
Options

api v2

parent c5e52e4c
Show whitespace changes
Inline Side-by-side
Showing with 139 additions and 119 deletions
geosentinel/APISentinel.py
View file @ 7bf86fe7
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
""" """
Install: Install:
$ sudo pip3 install sentinelsat # $ sudo pip3 install sentinelsat
$ sudo apt-get install libgdal-dev $ sudo apt-get install libgdal-dev
Example: Example:
...@@ -12,45 +12,54 @@ Example: ...@@ -12,45 +12,54 @@ Example:
""" """
import os import os
from sentinelsat.sentinel import SentinelAPI from oauthlib.oauth2 import BackendApplicationClient
from requests_oauthlib import OAuth2Session
#from sentinelsat.sentinel import SentinelAPI
from datetime import date from datetime import date
from collections import OrderedDict from collections import OrderedDict
import time import time
import shapely.wkt
import shapely.geometry
import json
#from osgeo import ogr #from osgeo import ogr
'C4+rC@W>7C5s$b'
def sentinelhub_compliance_hook(response):
response.raise_for_status()
return response
class APISentinel(object): class APISentinel(object):
""" Class for Sentinel satellites configuration """ Class for Sentinel satellites configuration
Test Case Test Case
>>> sentinel = APISentinel('mario-chirinos', 'r4nc0r4u') >>> sentinel = APISentinel("cfg.json")
>>> products = sentinel.getProducts("POLYGON((-89.99450683593753 21.279137394108716,-89.13757324218751 21.2996106049456,-89.30236816406251 20.68418377935238,-90.0494384765625 20.715015145512098,-89.99450683593753 21.279137394108716))", ('20151219', date(2015,12,29)), {"platformname":"Sentinel-2"}) >>> products = sentinel.getProducts("POLYGON((-89.99450683593753 21.279137394108716,-89.13757324218751 21.2996106049456,-89.30236816406251 20.68418377935238,-90.0494384765625 20.715015145512098,-89.99450683593753 21.279137394108716))", ('20151219', date(2015,12,29)), {"platformname":"Sentinel-2"})
>>> print(len(products)) >>> print(len(products))
6 6
""" """
def __init__(self,usernam,passw): def __init__(self, client_id, client_secret, s3cfg):
"""The constructor Initialize Sentinel Data. """The constructor Initialize Sentinel Data.
Args: Args:
self: The object pointer. self: The object pointer.
usernam (str): Username, access credentials to https://scihub.copernicus.eu/dhus/#/home. cfg: JSON configuration file.
passw (str): Password, access credentials to https://scihub.copernicus.eu/dhus/#/home.
Returns: Returns:
pointer: The object pointer. pointer: The object pointer.
""" """
self.username = usernam self.client_id = client_id
self.password = passw self.client_secret = client_secret
# self.URL_Sentinel = 'https://scihub.copernicus.eu/dhus' self.s3cfg=s3cfg
# self.URL_Sentinel = 'https://scihub.copernicus.eu/apihub' self.oauth_token_url='https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token'
self.URL_Sentinel = 'https://apihub.copernicus.eu/apihub/'
# self.plot_width = 7 # Create a session
# self.plot_height = 7 client = BackendApplicationClient(client_id=client_id)
# self.intersection_th = 30 self.oauth = OAuth2Session(client=client)
self.api = SentinelAPI(self.username, self.password, self.URL_Sentinel) self.oauth.register_compliance_hook("access_token_response", sentinelhub_compliance_hook)
# Get token for the session
def getProducts(self, area, date, searchParameters, areaRelation="Intersects"): self.token = self.oauth.fetch_token(token_url=self.oauth_token_url, client_secret=client_secret)
def getProducts(self, area, date, searchParameters):
"""Gets Sentinel products list that match the search parameters. """Gets Sentinel products list that match the search parameters.
Args: Args:
...@@ -58,33 +67,38 @@ class APISentinel(object): ...@@ -58,33 +67,38 @@ class APISentinel(object):
area (str): Area of interest using a Polygon in WKT format. area (str): Area of interest using a Polygon in WKT format.
date (tuple): A tuple contining the initial and end dates. date (tuple): A tuple contining the initial and end dates.
searchParameters (dict): Additional keywords can be used to specify other query parameters, searchParameters (dict): Additional keywords can be used to specify other query parameters,
Other Parameters: Additional keywords can be used to specify other query parameters,
e.g. {"platformname":"Sentinel-2", "cloudcoverpercentage":"[0 TO 40]"}
See https://scihub.copernicus.eu/twiki/do/view/SciHubUserGuide/3FullTextSearch for a full list. Ran-
ge values can be passed as two-element tuples, e.g. cloudcoverpercentage=(0, 30).
The time interval formats accepted by the `date` parameter can also be used with any other parameters that expect time intervals
(that is: 'beginposition', 'endposition', 'date', 'creationdate', and 'ingestiondate').
Example: Example:
getProducts("POLYGON((-89.99 21.27,-89.13 21.29,-89.30 20.68,-90.04 20.71,-89.99 21.27))", ('20150101', '20181024'), {"platformname":"Sentinel-2", "cloudcoverpercentage":"[0 TO 40]"}) getProducts("POLYGON((-89.99 21.27,-89.13 21.29,-89.30 20.68,-90.04 20.71,-89.99 21.27))", ('20150101', '20181024'), {"platformname":"Sentinel-2", "cloudcoverpercentage":"[0 TO 40]"})
Returns: Returns:
OrderedDict: Sentinel Product list found. OrderedDict: Sentinel Product list found.
Notes:
area_relation : {'Intersects', 'Contains', 'IsWithin'}, optional
What relation to use for testing the AOI. Case insensitive.
- Intersects: true if the AOI and the footprint intersect (default)
- Contains: true if the AOI is inside the footprint
- IsWithin: true if the footprint is inside the AOI
""" """
url = "https://sh.dataspace.copernicus.eu/api/v1/catalog/1.0.0/search"
data = {
products_list = self.api.query(area, date, area_relation=areaRelation, **searchParameters) "intersects": shapely.geometry.mapping(shapely.wkt.loads(area)),
return products_list "datetime": date[0].strftime('%Y-%m-%dT%H:%M:%SZ')+"/"+date[1].strftime('%Y-%m-%dT%H:%M:%SZ'),
"collections": ["sentinel-2-l2a"],
def downloadProducts(self,products,dir): "limit": 50,
}
data.update(searchParameters)
productList = {}
response = None
while response is None or response.status_code==200:
response = self.oauth.post(url, json=data)
content = json.loads(response.content)
for i in json.loads(response.content)["features"]:
productList[i["id"]]={"filename":i["assets"]["data"]["href"]}
if "next" in content["context"]:
data["next"]= content["context"]["next"]
print(content["context"])
else:
break;
return productList
def downloadProducts(self, products, outdir):
"""Download product(s) list Sentinel """Download product(s) list Sentinel
Args: Args:
...@@ -92,17 +106,22 @@ class APISentinel(object): ...@@ -92,17 +106,22 @@ class APISentinel(object):
products (OrderedDict): Product(s) to download. products (OrderedDict): Product(s) to download.
dir (str): destination directory. dir (str): destination directory.
""" """
print(products)
os.chdir(dir) os.chdir(outdir)
# self.api.download_all(products) # self.api.download_all(products)
longTermArchiveList=[] longTermArchiveList=[]
for p in products: for p in products:
print(products[p]['filename']) print(products[p]['filename'])
try: try:
info = self.api.download(p) # info = self.api.download(p)
if not info["Online"]: os.system("s3cmd -c "+self.s3cfg+" -r get "+products[p]['filename'])
print("Adding producto to long term list") base_id=p.replace(".SAFE","")
longTermArchiveList.append(p) os.system("zip -r "+base_id+".zip "+p+"/")
os.system("rm -r "+p)
# rm -r $DIRNAME/
# if not info["Online"]:
# print("Adding producto to long term list")
# longTermArchiveList.append(p)
time.sleep(1) time.sleep(1)
except: except:
......
geosentinel/SentinelHub/sentielhub.py
View file @ 7bf86fe7
...@@ -4,7 +4,8 @@ import sys ...@@ -4,7 +4,8 @@ import sys
from oauthlib.oauth2 import BackendApplicationClient from oauthlib.oauth2 import BackendApplicationClient
from requests_oauthlib import OAuth2Session from requests_oauthlib import OAuth2Session
import json import json
import shapely.wkt
import shapely.geometry
#Access key #Access key
#YZ7UPLDGVGGSHQTXSIUN #YZ7UPLDGVGGSHQTXSIUN
...@@ -36,28 +37,14 @@ def example(): ...@@ -36,28 +37,14 @@ def example():
# url = "https://sh.dataspace.copernicus.eu/api/v1/catalog/1.0.0/collections/sentinel-2-l2a" # url = "https://sh.dataspace.copernicus.eu/api/v1/catalog/1.0.0/collections/sentinel-2-l2a"
resp = oauth.get(url) resp = oauth.get(url)
# print(json.dumps(json.loads(resp.content), indent=2))
wkt = "POLYGON((-90.43670654296876 21.076811782249436,-89.65667724609375 21.37635972888924,-89.0606689453125 21.419833053493477,-88.20648193359374 21.703369345524266,-87.79449462890625 21.619132728904603,-87.53631591796875 21.53995662308543,-87.53631591796875 21.460737306938086,-90.25543212890625 20.8639448490769,-90.450439453125 20.840844707411335,-90.43670654296876 21.076811782249436))"
area = shapely.geometry.mapping(shapely.wkt.loads(wkt))
print(area)
print(json.dumps(json.loads(resp.content), indent=2))
data = { data = {
# "bbox": [13, 45, 14, 46], # "bbox": [13, 45, 14, 46],
"intersects":{ "intersects":area,
"type": "Polygon",
"coordinates": [
[
[-90.43670654296876, 21.076811782249436],
[-89.65667724609375, 21.37635972888924 ],
[-89.0606689453125 , 21.419833053493477],
[-88.20648193359374, 21.703369345524266],
[-87.79449462890625, 21.619132728904603],
[-87.53631591796875, 21.53995662308543 ],
[-87.53631591796875, 21.460737306938086],
[-90.25543212890625, 20.8639448490769 ],
[-90.450439453125 , 20.840844707411335],
[-90.43670654296876, 21.076811782249436]
]
]
},
"datetime": "2019-01-01T00:00:00Z/..", #"2019-12-10T23:59:59Z", "datetime": "2019-01-01T00:00:00Z/..", #"2019-12-10T23:59:59Z",
"collections": ["sentinel-2-l2a"], "collections": ["sentinel-2-l2a"],
"limit": 100, "limit": 100,
...@@ -73,14 +60,16 @@ def example(): ...@@ -73,14 +60,16 @@ def example():
# print(i["id"], i["properties"]["eo:cloud_cover"], i["properties"]["datetime"]) # print(i["id"], i["properties"]["eo:cloud_cover"], i["properties"]["datetime"])
## print(i.keys()) ## print(i.keys())
# print(i["assets"]["data"]["href"]) # print(i["assets"]["data"]["href"])
productList = [] productList = {}
response = None response = None
while response is None or response.status_code==200: while response is None or response.status_code==200:
response = oauth.post(url, json=data) response = oauth.post(url, json=data)
content = json.loads(response.content) content = json.loads(response.content)
for i in json.loads(response.content)["features"]: for i in json.loads(response.content)["features"]:
# print(i["assets"]["data"]["href"]) # print(i["assets"]["data"]["href"])
productList.append(i["assets"]["data"]["href"]) # print(i["id"])
# productList.append(i["assets"]["data"]["href"])
productList[i["id"]]={"filename":i["assets"]["data"]["href"]}
if "next" in content["context"]: if "next" in content["context"]:
data["next"]= content["context"]["next"] data["next"]= content["context"]["next"]
print(content["context"]) print(content["context"])
...@@ -91,7 +80,7 @@ def example(): ...@@ -91,7 +80,7 @@ def example():
for item in productList: for item in productList:
fp.write("%s\n" % item) fp.write("%s\n" % item)
print(response) print(response)
# print(productList)
#=============================================================================== #===============================================================================
def main(argv): def main(argv):
......
geosentinel/findProducts.py
View file @ 7bf86fe7
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
import sys, os import sys, os
import json import json
if __name__ == "__main__": # Local Run if __name__ == "__main__": # Local Run
import APISentinel import APISentinel
import polygonToBox import polygonToBox
...@@ -14,12 +15,12 @@ else: # Module Run, When going production - delete if/else ...@@ -14,12 +15,12 @@ else: # Module Run, When going production - delete if/else
from osgeo import ogr from osgeo import ogr
from datetime import date from datetime import date
from collections import OrderedDict from collections import OrderedDict
from datetime import datetime
productType = {"L1C":"S2MSI1C", "L2A":"S2MSI2A"} productType = {"L1C":"S2MSI1C", "L2A":"S2MSI2A"}
#=============================================================================== #===============================================================================
def findSentinelProducts(cfg, uinput = False, limitto = 0):#main(argv): def findSentinelProducts(cfg, uinput = False, limitto = 0):#main(argv):
sentinel = APISentinel.APISentinel(cfg['username'], cfg['password']) sentinel = APISentinel.APISentinel(cfg['client_id'], cfg['client_secret'], cfg["s3cfg"])
geometry = ogr.CreateGeometryFromWkt(cfg['wkt']) geometry = ogr.CreateGeometryFromWkt(cfg['wkt'])
wktList = [] wktList = []
...@@ -38,76 +39,73 @@ def findSentinelProducts(cfg, uinput = False, limitto = 0):#main(argv): ...@@ -38,76 +39,73 @@ def findSentinelProducts(cfg, uinput = False, limitto = 0):#main(argv):
productList = OrderedDict() productList = OrderedDict()
for wkt in wktList: for wkt in wktList:
tmpList =sentinel.getProducts(wkt, (cfg['startDate'], cfg['endDate']), {"producttype":productType[cfg["productLevel"]], "platformname":cfg['platform'], "cloudcoverpercentage":"[0 TO "+str(cfg['clouds'])+"]"}, areaRelation="Contains") tmpList =sentinel.getProducts(wkt, ( datetime.strptime(cfg['startDate'], '%Y%m%d'), datetime.strptime(cfg['endDate'], '%Y%m%d')), {"filter": "eo:cloud_cover<3"})
if len(productList)<=0:
print ("No products found with 'Contains' trying with 'Intersects'")
tmpList = sentinel.getProducts(wkt, (cfg['startDate'], cfg['endDate']), {"producttype":productType[cfg["productLevel"]], "platformname":cfg['platform'], "cloudcoverpercentage":"[0 TO "+str(cfg['clouds'])+"]"})
productList.update(tmpList) productList.update(tmpList)
if limitto>0: # if limitto>0:
tilesdic={} # tilesdic={}
for k, v in productList.items(): # for k, v in productList.items():
# print("productList v", v) ## print("productList v", v)
tile = v["title"][38:44] # tile = v["title"][38:44]
date = v["title"][11:19] # date = v["title"][11:19]
if tile not in tilesdic: # if tile not in tilesdic:
tilesdic[tile]={} # tilesdic[tile]={}
tilesdic[tile][k]=v # tilesdic[tile][k]=v
for k,v in tilesdic.items(): # for k,v in tilesdic.items():
tilesdic[k] = {i[0]:i[1] for i in sorted(v.items(), key=lambda x: x[1]["title"][11:19], reverse=True)} # tilesdic[k] = {i[0]:i[1] for i in sorted(v.items(), key=lambda x: x[1]["title"][11:19], reverse=True)}
productList = {list(tilesdic[k].keys())[0]: list(tilesdic[k].items())[0][1] for k in tilesdic.keys()} # productList = {list(tilesdic[k].keys())[0]: list(tilesdic[k].items())[0][1] for k in tilesdic.keys()}
fileNames = [productList[k]['filename'].replace("SAFE", "zip") for k in productList.keys() ] fileNames = [k.replace("SAFE", "zip") for k in productList.keys() ]
downloadDir = cfg["productsDir"]+cfg["productLevel"]+"/" downloadDir = cfg["productsDir"]+cfg["productLevel"]+"/"
dirList = os.listdir(downloadDir) dirList = os.listdir(downloadDir)
matchingProducts = set(fileNames).intersection(set(dirList)) matchingProducts = set(fileNames).intersection(set(dirList))
print ( str(len(matchingProducts))+" of " + str(len(fileNames)) +" "+ cfg["productLevel"] + " products found.") print ( str(len(matchingProducts))+" of " + str(len(fileNames)) +" "+ cfg["productLevel"] + " products found.")
matchingProducts = [i.replace("zip", "SAFE") for i in matchingProducts]
downloadList = {k:v for k,v in productList.items() if k not in matchingProducts}
text ="" text =""
if uinput==True: if uinput==True:
while text != "yes" and text != "no": while text != "yes" and text != "no":
text = input("Do you want to download this products to "+downloadDir+ " ? (yes, no)") text = input("Do you want to download this products to "+downloadDir+ " ? (yes, no)")
if uinput==False or text=="yes": if uinput==False or text=="yes":
longTermArchiveList = sentinel.downloadProducts(productList,downloadDir) longTermArchiveList = sentinel.downloadProducts(downloadList, downloadDir)
if len(longTermArchiveList)>0: if len(longTermArchiveList)>0:
print( str(len(longTermArchiveList)) + " products are in the long term archive" ) print( str(len(longTermArchiveList)) + " products are in the long term archive" )
with open(cfg["projectDir"]+'longTermArchiv.json', 'w') as outfile: with open(cfg["projectDir"]+'longTermArchiv.json', 'w') as outfile:
json.dump(longTermArchiveList, outfile) json.dump(longTermArchiveList, outfile)
L2ADir = cfg["productsDir"]+"L2A/" L2ADir = cfg["productsDir"]+"L2A/"
if cfg["productLevel"]=="L1C": # if cfg["productLevel"]=="L1C":
fileNamesL2A = [productList[k]['filename'].replace("SAFE", "zip").replace("L1C", "L2A") for k in productList.keys() ] # fileNamesL2A = [productList[k]['filename'].replace("SAFE", "zip").replace("L1C", "L2A") for k in productList.keys() ]
L1CDir = cfg["productsDir"]+"L1C/" # L1CDir = cfg["productsDir"]+"L1C/"
linksDir=cfg["projectDir"]+"L1C/" # linksDir=cfg["projectDir"]+"L1C/"
if not os.path.exists(linksDir): # if not os.path.exists(linksDir):
os.mkdir(linksDir) # os.mkdir(linksDir)
L1CDirList = os.listdir(L1CDir) # L1CDirList = os.listdir(L1CDir)
#LinkProducts # #LinkProducts
fileNames = [productList[k]['filename'].replace("SAFE", "zip") for k in productList.keys() ] # fileNames = [productList[k]['filename'].replace("SAFE", "zip") for k in productList.keys() ]
matchingProducts = set(fileNames).intersection(set(L1CDirList)) # matchingProducts = set(fileNames).intersection(set(L1CDirList))
for f in matchingProducts: # for f in matchingProducts:
print (f) # print (f)
print("ln -s " + L1CDir+f + " " + linksDir+f) # print("ln -s " + L1CDir+f + " " + linksDir+f)
os.system("ln -s " + L1CDir+f + " " + linksDir+f) # os.system("ln -s " + L1CDir+f + " " + linksDir+f)
print (str(len(matchingProducts)) + " Linked to " + linksDir) # print (str(len(matchingProducts)) + " Linked to " + linksDir)
print ( str(len(matchingProducts))+" of " + str(len(fileNames)) +" L1C products found in L2A.") # print ( str(len(matchingProducts))+" of " + str(len(fileNames)) +" L1C products found in L2A.")
text ="" # text =""
if uinput==True: # if uinput==True:
while text != "yes" and text != "no": # while text != "yes" and text != "no":
text = input("Do you want to convert this products and save them to "+downloadDir+ " ? (yes, no)") # text = input("Do you want to convert this products and save them to "+downloadDir+ " ? (yes, no)")
if uinput==False or text=="yes": # if uinput==False or text=="yes":
#print("L1CProductListToL2A.sh "+rawDir+" "+inDir+" 1") # #print("L1CProductListToL2A.sh "+rawDir+" "+inDir+" 1")
os.system("L1CProductListToL2A.sh "+linksDir+" "+L2ADir+" 1") # os.system("L1CProductListToL2A.sh "+linksDir+" "+L2ADir+" 1")
text ="" text =""
linksDir=cfg["projectDir"]+"L2A/" linksDir=cfg["projectDir"]+"L2A/"
......
geosentinel/findproducts.json 0 → 100644
View file @ 7bf86fe7
{
"wkt":"MULTIPOLYGON (((-98.3355929947266 19.5006236010058,-98.3355925430505 19.2506346897204,-98.3355920983444 19.0006458421814,-98.6689061346556 19.0006453020417,-99.0022125310807 19.0006447462906,-99.3355189169198 19.0006441749349,-99.668832921472 19.0006435879802,-100.00213928526 19.0006429854733,-100.002139795108 19.2506317969208,-100.002140312946 19.5006206721727,-100.002140838832 19.7506096122778,-99.6688344354453 19.7506102375839,-99.3355203912422 19.7506108467495,-99.0022139657033 19.7506114397257,-98.6689075295298 19.7506120165068,-98.3355934534222 19.7506125770858,-98.3355929947266 19.5006236010058)))",
"platform": "Sentinel-2",
"productLevel":"L2A",
"startDate": "20230801",
"endDate": "20231001",
"clouds":"5",
"productsDir":"/data/sentinelImages/",
"projectDir": "/data/sentinel-temp/",
"client_id": "sh-933e0ce8-1d52-487b-aa33-358524d21cee",
"client_secret":"R4a8WW4cuUPVdJXZGwr0BU5QfyMLho9l",
"s3cfg":"/data/git/GeoSentinel/geosentinel/SentinelHub/.s3cfg"
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment