Skip to content

GeoSentinel
GeoSentinel
Commit 7bf86fe7 authored by Mario Chirinos's avatar Mario Chirinos
Browse files
Options

api v2

parent c5e52e4c
Show whitespace changes
Inline Side-by-side
Showing with 139 additions and 119 deletions
geosentinel/APISentinel.py
View file @ 7bf86fe7
......@@ -3,7 +3,7 @@
"""
Install:
$ sudo pip3 install sentinelsat
# $ sudo pip3 install sentinelsat
$ sudo apt-get install libgdal-dev
Example:
......@@ -12,45 +12,54 @@ Example:
"""
import os
from sentinelsat.sentinel import SentinelAPI
from oauthlib.oauth2 import BackendApplicationClient
from requests_oauthlib import OAuth2Session
#from sentinelsat.sentinel import SentinelAPI
from datetime import date
from collections import OrderedDict
import time
import shapely.wkt
import shapely.geometry
import json
#from osgeo import ogr
'C4+rC@W>7C5s$b'
def sentinelhub_compliance_hook(response):
response.raise_for_status()
return response
class APISentinel(object):
""" Class for Sentinel satellites configuration
Test Case
>>> sentinel = APISentinel('mario-chirinos', 'r4nc0r4u')
>>> sentinel = APISentinel("cfg.json")
>>> products = sentinel.getProducts("POLYGON((-89.99450683593753 21.279137394108716,-89.13757324218751 21.2996106049456,-89.30236816406251 20.68418377935238,-90.0494384765625 20.715015145512098,-89.99450683593753 21.279137394108716))", ('20151219', date(2015,12,29)), {"platformname":"Sentinel-2"})
>>> print(len(products))
6
"""
def __init__(self,usernam,passw):
def __init__(self, client_id, client_secret, s3cfg):
"""The constructor Initialize Sentinel Data.
Args:
self: The object pointer.
usernam (str): Username, access credentials to https://scihub.copernicus.eu/dhus/#/home.
passw (str): Password, access credentials to https://scihub.copernicus.eu/dhus/#/home.
cfg: JSON configuration file.
Returns:
pointer: The object pointer.
"""
self.username = usernam
self.password = passw
# self.URL_Sentinel = 'https://scihub.copernicus.eu/dhus'
# self.URL_Sentinel = 'https://scihub.copernicus.eu/apihub'
self.URL_Sentinel = 'https://apihub.copernicus.eu/apihub/'
# self.plot_width = 7
# self.plot_height = 7
# self.intersection_th = 30
self.api = SentinelAPI(self.username, self.password, self.URL_Sentinel)
def getProducts(self, area, date, searchParameters, areaRelation="Intersects"):
self.client_id = client_id
self.client_secret = client_secret
self.s3cfg=s3cfg
self.oauth_token_url='https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token'
# Create a session
client = BackendApplicationClient(client_id=client_id)
self.oauth = OAuth2Session(client=client)
self.oauth.register_compliance_hook("access_token_response", sentinelhub_compliance_hook)
# Get token for the session
self.token = self.oauth.fetch_token(token_url=self.oauth_token_url, client_secret=client_secret)
def getProducts(self, area, date, searchParameters):
"""Gets Sentinel products list that match the search parameters.
Args:
......@@ -58,33 +67,38 @@ class APISentinel(object):
area (str): Area of interest using a Polygon in WKT format.
date (tuple): A tuple contining the initial and end dates.
searchParameters (dict): Additional keywords can be used to specify other query parameters,
Other Parameters: Additional keywords can be used to specify other query parameters,
e.g. {"platformname":"Sentinel-2", "cloudcoverpercentage":"[0 TO 40]"}
See https://scihub.copernicus.eu/twiki/do/view/SciHubUserGuide/3FullTextSearch for a full list. Ran-
ge values can be passed as two-element tuples, e.g. cloudcoverpercentage=(0, 30).
The time interval formats accepted by the `date` parameter can also be used with any other parameters that expect time intervals
(that is: 'beginposition', 'endposition', 'date', 'creationdate', and 'ingestiondate').
Example:
getProducts("POLYGON((-89.99 21.27,-89.13 21.29,-89.30 20.68,-90.04 20.71,-89.99 21.27))", ('20150101', '20181024'), {"platformname":"Sentinel-2", "cloudcoverpercentage":"[0 TO 40]"})
Returns:
OrderedDict: Sentinel Product list found.
Notes:
area_relation : {'Intersects', 'Contains', 'IsWithin'}, optional
What relation to use for testing the AOI. Case insensitive.
- Intersects: true if the AOI and the footprint intersect (default)
- Contains: true if the AOI is inside the footprint
- IsWithin: true if the footprint is inside the AOI
"""
products_list = self.api.query(area, date, area_relation=areaRelation, **searchParameters)
return products_list
def downloadProducts(self,products,dir):
url = "https://sh.dataspace.copernicus.eu/api/v1/catalog/1.0.0/search"
data = {
"intersects": shapely.geometry.mapping(shapely.wkt.loads(area)),
"datetime": date[0].strftime('%Y-%m-%dT%H:%M:%SZ')+"/"+date[1].strftime('%Y-%m-%dT%H:%M:%SZ'),
"collections": ["sentinel-2-l2a"],
"limit": 50,
}
data.update(searchParameters)
productList = {}
response = None
while response is None or response.status_code==200:
response = self.oauth.post(url, json=data)
content = json.loads(response.content)
for i in json.loads(response.content)["features"]:
productList[i["id"]]={"filename":i["assets"]["data"]["href"]}
if "next" in content["context"]:
data["next"]= content["context"]["next"]
print(content["context"])
else:
break;
return productList
def downloadProducts(self, products, outdir):
"""Download product(s) list Sentinel
Args:
......@@ -92,17 +106,22 @@ class APISentinel(object):
products (OrderedDict): Product(s) to download.
dir (str): destination directory.
"""
os.chdir(dir)
print(products)
os.chdir(outdir)
# self.api.download_all(products)
longTermArchiveList=[]
for p in products:
print(products[p]['filename'])
try:
info = self.api.download(p)
if not info["Online"]:
print("Adding producto to long term list")
longTermArchiveList.append(p)
# info = self.api.download(p)
os.system("s3cmd -c "+self.s3cfg+" -r get "+products[p]['filename'])
base_id=p.replace(".SAFE","")
os.system("zip -r "+base_id+".zip "+p+"/")
os.system("rm -r "+p)
# rm -r $DIRNAME/
# if not info["Online"]:
# print("Adding producto to long term list")
# longTermArchiveList.append(p)
time.sleep(1)
except:
......
geosentinel/SentinelHub/sentielhub.py
View file @ 7bf86fe7
......@@ -4,7 +4,8 @@ import sys
from oauthlib.oauth2 import BackendApplicationClient
from requests_oauthlib import OAuth2Session
import json
import shapely.wkt
import shapely.geometry
#Access key
#YZ7UPLDGVGGSHQTXSIUN
......@@ -36,28 +37,14 @@ def example():
# url = "https://sh.dataspace.copernicus.eu/api/v1/catalog/1.0.0/collections/sentinel-2-l2a"
resp = oauth.get(url)
# print(json.dumps(json.loads(resp.content), indent=2))
wkt = "POLYGON((-90.43670654296876 21.076811782249436,-89.65667724609375 21.37635972888924,-89.0606689453125 21.419833053493477,-88.20648193359374 21.703369345524266,-87.79449462890625 21.619132728904603,-87.53631591796875 21.53995662308543,-87.53631591796875 21.460737306938086,-90.25543212890625 20.8639448490769,-90.450439453125 20.840844707411335,-90.43670654296876 21.076811782249436))"
area = shapely.geometry.mapping(shapely.wkt.loads(wkt))
print(area)
print(json.dumps(json.loads(resp.content), indent=2))
data = {
# "bbox": [13, 45, 14, 46],
"intersects":{
"type": "Polygon",
"coordinates": [
[
[-90.43670654296876, 21.076811782249436],
[-89.65667724609375, 21.37635972888924 ],
[-89.0606689453125 , 21.419833053493477],
[-88.20648193359374, 21.703369345524266],
[-87.79449462890625, 21.619132728904603],
[-87.53631591796875, 21.53995662308543 ],
[-87.53631591796875, 21.460737306938086],
[-90.25543212890625, 20.8639448490769 ],
[-90.450439453125 , 20.840844707411335],
[-90.43670654296876, 21.076811782249436]
]
]
},
"intersects":area,
"datetime": "2019-01-01T00:00:00Z/..", #"2019-12-10T23:59:59Z",
"collections": ["sentinel-2-l2a"],
"limit": 100,
......@@ -73,14 +60,16 @@ def example():
# print(i["id"], i["properties"]["eo:cloud_cover"], i["properties"]["datetime"])
## print(i.keys())
# print(i["assets"]["data"]["href"])
productList = []
productList = {}
response = None
while response is None or response.status_code==200:
response = oauth.post(url, json=data)
content = json.loads(response.content)
for i in json.loads(response.content)["features"]:
# print(i["assets"]["data"]["href"])
productList.append(i["assets"]["data"]["href"])
# print(i["id"])
# productList.append(i["assets"]["data"]["href"])
productList[i["id"]]={"filename":i["assets"]["data"]["href"]}
if "next" in content["context"]:
data["next"]= content["context"]["next"]
print(content["context"])
......@@ -91,7 +80,7 @@ def example():
for item in productList:
fp.write("%s\n" % item)
print(response)
# print(productList)
#===============================================================================
def main(argv):
......
geosentinel/findProducts.py
View file @ 7bf86fe7
......@@ -4,6 +4,7 @@
import sys, os
import json
if __name__ == "__main__": # Local Run
import APISentinel
import polygonToBox
......@@ -14,12 +15,12 @@ else: # Module Run, When going production - delete if/else
from osgeo import ogr
from datetime import date
from collections import OrderedDict
from datetime import datetime
productType = {"L1C":"S2MSI1C", "L2A":"S2MSI2A"}
#===============================================================================
def findSentinelProducts(cfg, uinput = False, limitto = 0):#main(argv):
sentinel = APISentinel.APISentinel(cfg['username'], cfg['password'])
sentinel = APISentinel.APISentinel(cfg['client_id'], cfg['client_secret'], cfg["s3cfg"])
geometry = ogr.CreateGeometryFromWkt(cfg['wkt'])
wktList = []
......@@ -38,76 +39,73 @@ def findSentinelProducts(cfg, uinput = False, limitto = 0):#main(argv):
productList = OrderedDict()
for wkt in wktList:
tmpList =sentinel.getProducts(wkt, (cfg['startDate'], cfg['endDate']), {"producttype":productType[cfg["productLevel"]], "platformname":cfg['platform'], "cloudcoverpercentage":"[0 TO "+str(cfg['clouds'])+"]"}, areaRelation="Contains")
if len(productList)<=0:
print ("No products found with 'Contains' trying with 'Intersects'")
tmpList = sentinel.getProducts(wkt, (cfg['startDate'], cfg['endDate']), {"producttype":productType[cfg["productLevel"]], "platformname":cfg['platform'], "cloudcoverpercentage":"[0 TO "+str(cfg['clouds'])+"]"})
tmpList =sentinel.getProducts(wkt, ( datetime.strptime(cfg['startDate'], '%Y%m%d'), datetime.strptime(cfg['endDate'], '%Y%m%d')), {"filter": "eo:cloud_cover<3"})
productList.update(tmpList)
if limitto>0:
tilesdic={}
for k, v in productList.items():
# print("productList v", v)
tile = v["title"][38:44]
date = v["title"][11:19]
if tile not in tilesdic:
tilesdic[tile]={}
tilesdic[tile][k]=v
# if limitto>0:
# tilesdic={}
# for k, v in productList.items():
## print("productList v", v)
# tile = v["title"][38:44]
# date = v["title"][11:19]
# if tile not in tilesdic:
# tilesdic[tile]={}
# tilesdic[tile][k]=v
for k,v in tilesdic.items():
tilesdic[k] = {i[0]:i[1] for i in sorted(v.items(), key=lambda x: x[1]["title"][11:19], reverse=True)}
# for k,v in tilesdic.items():
# tilesdic[k] = {i[0]:i[1] for i in sorted(v.items(), key=lambda x: x[1]["title"][11:19], reverse=True)}
productList = {list(tilesdic[k].keys())[0]: list(tilesdic[k].items())[0][1] for k in tilesdic.keys()}
# productList = {list(tilesdic[k].keys())[0]: list(tilesdic[k].items())[0][1] for k in tilesdic.keys()}
fileNames = [productList[k]['filename'].replace("SAFE", "zip") for k in productList.keys() ]
fileNames = [k.replace("SAFE", "zip") for k in productList.keys() ]
downloadDir = cfg["productsDir"]+cfg["productLevel"]+"/"
dirList = os.listdir(downloadDir)
matchingProducts = set(fileNames).intersection(set(dirList))
print ( str(len(matchingProducts))+" of " + str(len(fileNames)) +" "+ cfg["productLevel"] + " products found.")
matchingProducts = [i.replace("zip", "SAFE") for i in matchingProducts]
downloadList = {k:v for k,v in productList.items() if k not in matchingProducts}
text =""
if uinput==True:
while text != "yes" and text != "no":
text = input("Do you want to download this products to "+downloadDir+ " ? (yes, no)")
if uinput==False or text=="yes":
longTermArchiveList = sentinel.downloadProducts(productList,downloadDir)
longTermArchiveList = sentinel.downloadProducts(downloadList, downloadDir)
if len(longTermArchiveList)>0:
print( str(len(longTermArchiveList)) + " products are in the long term archive" )
with open(cfg["projectDir"]+'longTermArchiv.json', 'w') as outfile:
json.dump(longTermArchiveList, outfile)
L2ADir = cfg["productsDir"]+"L2A/"
if cfg["productLevel"]=="L1C":
fileNamesL2A = [productList[k]['filename'].replace("SAFE", "zip").replace("L1C", "L2A") for k in productList.keys() ]
L1CDir = cfg["productsDir"]+"L1C/"
linksDir=cfg["projectDir"]+"L1C/"
if not os.path.exists(linksDir):
os.mkdir(linksDir)
L1CDirList = os.listdir(L1CDir)
#LinkProducts
fileNames = [productList[k]['filename'].replace("SAFE", "zip") for k in productList.keys() ]
matchingProducts = set(fileNames).intersection(set(L1CDirList))
for f in matchingProducts:
print (f)
print("ln -s " + L1CDir+f + " " + linksDir+f)
os.system("ln -s " + L1CDir+f + " " + linksDir+f)
print (str(len(matchingProducts)) + " Linked to " + linksDir)
print ( str(len(matchingProducts))+" of " + str(len(fileNames)) +" L1C products found in L2A.")
text =""
if uinput==True:
while text != "yes" and text != "no":
text = input("Do you want to convert this products and save them to "+downloadDir+ " ? (yes, no)")
if uinput==False or text=="yes":
#print("L1CProductListToL2A.sh "+rawDir+" "+inDir+" 1")
os.system("L1CProductListToL2A.sh "+linksDir+" "+L2ADir+" 1")
# if cfg["productLevel"]=="L1C":
# fileNamesL2A = [productList[k]['filename'].replace("SAFE", "zip").replace("L1C", "L2A") for k in productList.keys() ]
# L1CDir = cfg["productsDir"]+"L1C/"
# linksDir=cfg["projectDir"]+"L1C/"
# if not os.path.exists(linksDir):
# os.mkdir(linksDir)
# L1CDirList = os.listdir(L1CDir)
# #LinkProducts
# fileNames = [productList[k]['filename'].replace("SAFE", "zip") for k in productList.keys() ]
# matchingProducts = set(fileNames).intersection(set(L1CDirList))
# for f in matchingProducts:
# print (f)
# print("ln -s " + L1CDir+f + " " + linksDir+f)
# os.system("ln -s " + L1CDir+f + " " + linksDir+f)
# print (str(len(matchingProducts)) + " Linked to " + linksDir)
# print ( str(len(matchingProducts))+" of " + str(len(fileNames)) +" L1C products found in L2A.")
# text =""
# if uinput==True:
# while text != "yes" and text != "no":
# text = input("Do you want to convert this products and save them to "+downloadDir+ " ? (yes, no)")
# if uinput==False or text=="yes":
# #print("L1CProductListToL2A.sh "+rawDir+" "+inDir+" 1")
# os.system("L1CProductListToL2A.sh "+linksDir+" "+L2ADir+" 1")
text =""
linksDir=cfg["projectDir"]+"L2A/"
......
geosentinel/findproducts.json 0 → 100644
View file @ 7bf86fe7
{
"wkt":"MULTIPOLYGON (((-98.3355929947266 19.5006236010058,-98.3355925430505 19.2506346897204,-98.3355920983444 19.0006458421814,-98.6689061346556 19.0006453020417,-99.0022125310807 19.0006447462906,-99.3355189169198 19.0006441749349,-99.668832921472 19.0006435879802,-100.00213928526 19.0006429854733,-100.002139795108 19.2506317969208,-100.002140312946 19.5006206721727,-100.002140838832 19.7506096122778,-99.6688344354453 19.7506102375839,-99.3355203912422 19.7506108467495,-99.0022139657033 19.7506114397257,-98.6689075295298 19.7506120165068,-98.3355934534222 19.7506125770858,-98.3355929947266 19.5006236010058)))",
"platform": "Sentinel-2",
"productLevel":"L2A",
"startDate": "20230801",
"endDate": "20231001",
"clouds":"5",
"productsDir":"/data/sentinelImages/",
"projectDir": "/data/sentinel-temp/",
"client_id": "sh-933e0ce8-1d52-487b-aa33-358524d21cee",
"client_secret":"R4a8WW4cuUPVdJXZGwr0BU5QfyMLho9l",
"s3cfg":"/data/git/GeoSentinel/geosentinel/SentinelHub/.s3cfg"
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment