Skip to content

Commit

Permalink
added earthdata endpoint and a set of pytests that verify the server-…
Browse files Browse the repository at this point in the history 
…side CMR functionality
  • Loading branch information
@jpswinski
jpswinski committed Oct 25, 2024
1 parent a192596 commit 28b7671
Show file tree
Hide file tree
Showing 5 changed files with 166 additions and 23 deletions.
5 changes: 3 additions & 2 deletions clients/python/sliderule/earthdata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,9 @@
"ATL06": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []},
"ATL08": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []},
"ATL09": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []},
"ATL13": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": {".h5"}, "collections": {}},
"GEDI01_B": {"provider": "LPCLOUD", "version": "002", "api": "cmr", "formats": [".h5"], "collections": []},
"GEDI02_A": {"provider": "LPCLOUD", "version": "002", "api": "cmr", "formats": [".h5"], "collections": []},
"GEDI02_B": {"provider": "LPCLOUD", "version": "002", "api": "cmr", "formats": [".tiff"], "collections": []},
"GEDI_L3_LandSurface_Metrics_V2_1952": {"provider": "ORNL_CLOUD", "version": None, "api": "cmr", "formats": [".h5"], "collections": []},
"GEDI_L4A_AGB_Density_V2_1_2056": {"provider": "ORNL_CLOUD", "version": None, "api": "cmr", "formats": [".h5"], "collections": []},
"GEDI_L4B_Gridded_Biomass_2017": {"provider": "ORNL_CLOUD", "version": None, "api": "cmr", "formats": [".tiff"], "collections": []},
Expand All @@ -83,7 +83,7 @@
"gedil3-canopy-stddev": "GEDI_L3_LandSurface_Metrics_V2_1952",
"gedil3-counts": "GEDI_L3_LandSurface_Metrics_V2_1952",
"gedil2a": "GEDI02_A",
"gedil1b": "GEDI02_B",
"gedil1b": "GEDI01_B",
"swot-sim-ecco-llc4320": "SWOT_SIMULATED_L2_KARIN_SSH_ECCO_LLC4320_CALVAL_V1",
"swot-sim-glorys": "SWOT_SIMULATED_L2_KARIN_SSH_GLORYS_CALVAL_V1",
"usgs3dep-1meter-dem": "Digital Elevation Model (DEM) 1 meter",
Expand All @@ -92,6 +92,7 @@
"icesat2-atl06": "ATL06",
"icesat2-atl08": "ATL08",
"icesat2-atl09": "ATL09",
"icesat2-atl13": "ATL13",
"atlas-local": "ATL03",
"nsidc-s3": "ATL03"
}
Expand Down
127 changes: 125 additions & 2 deletions clients/python/tests/test_earthdata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@

import pytest
import sliderule
from sliderule import icesat2, earthdata
from sliderule import earthdata
from sliderule.earthdata import __cmr_collection_query as cmr_collection_query
from sliderule.earthdata import __cmr_max_version as cmr_max_version
from pathlib import Path
from datetime import datetime
import os
import json

Expand Down Expand Up @@ -85,4 +86,126 @@ def test_asstr(self):
def test_bad_short_name(self):
region = sliderule.toregion(os.path.join(TESTDIR, 'data/polygon.geojson'))
geojson = earthdata.tnm(short_name='DOES_NOT_EXIST', polygon=region["poly"], as_str=False)
assert len(geojson['features']) == 0
assert len(geojson['features']) == 0

#
# SlideRule Endpoint
#
@pytest.mark.network
class TestSlideRule:
def test_atl03(self, init):
parms = {"asset": "icesat2", "poly": grandmesa, "t0": '2018-10-01', "t1": '2019-12-01'}
resources = earthdata.search(parms)
rsps = sliderule.source("earthdata", parms)
assert init
assert len(rsps) == 26
assert len(rsps) == len(set(rsps))
assert len(rsps) == len(resources)
for resource in resources:
assert resource in rsps
assert datetime.strptime(resource[6:14], '%Y%m%d') >= datetime.strptime(parms["t0"], '%Y-%m-%d')
assert datetime.strptime(resource[6:14], '%Y%m%d') <= datetime.strptime(parms["t1"], '%Y-%m-%d')

def test_atl06(self, init):
parms = {"asset": "icesat2-atl06", "poly": grandmesa, "t0": '2018-10-01', "t1": '2019-12-01'}
resources = earthdata.search(parms)
rsps = sliderule.source("earthdata", parms)
assert init
assert len(rsps) == 26
assert len(rsps) == len(set(rsps))
assert len(rsps) == len(resources)
for resource in resources:
assert resource in rsps
assert datetime.strptime(resource[6:14], '%Y%m%d') >= datetime.strptime(parms["t0"], '%Y-%m-%d')
assert datetime.strptime(resource[6:14], '%Y%m%d') <= datetime.strptime(parms["t1"], '%Y-%m-%d')

def test_atl09(self, init):
parms = {"asset": "icesat2-atl09", "poly": grandmesa, "t0": '2018-10-01', "t1": '2019-12-01'}
resources = earthdata.search(parms)
rsps = sliderule.source("earthdata", parms)
assert init
assert len(rsps) == 36
assert len(rsps) == len(set(rsps))
assert len(rsps) == len(resources)
for resource in resources:
assert resource in rsps
assert datetime.strptime(resource[6:14], '%Y%m%d') >= datetime.strptime(parms["t0"], '%Y-%m-%d')
assert datetime.strptime(resource[6:14], '%Y%m%d') <= datetime.strptime(parms["t1"], '%Y-%m-%d')

def test_atl13(self, init):
saltlake = [
{
"lon": -111.78656302303546,
"lat": 40.474445992545355
},
{
"lon": -111.78656302303546,
"lat": 41.745511885629725
},
{
"lon": -113.25842634761666,
"lat": 41.745511885629725
},
{
"lon": -113.25842634761666,
"lat": 40.474445992545355
},
{
"lon": -111.78656302303546,
"lat": 40.474445992545355
}
]
parms = {"asset": "icesat2-atl13", "poly": saltlake, "t0": '2018-10-01', "t1": '2019-12-01'}
resources = earthdata.search(parms)
rsps = sliderule.source("earthdata", parms)
assert init
print(resources)
assert len(rsps) == 67
assert len(rsps) == len(set(rsps))
assert len(rsps) == len(resources)
for resource in resources:
assert resource in rsps
assert datetime.strptime(resource[6:14], '%Y%m%d') >= datetime.strptime(parms["t0"], '%Y-%m-%d')
assert datetime.strptime(resource[6:14], '%Y%m%d') <= datetime.strptime(parms["t1"], '%Y-%m-%d')

def test_gedil1b(self, init):
parms = {"asset": "gedil1b", "poly": grandmesa, "t0": '2018-10-01', "t1": '2019-12-01'}
resources = earthdata.search(parms)
rsps = sliderule.source("earthdata", parms)
print(resources)
assert init
assert len(rsps) == 26
assert len(rsps) == len(set(rsps))
assert len(rsps) == len(resources)
for resource in resources:
assert resource in rsps
assert datetime.strptime(resource[9:16], '%Y%j') >= datetime.strptime(parms["t0"], '%Y-%m-%d')
assert datetime.strptime(resource[9:16], '%Y%j') <= datetime.strptime(parms["t1"], '%Y-%m-%d')

def test_gedil2a(self, init):
parms = {"asset": "gedil2a", "poly": grandmesa, "t0": '2018-10-01', "t1": '2019-12-01'}
resources = earthdata.search(parms)
rsps = sliderule.source("earthdata", parms)
print(resources)
assert init
assert len(rsps) == 26
assert len(rsps) == len(set(rsps))
assert len(rsps) == len(resources)
for resource in resources:
assert resource in rsps
assert datetime.strptime(resource[9:16], '%Y%j') >= datetime.strptime(parms["t0"], '%Y-%m-%d')
assert datetime.strptime(resource[9:16], '%Y%j') <= datetime.strptime(parms["t1"], '%Y-%m-%d')

def test_gedil4a(self, init):
parms = {"asset": "gedil4a", "poly": grandmesa, "t0": '2018-10-01', "t1": '2019-12-01'}
resources = earthdata.search(parms)
rsps = sliderule.source("earthdata", parms)
print(resources)
assert init
assert len(rsps) == 26
assert len(rsps) == len(set(rsps))
assert len(rsps) == len(resources)
for resource in resources:
assert resource in rsps
assert datetime.strptime(resource[9:16], '%Y%j') >= datetime.strptime(parms["t0"], '%Y-%m-%d')
assert datetime.strptime(resource[9:16], '%Y%j') <= datetime.strptime(parms["t1"], '%Y-%m-%d')
1 change: 1 addition & 0 deletions scripts/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
install (
FILES
${CMAKE_CURRENT_LIST_DIR}/endpoints/definition.lua
${CMAKE_CURRENT_LIST_DIR}/endpoints/earthdata.lua
${CMAKE_CURRENT_LIST_DIR}/endpoints/event.lua
${CMAKE_CURRENT_LIST_DIR}/selftests/example_engine_endpoint.lua
${CMAKE_CURRENT_LIST_DIR}/selftests/example_source_endpoint.lua
Expand Down
17 changes: 17 additions & 0 deletions scripts/endpoints/earthdata.lua
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
--
-- ENDPOINT: /source/earthdata
--
-- INPUT: none
--
-- OUTPUT: json string of default parameters
--
local json = require("json")
local earthdata = require("earth_data_query")
local parm = json.decode(arg[1])
local status,rsps = earthdata.search(parm)
if status == earthdata.SUCCESS then
return json.encode(rsps)
else
return json.encode({error=rsps})
end

39 changes: 20 additions & 19 deletions scripts/extensions/earth_data_query.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,6 @@

local json = require("json")


local pp = require("prettyprint")

--
-- Constants
--
Expand All @@ -24,7 +21,6 @@ DATASETS = {
ATL13 = {provider = "NSIDC_CPRD", version = "006", api = "cmr", formats = {".h5"}, collections = {}},
GEDI01_B = {provider = "LPCLOUD", version = "002", api = "cmr", formats = {".h5"}, collections = {}},
GEDI02_A = {provider = "LPCLOUD", version = "002", api = "cmr", formats = {".h5"}, collections = {}},
GEDI02_B = {provider = "LPCLOUD", version = "002", api = "cmr", formats = {".tiff"}, collections = {}},
GEDI_L3_LandSurface_Metrics_V2_1952 = {provider = "ORNL_CLOUD", version = nil, api = "cmr", formats = {".h5"}, collections = {}},
GEDI_L4A_AGB_Density_V2_1_2056 = {provider = "ORNL_CLOUD", version = nil, api = "cmr", formats = {".h5"}, collections = {}},
GEDI_L4B_Gridded_Biomass_2017 = {provider = "ORNL_CLOUD", version = nil, api = "cmr", formats = {".tiff"}, collections = {}},
Expand All @@ -44,7 +40,7 @@ ASSETS_TO_DATASETS = {
["gedil3-canopy-stddev"] = "GEDI_L3_LandSurface_Metrics_V2_1952",
["gedil3-counts"] = "GEDI_L3_LandSurface_Metrics_V2_1952",
["gedil2a"] = "GEDI02_A",
["gedil1b"] = "GEDI02_B",
["gedil1b"] = "GEDI01_B",
["swot-sim-ecco-llc4320"] = "SWOT_SIMULATED_L2_KARIN_SSH_ECCO_LLC4320_CALVAL_V1",
["swot-sim-glorys"] = "SWOT_SIMULATED_L2_KARIN_SSH_GLORYS_CALVAL_V1",
["usgs3dep-1meter-dem"] = "Digital Elevation Model (DEM) 1 meter",
Expand Down Expand Up @@ -247,21 +243,26 @@ local function cmr (parms, poly, with_meta)
for _,l in ipairs(links) do
local link = l["href"]
if link and (total_links < max_resources) then
-- grab only links with the desired format (checks extension match)
for _,format in ipairs(dataset["formats"]) do
if link:sub(-#format) == format then
-- split link over '/' and grab last element
local url = link
for token in link:gmatch("[^" .. "/" .. "]+") do
url = token
end
-- add url to table of links with metadata
if not link_table[url] then
link_table[url] = metadata
num_links = num_links + 1
total_links = total_links + 1
-- exclude opendap links - this is a hack, but is the only way
-- to currently remove these duplicate responses (and is the way
-- CMR code currently does it as well)
if not string.find(link, "opendap") then
-- grab only links with the desired format (checks extension match)
for _,format in ipairs(dataset["formats"]) do
if link:sub(-#format) == format then
-- split link over '/' and grab last element
local url = link
for token in link:gmatch("[^" .. "/" .. "]+") do
url = token
end
-- add url to table of links with metadata
if not link_table[url] then
link_table[url] = metadata
num_links = num_links + 1
total_links = total_links + 1
end
break
end
break
end
end
end
Expand Down

0 comments on commit 28b7671

Please sign in to comment.