diff --git a/clients/python/sliderule/earthdata.py b/clients/python/sliderule/earthdata.py index 0ee6d209..5074f336 100644 --- a/clients/python/sliderule/earthdata.py +++ b/clients/python/sliderule/earthdata.py @@ -61,9 +61,9 @@ "ATL06": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []}, "ATL08": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []}, "ATL09": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": [".h5"], "collections": []}, + "ATL13": {"provider": "NSIDC_CPRD", "version": "006", "api": "cmr", "formats": {".h5"}, "collections": {}}, "GEDI01_B": {"provider": "LPCLOUD", "version": "002", "api": "cmr", "formats": [".h5"], "collections": []}, "GEDI02_A": {"provider": "LPCLOUD", "version": "002", "api": "cmr", "formats": [".h5"], "collections": []}, - "GEDI02_B": {"provider": "LPCLOUD", "version": "002", "api": "cmr", "formats": [".tiff"], "collections": []}, "GEDI_L3_LandSurface_Metrics_V2_1952": {"provider": "ORNL_CLOUD", "version": None, "api": "cmr", "formats": [".h5"], "collections": []}, "GEDI_L4A_AGB_Density_V2_1_2056": {"provider": "ORNL_CLOUD", "version": None, "api": "cmr", "formats": [".h5"], "collections": []}, "GEDI_L4B_Gridded_Biomass_2017": {"provider": "ORNL_CLOUD", "version": None, "api": "cmr", "formats": [".tiff"], "collections": []}, @@ -83,7 +83,7 @@ "gedil3-canopy-stddev": "GEDI_L3_LandSurface_Metrics_V2_1952", "gedil3-counts": "GEDI_L3_LandSurface_Metrics_V2_1952", "gedil2a": "GEDI02_A", - "gedil1b": "GEDI02_B", + "gedil1b": "GEDI01_B", "swot-sim-ecco-llc4320": "SWOT_SIMULATED_L2_KARIN_SSH_ECCO_LLC4320_CALVAL_V1", "swot-sim-glorys": "SWOT_SIMULATED_L2_KARIN_SSH_GLORYS_CALVAL_V1", "usgs3dep-1meter-dem": "Digital Elevation Model (DEM) 1 meter", @@ -92,6 +92,7 @@ "icesat2-atl06": "ATL06", "icesat2-atl08": "ATL08", "icesat2-atl09": "ATL09", + "icesat2-atl13": "ATL13", "atlas-local": "ATL03", "nsidc-s3": "ATL03" } diff --git a/clients/python/tests/test_earthdata.py b/clients/python/tests/test_earthdata.py index c4c8f673..e40a4efe 100644 --- a/clients/python/tests/test_earthdata.py +++ b/clients/python/tests/test_earthdata.py @@ -2,10 +2,11 @@ import pytest import sliderule -from sliderule import icesat2, earthdata +from sliderule import earthdata from sliderule.earthdata import __cmr_collection_query as cmr_collection_query from sliderule.earthdata import __cmr_max_version as cmr_max_version from pathlib import Path +from datetime import datetime import os import json @@ -85,4 +86,126 @@ def test_asstr(self): def test_bad_short_name(self): region = sliderule.toregion(os.path.join(TESTDIR, 'data/polygon.geojson')) geojson = earthdata.tnm(short_name='DOES_NOT_EXIST', polygon=region["poly"], as_str=False) - assert len(geojson['features']) == 0 \ No newline at end of file + assert len(geojson['features']) == 0 + +# +# SlideRule Endpoint +# +@pytest.mark.network +class TestSlideRule: + def test_atl03(self, init): + parms = {"asset": "icesat2", "poly": grandmesa, "t0": '2018-10-01', "t1": '2019-12-01'} + resources = earthdata.search(parms) + rsps = sliderule.source("earthdata", parms) + assert init + assert len(rsps) == 26 + assert len(rsps) == len(set(rsps)) + assert len(rsps) == len(resources) + for resource in resources: + assert resource in rsps + assert datetime.strptime(resource[6:14], '%Y%m%d') >= datetime.strptime(parms["t0"], '%Y-%m-%d') + assert datetime.strptime(resource[6:14], '%Y%m%d') <= datetime.strptime(parms["t1"], '%Y-%m-%d') + + def test_atl06(self, init): + parms = {"asset": "icesat2-atl06", "poly": grandmesa, "t0": '2018-10-01', "t1": '2019-12-01'} + resources = earthdata.search(parms) + rsps = sliderule.source("earthdata", parms) + assert init + assert len(rsps) == 26 + assert len(rsps) == len(set(rsps)) + assert len(rsps) == len(resources) + for resource in resources: + assert resource in rsps + assert datetime.strptime(resource[6:14], '%Y%m%d') >= datetime.strptime(parms["t0"], '%Y-%m-%d') + assert datetime.strptime(resource[6:14], '%Y%m%d') <= datetime.strptime(parms["t1"], '%Y-%m-%d') + + def test_atl09(self, init): + parms = {"asset": "icesat2-atl09", "poly": grandmesa, "t0": '2018-10-01', "t1": '2019-12-01'} + resources = earthdata.search(parms) + rsps = sliderule.source("earthdata", parms) + assert init + assert len(rsps) == 36 + assert len(rsps) == len(set(rsps)) + assert len(rsps) == len(resources) + for resource in resources: + assert resource in rsps + assert datetime.strptime(resource[6:14], '%Y%m%d') >= datetime.strptime(parms["t0"], '%Y-%m-%d') + assert datetime.strptime(resource[6:14], '%Y%m%d') <= datetime.strptime(parms["t1"], '%Y-%m-%d') + + def test_atl13(self, init): + saltlake = [ + { + "lon": -111.78656302303546, + "lat": 40.474445992545355 + }, + { + "lon": -111.78656302303546, + "lat": 41.745511885629725 + }, + { + "lon": -113.25842634761666, + "lat": 41.745511885629725 + }, + { + "lon": -113.25842634761666, + "lat": 40.474445992545355 + }, + { + "lon": -111.78656302303546, + "lat": 40.474445992545355 + } + ] + parms = {"asset": "icesat2-atl13", "poly": saltlake, "t0": '2018-10-01', "t1": '2019-12-01'} + resources = earthdata.search(parms) + rsps = sliderule.source("earthdata", parms) + assert init + print(resources) + assert len(rsps) == 67 + assert len(rsps) == len(set(rsps)) + assert len(rsps) == len(resources) + for resource in resources: + assert resource in rsps + assert datetime.strptime(resource[6:14], '%Y%m%d') >= datetime.strptime(parms["t0"], '%Y-%m-%d') + assert datetime.strptime(resource[6:14], '%Y%m%d') <= datetime.strptime(parms["t1"], '%Y-%m-%d') + + def test_gedil1b(self, init): + parms = {"asset": "gedil1b", "poly": grandmesa, "t0": '2018-10-01', "t1": '2019-12-01'} + resources = earthdata.search(parms) + rsps = sliderule.source("earthdata", parms) + print(resources) + assert init + assert len(rsps) == 26 + assert len(rsps) == len(set(rsps)) + assert len(rsps) == len(resources) + for resource in resources: + assert resource in rsps + assert datetime.strptime(resource[9:16], '%Y%j') >= datetime.strptime(parms["t0"], '%Y-%m-%d') + assert datetime.strptime(resource[9:16], '%Y%j') <= datetime.strptime(parms["t1"], '%Y-%m-%d') + + def test_gedil2a(self, init): + parms = {"asset": "gedil2a", "poly": grandmesa, "t0": '2018-10-01', "t1": '2019-12-01'} + resources = earthdata.search(parms) + rsps = sliderule.source("earthdata", parms) + print(resources) + assert init + assert len(rsps) == 26 + assert len(rsps) == len(set(rsps)) + assert len(rsps) == len(resources) + for resource in resources: + assert resource in rsps + assert datetime.strptime(resource[9:16], '%Y%j') >= datetime.strptime(parms["t0"], '%Y-%m-%d') + assert datetime.strptime(resource[9:16], '%Y%j') <= datetime.strptime(parms["t1"], '%Y-%m-%d') + + def test_gedil4a(self, init): + parms = {"asset": "gedil4a", "poly": grandmesa, "t0": '2018-10-01', "t1": '2019-12-01'} + resources = earthdata.search(parms) + rsps = sliderule.source("earthdata", parms) + print(resources) + assert init + assert len(rsps) == 26 + assert len(rsps) == len(set(rsps)) + assert len(rsps) == len(resources) + for resource in resources: + assert resource in rsps + assert datetime.strptime(resource[9:16], '%Y%j') >= datetime.strptime(parms["t0"], '%Y-%m-%d') + assert datetime.strptime(resource[9:16], '%Y%j') <= datetime.strptime(parms["t1"], '%Y-%m-%d') diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index a5aee643..0ee1e655 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -3,6 +3,7 @@ install ( FILES ${CMAKE_CURRENT_LIST_DIR}/endpoints/definition.lua + ${CMAKE_CURRENT_LIST_DIR}/endpoints/earthdata.lua ${CMAKE_CURRENT_LIST_DIR}/endpoints/event.lua ${CMAKE_CURRENT_LIST_DIR}/selftests/example_engine_endpoint.lua ${CMAKE_CURRENT_LIST_DIR}/selftests/example_source_endpoint.lua diff --git a/scripts/endpoints/earthdata.lua b/scripts/endpoints/earthdata.lua new file mode 100644 index 00000000..448a9f11 --- /dev/null +++ b/scripts/endpoints/earthdata.lua @@ -0,0 +1,17 @@ +-- +-- ENDPOINT: /source/earthdata +-- +-- INPUT: none +-- +-- OUTPUT: json string of default parameters +-- +local json = require("json") +local earthdata = require("earth_data_query") +local parm = json.decode(arg[1]) +local status,rsps = earthdata.search(parm) +if status == earthdata.SUCCESS then + return json.encode(rsps) +else + return json.encode({error=rsps}) +end + diff --git a/scripts/extensions/earth_data_query.lua b/scripts/extensions/earth_data_query.lua index 51f1d119..75ec9ff8 100644 --- a/scripts/extensions/earth_data_query.lua +++ b/scripts/extensions/earth_data_query.lua @@ -5,9 +5,6 @@ local json = require("json") - -local pp = require("prettyprint") - -- -- Constants -- @@ -24,7 +21,6 @@ DATASETS = { ATL13 = {provider = "NSIDC_CPRD", version = "006", api = "cmr", formats = {".h5"}, collections = {}}, GEDI01_B = {provider = "LPCLOUD", version = "002", api = "cmr", formats = {".h5"}, collections = {}}, GEDI02_A = {provider = "LPCLOUD", version = "002", api = "cmr", formats = {".h5"}, collections = {}}, - GEDI02_B = {provider = "LPCLOUD", version = "002", api = "cmr", formats = {".tiff"}, collections = {}}, GEDI_L3_LandSurface_Metrics_V2_1952 = {provider = "ORNL_CLOUD", version = nil, api = "cmr", formats = {".h5"}, collections = {}}, GEDI_L4A_AGB_Density_V2_1_2056 = {provider = "ORNL_CLOUD", version = nil, api = "cmr", formats = {".h5"}, collections = {}}, GEDI_L4B_Gridded_Biomass_2017 = {provider = "ORNL_CLOUD", version = nil, api = "cmr", formats = {".tiff"}, collections = {}}, @@ -44,7 +40,7 @@ ASSETS_TO_DATASETS = { ["gedil3-canopy-stddev"] = "GEDI_L3_LandSurface_Metrics_V2_1952", ["gedil3-counts"] = "GEDI_L3_LandSurface_Metrics_V2_1952", ["gedil2a"] = "GEDI02_A", - ["gedil1b"] = "GEDI02_B", + ["gedil1b"] = "GEDI01_B", ["swot-sim-ecco-llc4320"] = "SWOT_SIMULATED_L2_KARIN_SSH_ECCO_LLC4320_CALVAL_V1", ["swot-sim-glorys"] = "SWOT_SIMULATED_L2_KARIN_SSH_GLORYS_CALVAL_V1", ["usgs3dep-1meter-dem"] = "Digital Elevation Model (DEM) 1 meter", @@ -247,21 +243,26 @@ local function cmr (parms, poly, with_meta) for _,l in ipairs(links) do local link = l["href"] if link and (total_links < max_resources) then - -- grab only links with the desired format (checks extension match) - for _,format in ipairs(dataset["formats"]) do - if link:sub(-#format) == format then - -- split link over '/' and grab last element - local url = link - for token in link:gmatch("[^" .. "/" .. "]+") do - url = token - end - -- add url to table of links with metadata - if not link_table[url] then - link_table[url] = metadata - num_links = num_links + 1 - total_links = total_links + 1 + -- exclude opendap links - this is a hack, but is the only way + -- to currently remove these duplicate responses (and is the way + -- CMR code currently does it as well) + if not string.find(link, "opendap") then + -- grab only links with the desired format (checks extension match) + for _,format in ipairs(dataset["formats"]) do + if link:sub(-#format) == format then + -- split link over '/' and grab last element + local url = link + for token in link:gmatch("[^" .. "/" .. "]+") do + url = token + end + -- add url to table of links with metadata + if not link_table[url] then + link_table[url] = metadata + num_links = num_links + 1 + total_links = total_links + 1 + end + break end - break end end end