issue 104

src/nsidc/metgen/cli.py

@@ -21,6 +21,8 @@ def init(config):
     """Populates a configuration file based on user input."""
     click.echo(metgen.banner())
     config = metgen.init_config(config)
+    # add step here to evaluate input files?
+    # if netcdf: header information
     click.echo(f"Initialized the metgen configuration file {config}")
 
 
@@ -67,6 +69,17 @@ def validate(config_filename, content_type):
     metgen.init_logging(configuration)
     metgen.validate(configuration, content_type)
 
+@cli.command()
+@click.option(
+    "-f",
+    "--file",
+    "data_filename",
+    help="Path to sample data file",
+    required=True,
+)
+def assess():
+    """Examine a sample data file for metadata completeness"""
+    return True
 
 @cli.command()
 @click.option(

src/nsidc/metgen/config.py

@@ -31,6 +31,9 @@ class Config:
     number: int
     dry_run: bool
 
+    def __post_init__(self):
+        self.collection = None
+
     def show(self):
         # TODO: add section headings in the right spot
         #       (if we think we need them in the output)
@@ -43,8 +46,8 @@ def show(self):
         if self.dry_run:
             LOGGER.info("")
             LOGGER.info(
-                """Note: The dry-run option was included, so no files will be\
-                staged and no CNM messages published."""
+                """Note: The dry-run option was included, so no files will be \
+staged and no CNM messages published."""
             )
             LOGGER.info("")
 

src/nsidc/metgen/metgen.py

@@ -7,10 +7,10 @@
 import os.path
 import sys
 import uuid
+from collections.abc import Callable
 from importlib.resources import open_text
 from pathlib import Path
 from string import Template
-from typing import Callable
 
 import jsonschema
 from funcy import all, filter, partial, rcompose, take
@@ -175,31 +175,6 @@ def init_config(configuration_file):
     return configuration_file
 
 
-def prepare_output_dirs(configuration):
-    """
-    Generate paths to ummg and cnm output directories.
-    Remove any existing UMM-G files if needed.
-    TODO: create local_output_dir, ummg_dir, and cnm subdir if they don't exist
-    """
-    ummg_path = Path(configuration.local_output_dir, configuration.ummg_dir)
-    cnm_path = Path(configuration.local_output_dir, "cnm")
-
-    if configuration.overwrite_ummg:
-        scrub_json_files(ummg_path)
-
-    return (ummg_path, cnm_path)
-
-
-def scrub_json_files(path):
-    print(f"Removing existing files in {path}")
-    for file_path in path.glob("*.json"):
-        try:
-            if os.path.isfile(file_path) or os.path.islink(file_path):
-                os.unlink(file_path)
-        except Exception as e:
-            print("Failed to delete %s: %s" % (file_path, e))
-
-
 # -------------------------------------------------------------------
 # Data structures for processing Granules and recording results
 # -------------------------------------------------------------------
@@ -211,6 +186,7 @@ class Collection:
 
     auth_id: str
     version: int
+    data_reader: Callable[[str], dict]
 
 
 @dataclasses.dataclass
@@ -255,7 +231,15 @@ def process(configuration: config.Config) -> None:
     """
     Process all Granules and record the results and summary.
     """
-    # TODO: Do any prep actions, like mkdir, etc
+    # TODO:
+    #   Do any prep actions, like mkdir, etc
+    #   Get real collection information from CMR
+    #   Determine data reader based on actual data file characteristics (e.g. suffix)
+    configuration.collection = Collection(
+            configuration.auth_id,
+            configuration.version,
+            netcdf_reader.extract_metadata
+    )
 
     # Ordered list of operations to perform on each granule
     operations = [
@@ -361,13 +345,10 @@ def null_operation(configuration: config.Config, granule: Granule) -> Granule:
 
 def granule_collection(configuration: config.Config, granule: Granule) -> Granule:
     """
-    Find the Granule's Collection and add it to the Granule.
+    Associate the Collection with the Granule.
     """
-    # TODO: When we start querying CMR, refactor the pipeline to retrieve
-    # collection information from CMR once, then associate it with each
-    # granule.
     return dataclasses.replace(
-        granule, collection=Collection(configuration.auth_id, configuration.version)
+        granule, collection=configuration.collection
     )
 
 
@@ -418,7 +399,7 @@ def create_ummg(configuration: config.Config, granule: Granule) -> Granule:
     # }
     metadata_details = {}
     for data_file in granule.data_filenames:
-        metadata_details[data_file] = netcdf_reader.extract_metadata(data_file)
+        metadata_details[data_file] = granule.collection.data_reader(data_file)
 
     # Collapse information about (possibly) multiple files into a granule summary.
     summary = metadata_summary(metadata_details)

src/nsidc/metgen/netcdf_reader.py

@@ -10,6 +10,10 @@
 
 
 def extract_metadata(netcdf_path):
+    # provide some sort of "review" command line function to
+    # assess what's missing from netcdf file?
+    # or add to ini file generator a step that evaluates an
+    # example file for completeness?
     """
     Read the content at netcdf_path and return a structure with temporal coverage
     information, spatial coverage information, file size, and production datetime.
@@ -20,7 +24,9 @@ def extract_metadata(netcdf_path):
 
     return {
         "size_in_bytes": os.path.getsize(netcdf_path),
+        # no date modified in file
         "production_date_time": ensure_iso(netcdf.attrs["date_modified"]),
+        # no time range in file
         "temporal": time_range(netcdf),
         "geometry": {"points": json.dumps(spatial_values(netcdf))},
     }
@@ -51,9 +57,7 @@ def spatial_values(netcdf):
     crs_4326 = CRS.from_epsg(4326)
     xformer = Transformer.from_crs(data_crs, crs_4326, always_xy=True)
 
-    # Adding padding should give us values that match up to the
-    # netcdf.attrs.geospatial_bounds
-    pad = abs(float(netcdf.crs.GeoTransform.split()[1])) / 2
+    pad = pixel_padding(netcdf)
     xdata = [x - pad if x < 0 else x + pad for x in netcdf.x.data]
     ydata = [y - pad if y < 0 else y + pad for y in netcdf.y.data]
 
@@ -65,6 +69,10 @@ def spatial_values(netcdf):
         for (lon, lat) in perimeter
     ]
 
+def pixel_padding(netcdf):
+    # Adding padding should give us values that match up to the
+    # netcdf.attrs.geospatial_bounds
+    return abs(float(netcdf.crs.GeoTransform.split()[1])) / 2
 
 def thinned_perimeter(xdata, ydata):
     """

tests/test_config.py

@@ -32,6 +32,7 @@ def expected_keys():
             "checksum_type",
             "number",
             "dry_run",
+            "collection"
         ]
     )
 

tests/test_metgen.py

@@ -102,14 +102,22 @@ def test_returns_datetime_range():
 
 
 def test_s3_object_path_has_no_leading_slash():
-    granule = metgen.Granule("foo", metgen.Collection("ABCD", 2), uuid="abcd-1234")
+    granule = metgen.Granule(
+        "foo",
+        metgen.Collection("ABCD", 2, "my_reader"),
+        uuid="abcd-1234"
+    )
     expected = "external/ABCD/2/abcd-1234/xyzzy.bin"
     assert metgen.s3_object_path(granule, "xyzzy.bin") == expected
 
 
 def test_s3_url_simple_case():
     staging_bucket_name = "xyzzy-bucket"
-    granule = metgen.Granule("foo", metgen.Collection("ABCD", 2), uuid="abcd-1234")
+    granule = metgen.Granule(
+        "foo",
+        metgen.Collection("ABCD", 2, "my_reader"),
+        uuid="abcd-1234"
+    )
     expected = "s3://xyzzy-bucket/external/ABCD/2/abcd-1234/xyzzy.bin"
     assert metgen.s3_url(staging_bucket_name, granule, "xyzzy.bin") == expected