Merge pull request #738 from nsidc/fix-integration-test-silent-failure

Enable integration test workflow to better communicate failure

earthaccess/api.py

@@ -68,7 +68,9 @@ def search_datasets(count: int = -1, **kwargs: Any) -> List[DataCollection]:
         ```
     """
     if not validate.valid_dataset_parameters(**kwargs):
-        logger.warn("A valid set of parameters is needed to search for datasets on CMR")
+        logger.warning(
+            "A valid set of parameters is needed to search for datasets on CMR"
+        )
         return []
     if earthaccess.__auth__.authenticated:
         query = DataCollections(auth=earthaccess.__auth__).parameters(**kwargs)

earthaccess/store.py

@@ -113,7 +113,7 @@ def __init__(self, auth: Any, pre_authorize: bool = False) -> None:
                     self.set_requests_session(url)
 
         else:
-            logger.warn("The current session is not authenticated with NASA")
+            logger.warning("The current session is not authenticated with NASA")
             self.auth = None
         self.in_region = self._running_in_us_west_2()
 

pyproject.toml

@@ -144,8 +144,8 @@ line-length = 88
 src = ["earthaccess", "stubs", "tests"]
 
 [tool.ruff.lint]
-extend-select = ["I", "T20", "D"]
-ignore = ["D1", "D205", "D401", "D417"]
+extend-select = ["I", "T20", "D", "G"]
+ignore = ["D1", "D205", "D401", "D417", "G004"]
 
 [tool.ruff.lint.pydocstyle]
 convention = "google"

scripts/integration-test.sh

@@ -1,7 +1,18 @@
 #!/usr/bin/env bash
 
-set -e
 set -x
+pytest tests/integration --cov=earthaccess --cov=tests/integration --cov-report=term-missing ${@} --capture=no --tb=native --log-cli-level=INFO
+RET=$?
+set +x
+
+set -e
+# NOTE: 99 is a special return code we selected for this case, it has no other meaning.
+if [[ $RET == 99 ]]; then
+    echo -e "\e[0;31mWARNING: The integration test suite has been permitted to return 0 because the failure rate was less than a hardcoded threshold.\e[0m"
+    echo "For more details, see conftest.py."
+    exit 0
+elif [[ $RET != 0 ]]; then
+    exit $RET
+fi
 
-pytest --cov=earthaccess --cov=tests/integration --cov-report=term-missing ${@} -s --tb=native --log-cli-level=INFO
 bash ./scripts/lint.sh

tests/integration/conftest.py

@@ -5,8 +5,27 @@
 
 @pytest.hookimpl()
 def pytest_sessionfinish(session, exitstatus):
-    if exitstatus == 0:
+    """Return exit code 99 if up to N% of tests have failed.
+
+        N = ACCEPTABLE_FAILURE_RATE
+
+    99 was chosen arbitrarily to avoid conflict with current and future pytest error
+    codes (https://docs.pytest.org/en/stable/reference/exit-codes.html), and avoid
+    other exit codes with special meanings
+    (https://tldp.org/LDP/abs/html/exitcodes.html).
+
+    IMPORTANT: This is calculated against every test collected in the session, so the
+    ratio will change depending on which tests are executed! E.g. executing integration
+    tests and unit tests at the same time allows more tests to fail than executing
+    integration tests alone.
+    """
+    if exitstatus != pytest.ExitCode.TESTS_FAILED:
+        # Exit status 1 in PyTest indicates "Tests were collected and run but some of
+        # the tests failed". In all other cases, for example "an internal error happened
+        # while executing the tests", or "test execution interrupted by the user", we
+        # want to defer to original pytest behavior.
         return
+
     failure_rate = (100.0 * session.testsfailed) / session.testscollected
     if failure_rate <= ACCEPTABLE_FAILURE_RATE:
-        session.exitstatus = 0
+        session.exitstatus = 99