fixed time bug, updated arrowsampler point time

SlideRuleEarth · Nov 20, 2024 · d22e4b9 · d22e4b9
1 parent 9500edf
commit d22e4b9
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 16 deletions.
diff --git a/clients/python/sliderule/raster.py b/clients/python/sliderule/raster.py
@@ -142,7 +142,7 @@ def sample(asset, coordinates, parms={}):
         latitude = coordinates[per_coord_index][1]
         per_coord_index += 1
         for raster_sample in input_coord_response:
-            columns['time'][i] = numpy.int64((raster_sample['time'] + 315564800.0) * 1e9)
+            columns['time'][i] = numpy.int64((raster_sample['time'] + 315964800.0) * 1e9)
             columns['longitude'][i] = numpy.double(longitude)
             columns['latitude'][i] = numpy.double(latitude)
             columns['file'] += raster_sample['file'],

diff --git a/clients/python/tests/test_worldcover.py b/clients/python/tests/test_worldcover.py
@@ -15,7 +15,8 @@
 
 expValue     = 10
 vrtFile      = '/vsis3/sliderule/data/WORLDCOVER/ESA_WorldCover_10m_2021_v200_Map.vrt'
-vrtFileTime  = 1309046418  # gpsTime in seconds 2021-06-30
+timeAsDateStr  = '2021-06-30 00:00:18'
+timeAsUnixSecs = 1309046418  # includes 18 leap seconds
 
 @pytest.mark.network
 class TestMosaic:
@@ -25,7 +26,7 @@ def test_vrt(self, init):
         assert init
         assert abs(rsps["samples"][0][0]["value"] - expValue) < sigma
         assert rsps["samples"][0][0]["file"] ==  vrtFile
-        assert rsps["samples"][0][0]["time"] ==  vrtFileTime  # datetime in seconds
+        assert rsps["samples"][0][0]["time"] ==  timeAsUnixSecs  # datetime in seconds
 
     def test_time_overflow(self, init):
         region = [  {"lon": -108.34, "lat": 38.89},
@@ -41,16 +42,12 @@ def test_time_overflow(self, init):
         # Reset the index to make 'time' a regular column
         gdf_reset = gdf.reset_index()
 
-        # Access raw time column values as NumPy array
-        raw_time_values = gdf_reset['time'].astype('int64')
+        # Access the 'time' column
+        time_values = gdf_reset['time'].tolist()
+        print(time_values)
 
-        # Expected gps epoch time in nanoseconds
-        expected_time = (vrtFileTime + 315564800) * 1e9
-
-        print("Raw time column values (NumPy array):", raw_time_values)
-        print("expected_time:", expected_time)
-
-        assert (raw_time_values == expected_time).all()
+        assert [str(t) for t in time_values] == [timeAsDateStr, timeAsDateStr], \
+            f"Time values do not match. Expected: {[timeAsDateStr, timeAsDateStr]}, Found: {[str(t) for t in time_values]}"
 
         values = gdf_reset['value']
         assert (values == expValue).all()
diff --git a/packages/arrow/ArrowSamplerImpl.cpp b/packages/arrow/ArrowSamplerImpl.cpp
@@ -281,13 +281,23 @@ void ArrowSamplerImpl::getPoints(std::vector<point_info_t>& points)
     if(time_column_index > -1)
     {
         auto time_column = std::static_pointer_cast<arrow::TimestampArray>(table->column(time_column_index)->chunk(0));
-        mlog(DEBUG, "Time column elements: %ld", time_column->length());
+        auto timestamp_type = std::static_pointer_cast<arrow::TimestampType>(table->column(time_column_index)->type());
 
-        /* Update gps time for each point */
+        if(timestamp_type->unit() != arrow::TimeUnit::NANO)
+        {
+            mlog(ERROR, "Time column must be in nanoseconds.");
+            points.clear();
+            return;
+        }
+
+        /* Convert unix nanoseconds to gps seconds */
         for(int64_t i = 0; i < time_column->length(); i++)
-            points[i].gps = time_column->Value(i);
+        {
+            double unix_nsecs = time_column->Value(i);
+            points[i].gps = TimeLib::sys2gpstime(unix_nsecs / 1000.0) * 1000.0;
+        }
     }
-    else mlog(DEBUG, "Time column not found.");
+    else mlog(ERROR, "Time column not found.");
 }
 
 /*----------------------------------------------------------------------------