Fix bug in handling starttime and endtime (#486)

* Fix bug in handling starttime and endtime

Previous version was failing to set starttime and endtime on saves.  Also data loaded and cut had incorrect starttime and endtimes saved because they werne't being replaced.

* fix typo

* add endtime to schema

* fix placing new code in the wrong block

* 🎨 Format Python code with psf/black

* replace starttime by calib in testing exclude_keys

* now supports exclude_keys for starttime and endtime

---------

Co-authored-by: Gary Pavlis <[email protected]>
Co-authored-by: wangyinz <[email protected]>

data/yaml/mspass.yaml

@@ -66,6 +66,10 @@ Database:
           concept:  Time shift applied to define relative time of 0.0
           aliases:  t0shift
           constraint: required
+        endtime:
+          type: double
+          concept: Time of last sample of data (epoch time or relative to some other time mark)
+          constraint: optional 
         utc_convertible:
           type:  bool
           concept:  When true starttime_shift can be used to convert relative time to UTC.

data/yaml/mspass_fdsn.yaml

@@ -62,6 +62,10 @@ Database:
           concept:  Time shift applied to define relative time of 0.0
           aliases:  t0shift
           constraint: required
+        endtime:
+          type: double
+          concept: Time of last sample of data (epoch time or relative to some other time mark)
+          constraint: optional
         utc_convertible:
           type:  bool
           concept:  When true starttime_shift can be used to convert relative time to UTC.

data/yaml/mspass_lite.yaml

@@ -37,6 +37,10 @@ Database:
           concept:  Time shift applied to define relative time of 0.0
           aliases:  t0shift
           constraint: required
+        endtime:
+          type: double
+          concept: Time of last sample of data (epoch time or relative to some other time mark)
+          constraint: optional
         utc_convertible:
           type:  bool
           concept:  When true starttime_shift can be used to convert relative time to UTC.
@@ -283,4 +287,5 @@ Metadata:
         readonly: false
       loc:
         collection: wf_Seismogram
-        readonly: false 
+        readonly: false
+

data/yaml/mspass_s3.yaml

@@ -62,6 +62,10 @@ Database:
           concept:  Time shift applied to define relative time of 0.0
           aliases:  t0shift
           constraint: required
+        endtime:
+          type: double
+          concept: Time of last sample of data (epoch time or relative to some other time mark)
+          constraint: optional
         utc_convertible:
           type:  bool
           concept:  When true starttime_shift can be used to convert relative time to UTC.

python/mspasspy/db/database.py

@@ -1,4 +1,3 @@
-
 import os
 import io
 import copy
@@ -4960,13 +4959,13 @@ def save_inventory(self, inv, networks_to_exclude=["SY"], verbose=False):
                     loc_etime = self._handle_null_endtime(loc_etime)
                     rec["lat"] = loc_lat
                     rec["lon"] = loc_lon
-                    # save coordinates in both geoJSON and "legacy" 
-                    # format 
-                    rec["coords"] = [loc_lon,loc_lat]
-                    # Illegal lon,lat values will cause this to throw a 
-                    # ValueError exception.   We let it do that as 
+                    # save coordinates in both geoJSON and "legacy"
+                    # format
+                    rec["coords"] = [loc_lon, loc_lat]
+                    # Illegal lon,lat values will cause this to throw a
+                    # ValueError exception.   We let it do that as
                     # it indicates a problem datum
-                    rec = geoJSON_doc(loc_lat,loc_lon,doc=rec,key='location')
+                    rec = geoJSON_doc(loc_lat, loc_lon, doc=rec, key="location")
                     rec["elev"] = loc_elev
                     rec["edepth"] = loc_edepth
                     rec["starttime"] = starttime.timestamp
@@ -5493,12 +5492,12 @@ def save_catalog(self, cat, verbose=False):
             rec["lat"] = o.latitude
             rec["lon"] = o.longitude
             # save the epicenter data in both legacy format an d
-            # geoJSON format.  Either can technically be used in a 
-            # geospatial query but the geoJSON version is always 
+            # geoJSON format.  Either can technically be used in a
+            # geospatial query but the geoJSON version is always
             # preferred
             rec["coords"] = [o.longitude, o.latitude]
             # note this function updates rec and returns the upodate
-            rec = geoJSON_doc(o.latitude, o.longitude, doc=rec,key='epicenter')
+            rec = geoJSON_doc(o.latitude, o.longitude, doc=rec, key="epicenter")
             # It appears quakeml puts source depths in meter
             # convert to km
             # also obspy's catalog object seesm to allow depth to be
@@ -6505,6 +6504,12 @@ def _atomic_save_all_documents(
             # Warning:  this can lead to sample data save orphans
             mspass_object.kill()
 
+        # Always set starttime and endtime
+        if not exclude_keys or "starttime" not in exclude_keys:
+            insertion_dict["starttime"] = mspass_object.t0
+        if not exclude_keys or "endtime" not in exclude_keys:
+            insertion_dict["endtime"] = mspass_object.endtime()
+
         # add tag - intentionally not set in mspass_object returned
         if data_tag:
             insertion_dict["data_tag"] = data_tag
@@ -8186,54 +8191,57 @@ def _erase_normalized(
                     mdout.erase(k)
     return mdout
 
-def geoJSON_doc(lat,lon,doc=None,key='epicenter')->dict:
+
+def geoJSON_doc(lat, lon, doc=None, key="epicenter") -> dict:
     """
     Convenience function to create a geoJSON format point object document
-    from a points specified by latitude and longitude. The format 
-    for a geoJSON point isn't that strange but how to structure it into 
+    from a points specified by latitude and longitude. The format
+    for a geoJSON point isn't that strange but how to structure it into
     a mongoDB document for use with geospatial queries is not as
-    clear from current MongoDB documentation.  This function makes that 
+    clear from current MongoDB documentation.  This function makes that
     proess easier.
-    
-    The required inpput is latitude (lat) and longitude (lon).  The 
+
+    The required inpput is latitude (lat) and longitude (lon).  The
     values are assumed to be in degrees for compatibility with MongoDB.
-    That means latitude must be -90<=lat<=90 and longitude 
-    must satisfy -180<=lat<=180.  The function will try to handle the 
-    common situation with 0<=lon<=360 by wrapping 90->180 values to 
-    -180->0,  A ValueError exception is thrown if  
+    That means latitude must be -90<=lat<=90 and longitude
+    must satisfy -180<=lat<=180.  The function will try to handle the
+    common situation with 0<=lon<=360 by wrapping 90->180 values to
+    -180->0,  A ValueError exception is thrown if
     for any other situation with lot or lon outside those bounds.
-    
-    If you specify the optional "doc" argument it is assumed to be 
-    a python dict to which the geoJSON point data is to be added.  
-    By default a new dict is created that will contain only the 
-    geoJSON point data.  The doc options is useful if you want to 
-    add the geoJSON data to the document before appending it.  
-    The default is more useful for updates to add geospatial 
-    query capabilities to a collection with lat-lon data that is 
+
+    If you specify the optional "doc" argument it is assumed to be
+    a python dict to which the geoJSON point data is to be added.
+    By default a new dict is created that will contain only the
+    geoJSON point data.  The doc options is useful if you want to
+    add the geoJSON data to the document before appending it.
+    The default is more useful for updates to add geospatial
+    query capabilities to a collection with lat-lon data that is
     not properl structure.  In all cases the geoJSON data is a
-    itself a python dict but a value associated accessible from 
+    itself a python dict but a value associated accessible from
     the output dict with te key defined by the "key" argument
-    (default is 'epicenter', which is appropriate for earthquake 
+    (default is 'epicenter', which is appropriate for earthquake
     source data.)
-    with a 
+    with a
     """
     outval = dict()
-    outval['type'] = 'Point'
+    outval["type"] = "Point"
     lon = float(lon)  # make this it is a float for database consistency
-    if lon>180.0 and lon<=360.0:
+    if lon > 180.0 and lon <= 360.0:
         # we do this correction silently
         lon -= 360.0
     lat = float(lat)
     if lat < -90.0 or lat > 90.0 or lon < -180.0 or lon > 180.0:
         message = "geoJSON_doc:  Illegal geographic input\n"
         message += "latitude received={}  MongoDB requires [-90,90] range\n".format(lat)
-        message += "longitude received={}  MongoDB requires [-180,180] range".format(lon)
+        message += "longitude received={}  MongoDB requires [-180,180] range".format(
+            lon
+        )
         raise ValueError(message)
-        
-    outval['coordinates'] = [float(lon), float(lat)]
+
+    outval["coordinates"] = [float(lon), float(lat)]
     if doc:
         retdoc = doc
         retdoc[key] = outval
     else:
-        retdoc={key : outval}
+        retdoc = {key: outval}
     return retdoc

python/tests/db/test_database.py

@@ -49,7 +49,7 @@
 from mspasspy.util import logging_helper
 from bson.objectid import ObjectId
 from datetime import datetime
-from mspasspy.db.database import Database,geoJSON_doc
+from mspasspy.db.database import Database, geoJSON_doc
 from mspasspy.db.client import DBClient
 
 
@@ -2128,13 +2128,13 @@ def test_check_undefined_keys(self):
             ts,
             mode="promiscuous",
             storage_mode="gridfs",
-            exclude_keys=["extra2", "starttime"],
+            exclude_keys=["extra2", "calib"],
             return_data=True,
         )
         assert save_res.live
         self.db["wf_TimeSeries"].update_one({"_id": ts["_id"]}, {"$set": {"t0": 1.0}})
         res = self.db["wf_TimeSeries"].find_one({"_id": ts["_id"]})
-        assert "starttime" not in res
+        assert "calib" not in res
         assert "npts" not in res
         assert "t0" in res
 
@@ -3451,51 +3451,49 @@ def test_set_schema(self):
         self.db.set_schema("mspass_lite.yaml")
         with pytest.raises(KeyError, match="site"):
             self.db.database_schema._attr_dict["site"]
-            
+
     def test_geoJSON_doc(self):
         """
-        Tests only the function geoJSON_doc added Jan 2024 to properly 
-        create geoJSON records that allow geospatial queries in 
-        site, channel, and source.  That function is now used in 
-        Database.save_inventory and Database.save_catalog to always 
-        save a geoJSOn format location data.   We assume the bulk of the 
-        code is already tested when those two functions are tested in 
-        other test functions.  The big thing here is testing the 
+        Tests only the function geoJSON_doc added Jan 2024 to properly
+        create geoJSON records that allow geospatial queries in
+        site, channel, and source.  That function is now used in
+        Database.save_inventory and Database.save_catalog to always
+        save a geoJSOn format location data.   We assume the bulk of the
+        code is already tested when those two functions are tested in
+        other test functions.  The big thing here is testing the
         bad data input handlers.
         """
         # test null doc input
-        doc = geoJSON_doc(22.0,44.0,key='testpoint')
-        assert 'testpoint' in doc
-        val = doc['testpoint']
-        assert val['type'] == "Point"
-        coords = val['coordinates']
+        doc = geoJSON_doc(22.0, 44.0, key="testpoint")
+        assert "testpoint" in doc
+        val = doc["testpoint"]
+        assert val["type"] == "Point"
+        coords = val["coordinates"]
         # note coordinates pair is lon,lat
         assert coords[0] == 44.0
         assert coords[1] == 22.0
 
-        doc = geoJSON_doc(33.0,55.0,doc=doc,key='testpoint2')
+        doc = geoJSON_doc(33.0, 55.0, doc=doc, key="testpoint2")
         #  doc should contain both testpoint and testpoint2
         #  This tests update feature of this function
-        assert 'testpoint' in doc
-        assert 'testpoint2' in doc
-        val = doc['testpoint2']
-        assert val['type'] == "Point"
-        coords = val['coordinates']
+        assert "testpoint" in doc
+        assert "testpoint2" in doc
+        val = doc["testpoint2"]
+        assert val["type"] == "Point"
+        coords = val["coordinates"]
         # note coordinates pair is lon,lat
         assert coords[0] == 55.0
         assert coords[1] == 33.0
 
         # recoverable value test
-        doc = geoJSON_doc(20,270.0,key='recoverable')
-        assert 'recoverable' in doc
-        val = doc['recoverable']
-        coords = val['coordinates']
-        # use is_close because -90 is computed but normal floating point math 
+        doc = geoJSON_doc(20, 270.0, key="recoverable")
+        assert "recoverable" in doc
+        val = doc["recoverable"]
+        coords = val["coordinates"]
+        # use is_close because -90 is computed but normal floating point math
         # for that simple calculation would allow an == to also work
-        assert np.isclose(coords[0],-90.0)
+        assert np.isclose(coords[0], -90.0)
         assert coords[1] == 20.0
-
-        with pytest.raises(ValueError, 
-                           match="Illegal geographic input"):
-            doc = geoJSON_doc(20,400)
 
+        with pytest.raises(ValueError, match="Illegal geographic input"):
+            doc = geoJSON_doc(20, 400)