Move from using sqla defaults to server side defaults. Change insert/…

…upsert/upsert_list to do things manually rather than relying on sqlalchmey merge or add statements, and using a real postgres upsert for upsert. Reduce table locks by using upsert.

models/background.py

@@ -35,7 +35,7 @@ def __table_args__(cls):
     _format = sa.Column(
         sa.SMALLINT,
         nullable=False,
-        default=BackgroundFormatConverter.convert('scalar'),
+        server_default=sa.sql.elements.TextClause( str(BackgroundFormatConverter.convert('scalar')) ),
         doc='Format of the Background model. Can include scalar, map, or polynomial. '
     )
 
@@ -55,7 +55,7 @@ def format(self, value):
     _method = sa.Column(
         sa.SMALLINT,
         nullable=False,
-        default=BackgroundMethodConverter.convert('zero'),
+        server_default=sa.sql.elements.TextClause( str(BackgroundMethodConverter.convert('zero')) ),
         doc='Method used to calculate the background. '
             'Can be an algorithm like "sep", or "zero" for an image that was already background subtracted. ',
     )

models/calibratorfile.py

@@ -20,7 +20,7 @@ class CalibratorFile(Base, UUIDMixin):
         sa.SMALLINT,
         nullable=False,
         index=True,
-        default=CalibratorTypeConverter.convert( 'unknown' ),
+        server_default=sa.sql.elements.TextClause(str(CalibratorTypeConverter.convert( 'unknown' )) ) ,
         doc="Type of calibrator (Dark, Flat, Linearity, etc.)"
     )
 
@@ -40,7 +40,7 @@ def type( self, value ):
         sa.SMALLINT,
         nullable=False,
         index=True,
-        default=CalibratorTypeConverter.convert('unknown'),
+        server_default=sa.sql.elements.TextClause( str(CalibratorTypeConverter.convert('unknown')) ),
         doc="Calibrator set for instrument (unknown, externally_supplied, general, nightly)"
     )
 
@@ -144,7 +144,7 @@ class CalibratorFileDownloadLock(Base, UUIDMixin):
         sa.SMALLINT,
         nullable=False,
         index=True,
-        default=CalibratorTypeConverter.convert( 'unknown' ),
+        server_default=sa.sql.elements.TextClause( str(CalibratorTypeConverter.convert( 'unknown' )) ),
         doc="Type of calibrator (Dark, Flat, Linearity, etc.)"
     )
 
@@ -166,7 +166,7 @@ def type( self, value ):
         sa.SMALLINT,
         nullable=False,
         index=True,
-        default=CalibratorTypeConverter.convert('unknown'),
+        server_default=sa.sql.elements.TextClause( str(CalibratorTypeConverter.convert('unknown')) ),
         doc="Calibrator set for instrument (unknown, externally_supplied, general, nightly)"
     )
 

models/catalog_excerpt.py

@@ -48,7 +48,7 @@ def __table_args__( cls ):
     _format = sa.Column(
         sa.SMALLINT,
         nullable=False,
-        default=CatalogExcerptFormatConverter.convert('fitsldac'),
+        server_default=sa.sql.elements.TextClause( str(CatalogExcerptFormatConverter.convert('fitsldac')) ),
         doc="Format of the file on disk.  Currently only fitsldac is supported. "
             "Saved as intetger but is converted to string when loaded."
     )
@@ -117,7 +117,7 @@ def data( self ):
     filters = sa.Column(
         ARRAY(sa.Text, zero_indexes=True),
         nullable=False,
-        default=[],
+        server_default='{}',
         doc=( "Filters covered by the catalog; names of the filters will be "
               "standard for the catalog source, not globally standard." )
     )

models/cutouts.py

@@ -60,7 +60,7 @@ def __table_args__(cls):
     _format = sa.Column(
         sa.SMALLINT,
         nullable=False,
-        default=CutoutsFormatConverter.convert('hdf5'),
+        server_default=sa.sql.elements.TextClause( str(CutoutsFormatConverter.convert('hdf5')) ),
         doc="Format of the file on disk. Should be fits, hdf5, csv or npy. "
             "Saved as integer but is converted to string when loaded. "
     )

models/exposure.py

@@ -177,7 +177,7 @@ def __table_args__( cls ):
     _type = sa.Column(
         sa.SMALLINT,
         nullable=False,
-        default=ImageTypeConverter.convert('Sci'),
+        server_default=sa.sql.elements.TextClause( str(ImageTypeConverter.convert('Sci')) ),
         index=True,
         doc=(
             "Type of image. One of: Sci, Diff, Bias, Dark, DomeFlat, SkyFlat, TwiFlat, "
@@ -202,7 +202,7 @@ def type(self, value):
     _format = sa.Column(
         sa.SMALLINT,
         nullable=False,
-        default=ImageFormatConverter.convert('fits'),
+        server_default=sa.sql.elements.TextClause( str(ImageFormatConverter.convert('fits')) ),
         doc="Format of the file on disk. Should be fits or hdf5. "
             "The value is saved as SMALLINT but translated to a string when read. "
     )
@@ -234,7 +234,7 @@ def format(self, value):
     info = sa.Column(
         JSONB,
         nullable=False,
-        default={},
+        server_default='{}',
         doc=(
             "Subset of the raw exposure's header. "
             "Only keep a subset of the keywords, "
@@ -294,7 +294,7 @@ def filter_short(self):
     _bitflag = sa.Column(
         sa.BIGINT,
         nullable=False,
-        default=0,
+        server_default=sa.sql.elements.TextClause( '0' ),
         index=True,
         doc='Bitflag for this exposure. Good exposures have a bitflag of 0. '
             'Bad exposures are each bad in their own way (i.e., have different bits set). '

models/image.py

@@ -86,7 +86,7 @@ def __table_args__( cls ):
     _format = sa.Column(
         sa.SMALLINT,
         nullable=False,
-        default=ImageFormatConverter.convert('fits'),
+        server_default=sa.sql.elements.TextClause( str(ImageFormatConverter.convert('fits')) ),
         doc="Format of the file on disk. Should be fits or hdf5. "
     )
 
@@ -152,23 +152,23 @@ def upstream_image_ids( self, val ):
     is_sub = sa.Column(
         sa.Boolean,
         nullable=False,
-        default=False,
+        server_default='false',
         index=True,
         doc='Is this a subtraction image.'
     )
 
     is_coadd = sa.Column(
         sa.Boolean,
         nullable=False,
-        default=False,
+        server_default='false',
         index=True,
         doc='Is this image made by stacking multiple images.'
     )
 
     _type = sa.Column(
         sa.SMALLINT,
         nullable=False,
-        default=ImageTypeConverter.convert('Sci'),
+        server_default=sa.sql.elements.TextClause( str(ImageTypeConverter.convert('Sci')) ),
         index=True,
         doc=(
             "Type of image. One of: [Sci, Diff, Bias, Dark, DomeFlat, SkyFlat, TwiFlat, Warped] "
@@ -205,7 +205,7 @@ def type(self, value):
     info = sa.Column(
         JSONB,
         nullable=False,
-        default={},
+        server_default='{}',
         doc=(
             "Additional information on the this image. "
             "Only keep a subset of the header keywords, "
@@ -304,7 +304,7 @@ def mid_mjd(self):
     preproc_bitflag = sa.Column(
         sa.SMALLINT,
         nullable=False,
-        default=0,
+        server_default=sa.sql.elements.TextClause( '0' ),
         index=False,
         doc='Bitflag specifying which preprocessing steps have been completed for the image.'
     )
@@ -321,7 +321,7 @@ def preprocessing_done(self, value):
     astro_cal_done = sa.Column(
         sa.BOOLEAN,
         nullable=False,
-        default=False,
+        server_default='false',
         index=False,
         doc=( 'Has a WCS been solved for this image.  This should be set to true after astro_cal '
               'has been run, or for images (like subtractions) that are derived from other images '
@@ -333,7 +333,7 @@ def preprocessing_done(self, value):
     sky_sub_done = sa.Column(
         sa.BOOLEAN,
         nullable=False,
-        default=False,
+        server_default='false',
         index=False,
         doc='Has the sky been subtracted from this image. '
     )
@@ -434,6 +434,9 @@ def __init__(self, *args, **kwargs):
         self._instrument_object = None
         self._bitflag = 0
         self.is_sub = False
+        self.is_coadd = False
+        self.astro_cal_done = False
+        self.photo_cal_done = False
 
         if 'header' in kwargs:
             kwargs['_header'] = kwargs.pop('header')
@@ -508,31 +511,17 @@ def insert( self, verifyupstreams=False, session=None ):
         """
 
         with SmartSession( session ) as sess:
-            try:
-                # Lock both the images and image_upstreams_association tables to avoid race
-                #   conditions with another process trying to insert the same image.
-                self._get_table_lock( sess )
-                if ( self._upstream_ids is not None ) and ( len(self._upstream_ids) > 0 ):
-                    self._get_table_lock( sess, 'image_upstreams_association' )
-
-                # This will raise an exception if the image is already present; in that case,
-                #   the upstream associations should also already be present (loaded by whoever
-                #   loaded the image), so it's fine to just error out.
-                UUIDMixin.insert( self, session=sess )
-
-                if ( self._upstream_ids is not None ) and ( len(self._upstream_ids) > 0 ):
-                    for ui in self._upstream_ids:
-                        sess.execute( sa.text( "INSERT INTO "
-                                               "image_upstreams_association(upstream_id,downstream_id) "
-                                                   "VALUES (:them,:me)" ),
-                                      { "them": ui, "me": self.id } )
-                    # SCLogger.debug( "Image.insert comitting" )
-                    sess.commit()
-
-            finally:
-                # Make sure the table locks are released
-                # SCLogger.debug( "Image.insert rolling back" )
-                sess.rollback()
+            # Insert the image.  If this raises an exception (because the image already exists),
+            # then we won't futz with the image_upstreams_association table.
+            SeeChangeBase.insert( self, session=sess )
+
+            if ( self._upstream_ids is not None ) and ( len(self._upstream_ids) > 0 ):
+                for ui in self._upstream_ids:
+                    sess.execute( sa.text( "INSERT INTO "
+                                           "image_upstreams_association(upstream_id,downstream_id) "
+                                               "VALUES (:them,:me)" ),
+                                  { "them": ui, "me": self.id } )
+                sess.commit()
 
 
     def set_corners_from_header_wcs( self, wcs=None, setradec=False ):

models/instrument.py

@@ -251,7 +251,7 @@ class SensorSection(Base, UUIDMixin):
     defective = sa.Column(
         sa.Boolean,
         nullable=False,
-        default=False,
+        server_default='false',
         index=True,
         doc='Whether this section is defective (i.e., if True, do not use it!). '
     )

models/knownexposure.py

@@ -87,6 +87,8 @@ class PipelineWorker(Base, UUIDMixin):
 
     cluster_id = sa.Column( sa.Text, nullable=False, doc="Cluster where the worker is running" )
     node_id = sa.Column( sa.Text, nullable=True, doc="Node where the worker is running" )
-    nexps = sa.Column( sa.SmallInteger, nullable=False, default=1,
+    nexps = sa.Column( sa.SmallInteger,
+                       nullable=False,
+                       server_default=sa.sql.elements.TextClause( '1' ),
                        doc="How many exposures this worker can do at once" )
     lastheartbeat = sa.Column( sa.DateTime, nullable=False, doc="Last time this pipeline worker checked in" )

models/measurements.py

@@ -99,7 +99,7 @@ def __table_args__( cls ):
     best_aperture = sa.Column(
         sa.SMALLINT,
         nullable=False,
-        default=-1,
+        server_default=sa.sql.elements.TextClause( '-1' ),
         doc="The index of the aperture that was chosen as the best aperture for this measurement. "
             "Set to -1 to select the PSF flux instead of one of the apertures. "
     )
@@ -305,7 +305,7 @@ def magnitude_err(self):
     disqualifier_scores = sa.Column(
         JSONB,
         nullable=False,
-        default={},
+        server_default='{}',
         index=True,
         doc="Values that may disqualify this object, and mark it as not a real source. "
             "This includes all sorts of analytical cuts defined by the provenance parameters. "

models/object.py

@@ -37,14 +37,14 @@ def __table_args__(cls):
     is_test = sa.Column(
         sa.Boolean,
         nullable=False,
-        default=False,
+        server_default='false',
         doc='Boolean flag to indicate if the object is a test object created during testing. '
     )
 
     is_fake = sa.Column(
         sa.Boolean,
         nullable=False,
-        default=False,
+        server_default='false',
         doc='Boolean flag to indicate if the object is a fake object that has been artificially injected. '
     )