Preprocessing (specifically WRT DECam)

alembic/versions/2023_09_29_1734-32690eb49d8d_calibrator_data_files.py

@@ -0,0 +1,96 @@
+"""calibrator_data_files
+
+Revision ID: 32690eb49d8d
+Revises: afc54edee946
+Create Date: 2023-09-29 17:34:10.839608
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '32690eb49d8d'
+down_revision = 'afc54edee946'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('data_files',
+    sa.Column('provenance_id', sa.String(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('modified', sa.DateTime(), nullable=False),
+    sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
+    sa.Column('filepath', sa.Text(), nullable=False),
+    sa.Column('filepath_extensions', sa.ARRAY(sa.Text()), nullable=True),
+    sa.Column('md5sum', sa.UUID(), nullable=True),
+    sa.Column('md5sum_extensions', sa.ARRAY(sa.UUID()), nullable=True),
+    sa.CheckConstraint('NOT(md5sum IS NULL AND md5sum_extensions IS NULL)', name='md5sum_or_md5sum_extensions_check'),
+    sa.ForeignKeyConstraint(['provenance_id'], ['provenances.id'], name='data_files_provenance_id_fkey', ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_data_files_created_at'), 'data_files', ['created_at'], unique=False)
+    op.create_index(op.f('ix_data_files_filepath'), 'data_files', ['filepath'], unique=True)
+    op.create_index(op.f('ix_data_files_id'), 'data_files', ['id'], unique=False)
+    op.create_index(op.f('ix_data_files_provenance_id'), 'data_files', ['provenance_id'], unique=False)
+    op.create_table('calibrator_files',
+    sa.Column('_type', sa.SMALLINT(), nullable=False),
+    sa.Column('_calibrator_set', sa.SMALLINT(), nullable=False),
+    sa.Column('_flat_type', sa.SMALLINT(), nullable=True),
+    sa.Column('instrument', sa.Text(), nullable=False),
+    sa.Column('sensor_section', sa.Text(), nullable=False),
+    sa.Column('image_id', sa.BigInteger(), nullable=True),
+    sa.Column('datafile_id', sa.BigInteger(), nullable=True),
+    sa.Column('validity_start', sa.DateTime(), nullable=True),
+    sa.Column('validity_end', sa.DateTime(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('modified', sa.DateTime(), nullable=False),
+    sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
+    sa.ForeignKeyConstraint(['datafile_id'], ['data_files.id'], name='calibrator_files_data_file_id_fkey', ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['image_id'], ['images.id'], name='calibrator_files_image_id_fkey', ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_calibrator_files__calibrator_set'), 'calibrator_files', ['_calibrator_set'], unique=False)
+    op.create_index(op.f('ix_calibrator_files__flat_type'), 'calibrator_files', ['_flat_type'], unique=False)
+    op.create_index(op.f('ix_calibrator_files__type'), 'calibrator_files', ['_type'], unique=False)
+    op.create_index(op.f('ix_calibrator_files_created_at'), 'calibrator_files', ['created_at'], unique=False)
+    op.create_index(op.f('ix_calibrator_files_datafile_id'), 'calibrator_files', ['datafile_id'], unique=False)
+    op.create_index(op.f('ix_calibrator_files_id'), 'calibrator_files', ['id'], unique=False)
+    op.create_index(op.f('ix_calibrator_files_image_id'), 'calibrator_files', ['image_id'], unique=False)
+    op.create_index(op.f('ix_calibrator_files_instrument'), 'calibrator_files', ['instrument'], unique=False)
+    op.create_index(op.f('ix_calibrator_files_sensor_section'), 'calibrator_files', ['sensor_section'], unique=False)
+    op.create_index(op.f('ix_calibrator_files_validity_end'), 'calibrator_files', ['validity_end'], unique=False)
+    op.create_index(op.f('ix_calibrator_files_validity_start'), 'calibrator_files', ['validity_start'], unique=False)
+    op.add_column('images', sa.Column('preproc_bitflag', sa.SMALLINT(), nullable=False))
+    op.alter_column('images', 'filter',
+               existing_type=sa.TEXT(),
+               nullable=True)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('images', 'filter',
+               existing_type=sa.TEXT(),
+               nullable=False)
+    op.drop_column('images', 'preproc_bitflag')
+    op.drop_index(op.f('ix_calibrator_files_validity_start'), table_name='calibrator_files')
+    op.drop_index(op.f('ix_calibrator_files_validity_end'), table_name='calibrator_files')
+    op.drop_index(op.f('ix_calibrator_files_sensor_section'), table_name='calibrator_files')
+    op.drop_index(op.f('ix_calibrator_files_instrument'), table_name='calibrator_files')
+    op.drop_index(op.f('ix_calibrator_files_image_id'), table_name='calibrator_files')
+    op.drop_index(op.f('ix_calibrator_files_id'), table_name='calibrator_files')
+    op.drop_index(op.f('ix_calibrator_files_datafile_id'), table_name='calibrator_files')
+    op.drop_index(op.f('ix_calibrator_files_created_at'), table_name='calibrator_files')
+    op.drop_index(op.f('ix_calibrator_files__type'), table_name='calibrator_files')
+    op.drop_index(op.f('ix_calibrator_files__flat_type'), table_name='calibrator_files')
+    op.drop_index(op.f('ix_calibrator_files__calibrator_set'), table_name='calibrator_files')
+    op.drop_table('calibrator_files')
+    op.drop_index(op.f('ix_data_files_provenance_id'), table_name='data_files')
+    op.drop_index(op.f('ix_data_files_id'), table_name='data_files')
+    op.drop_index(op.f('ix_data_files_filepath'), table_name='data_files')
+    op.drop_index(op.f('ix_data_files_created_at'), table_name='data_files')
+    op.drop_table('data_files')
+    # ### end Alembic commands ###

default_config.yaml

@@ -57,7 +57,7 @@ archive: null
 pipeline: {}
 
 preprocessing:
-  use_sky_subtraction: True
+  use_sky_subtraction: False
 
 extraction: {}
 
@@ -81,3 +81,33 @@ measurement:
   photometry_method: aperture
   aperture_radius: 3.0
   real_bogus_version: null
+
+# Specific configuration for specific instruments
+# Instruments should override the two defaults from
+# instrument_default; they may add additional
+# configuration that their code needs.
+
+instrument_default:
+  calibratorset: nightly
+  flattype: sky
+
+# DECam
+
+# For the observatory-supplied calibrations, NOIRLab distributes them on
+# a Google Drive, which is a nightmare to try to download
+# programmatically.  What's more, they come in ginormous tar files,
+# which are slow to process; especially for tests, where we just want to
+# do a couple of chips, it's not worth the time.  So, I've untarred the
+# files on the NERSC web portal to allow them to be grabbed
+# individually.  These can be found on Perlmutter at
+# /global/cfs/dirs/m2218/www/decam_calibration_files
+
+DECam:
+  calibratorset: externally_supplied
+  flattype: externally_supplied
+  calibfiles:
+    mjd: 56876
+    urlbase: https://portal.nersc.gov/cfs/m2218/decam_calibration_files/
+    linearity: DECamMasterCal_56475/linearity/linearity_table_v0.4.fits
+    fringebase: DECamMasterCal_56876/fringecor/DECam_Master_20131115v1
+    flatbase: DECamMasterCal_56876/starflat/DECam_Master_20130829v3

models/base.py

@@ -225,7 +225,7 @@ def recursive_merge(self, session, done_list=None):
 
         # only do the sub-properties if the object was already added to the session
         attributes = ['provenance', 'code_version',
-                      'exposure', 'image', 'ref_image', 'new_image', 'sub_image', 'source_list']
+                      'exposure', 'image', 'datafile', 'ref_image', 'new_image', 'sub_image', 'source_list']
 
         # recursively call this on the provenance and other parent objects
         for att in attributes:

models/calibratorfile.py

@@ -0,0 +1,129 @@
+import sqlalchemy as sa
+from sqlalchemy import orm
+from sqlalchemy.ext.hybrid import hybrid_property
+
+from models.base import Base, AutoIDMixin
+from models.image import Image
+from models.datafile import DataFile
+from models.enums_and_bitflags import CalibratorTypeConverter, CalibratorSetConverter, FlatTypeConverter
+
+class CalibratorFile(Base, AutoIDMixin):
+    __tablename__ = 'calibrator_files'
+
+    _type = sa.Column(
+        sa.SMALLINT,
+        nullable=False,
+        index=True,
+        default=CalibratorTypeConverter.convert( 'unknown' ),
+        doc="Type of calibrator (Dark, Flat, Linearity, etc.)"
+    )
+
+    @hybrid_property
+    def type( self ):
+        return CalibratorTypeConverter.convert( self._type )
+
+    @type.expression
+    def type( cls ):
+        return sa.case( CalibratorTypeConverter.dict, value=cls._type )
+
+    @type.setter
+    def type( self, value ):
+        self._type = CalibratorTypeConverter.convert( value )
+
+    _calibrator_set = sa.Column(
+        sa.SMALLINT,
+        nullable=False,
+        index=True,
+        default=CalibratorTypeConverter.convert('unknown'),
+        doc="Calibrator set for instrument (unknown, externally_supplied, general, nightly)"
+    )
+
+    @hybrid_property
+    def calibrator_set( self ):
+        return CalibratorSetConverter.convert( self._type )
+
+    @calibrator_set.expression
+    def calibrator_set( cls ):
+        return sa.case( CalibratorSetConverter.dict, value=cls._calibrator_set )
+
+    @calibrator_set.setter
+    def calibrator_set( self, value ):
+        self._calibrator_set = CalibratorSetConverter.convert( value )
+
+    _flat_type = sa.Column(
+        sa.SMALLINT,
+        nullable=True,
+        index=True,
+        doc="Type of flat (unknown, observatory_supplied, sky, twilight, dome), or None if not a flat"
+    )
+
+    @hybrid_property
+    def flat_type( self ):
+        return FlatTypeConverter.convert( self._type )
+
+    @flat_type.inplace.expression
+    @classmethod
+    def flat_type( cls ):
+        return sa.case( FlatTypeConverter.dict, value=cls._flat_type )
+
+    @flat_type.setter
+    def flat_type( self, value ):
+        self._flat_type = FlatTypeConverter.convert( value )
+
+    instrument = sa.Column(
+        sa.Text,
+        nullable=False,
+        index=True,
+        doc="Instrument this calibrator image is for"
+    )
+
+    sensor_section = sa.Column(
+        sa.Text,
+        nullable=False,
+        index=True,
+        doc="Sensor Section of the Instrument this calibrator image is for"
+    )
+
+    image_id = sa.Column(
+        sa.ForeignKey( 'images.id', ondelete='CASCADE', name='calibrator_files_image_id_fkey' ),
+        nullable=True,
+        index=True,
+        doc='ID of the image (if any) that is this calibrator'
+    )
+
+    image = orm.relationship(
+        'Image',
+        cascade='save-update, merge, refresh-expire, expunge',  # ROB REVIEW THIS
+        doc='Image for this CalibratorImage (if any)'
+    )
+
+    datafile_id = sa.Column(
+        sa.ForeignKey( 'data_files.id', ondelete='CASCADE', name='calibrator_files_data_file_id_fkey' ),
+        nullable=True,
+        index=True,
+        doc='ID of the miscellaneous data file (if any) that is this calibrator'
+    )
+
+    datafile = orm.relationship(
+        'DataFile',
+        cascade='save-update, merge, refresh-expire, expunge', # ROB REVIEW THIS
+        doc='DataFile for this CalibratorFile (if any)'
+    )
+
+    validity_start = sa.Column(
+        sa.DateTime,
+        nullable=True,
+        index=True,
+        doc=( 'The time (of exposure acquisition) for exposures for '
+              ' which this calibrator file becomes valid.  If None, this '
+              ' calibrator is valid from the beginning of time.' )
+    )
+
+    validity_end = sa.Column(
+        sa.DateTime,
+        nullable=True,
+        index=True,
+        doc=( 'The time (of exposure acquisition) for exposures for '
+              ' which this calibrator file is no longer.  If None, this '
+              ' calibrator is valid to the end of time.' )
+    )

models/datafile.py

@@ -0,0 +1,41 @@
+import sqlalchemy as sa
+from sqlalchemy import orm
+
+from models.base import Base, SeeChangeBase, AutoIDMixin, FileOnDiskMixin
+
+class DataFile( Base, AutoIDMixin, FileOnDiskMixin ):
+    """Miscellaneous data files."""
+
+    __tablename__ = "data_files"
+
+    provenance_id = sa.Column(
+        sa.ForeignKey( 'provenances.id', ondelete='CASCADE', name='data_files_provenance_id_fkey' ),
+        nullable=False,
+        index=True,
+        doc="ID of the provenance of this miscellaneous data file"
+    )
+
+    provenance = orm.relationship(
+        'Provenance',
+        cascade='save-update, merge, refresh-expire, expunge',
+        lazy='selectin',
+        doc=(
+            "Provenance of this data file. "
+            "The provenance will contain a record of the code version "
+            "and the parameters used to produce this file. "
+        )
+    )
+
+    def __init__( self, *args, **kwargs ):
+        FileOnDiskMixin.__init__(self, *args, **kwargs)
+        SeeChangeBase.__init__(self)  # don't pass kwargs as they could contain non-column key-values
+
+        # manually set all properties (columns or not)
+        for key, value in kwargs.items():
+            if hasattr(self, key):
+                setattr(self, key, value)
+
+    @orm.reconstructor
+    def init_on_load( self ):
+        Base.init_on_load( self )
+        FileOnDiskMixin.init_on_load( self )