my god i hate sqlalchem

alembic/versions/2024_07_01_2120-ceec8a848b40_authuser.py

@@ -0,0 +1,58 @@
+"""authuser
+
+Revision ID: ceec8a848b40
+Revises: 7384c6d07485
+Create Date: 2024-06-10 17:52:28.527093
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'ceec8a848b40'
+down_revision = '7384c6d07485'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('authuser',
+    sa.Column('id', sa.UUID(), nullable=False),
+    sa.Column('username', sa.Text(), nullable=False),
+    sa.Column('displayname', sa.Text(), nullable=False),
+    sa.Column('email', sa.Text(), nullable=False),
+    sa.Column('pubkey', sa.Text(), nullable=True),
+    sa.Column('privkey', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('modified', sa.DateTime(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_authuser_created_at'), 'authuser', ['created_at'], unique=False)
+    op.create_index(op.f('ix_authuser_email'), 'authuser', ['email'], unique=False)
+    op.create_index(op.f('ix_authuser_username'), 'authuser', ['username'], unique=True)
+    op.create_table('passwordlink',
+    sa.Column('id', sa.UUID(), nullable=False),
+    sa.Column('userid', sa.UUID(), nullable=True),
+    sa.Column('expires', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('modified', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['userid'], ['authuser.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_passwordlink_created_at'), 'passwordlink', ['created_at'], unique=False)
+    op.create_index(op.f('ix_passwordlink_userid'), 'passwordlink', ['userid'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_passwordlink_userid'), table_name='passwordlink')
+    op.drop_index(op.f('ix_passwordlink_created_at'), table_name='passwordlink')
+    op.drop_table('passwordlink')
+    op.drop_index(op.f('ix_authuser_username'), table_name='authuser')
+    op.drop_index(op.f('ix_authuser_email'), table_name='authuser')
+    op.drop_index(op.f('ix_authuser_created_at'), table_name='authuser')
+    op.drop_table('authuser')
+    # ### end Alembic commands ###

alembic/versions/2024_07_01_2121-235bbd00c9c2_conductor.py

@@ -0,0 +1,83 @@
+"""conductor
+
+Revision ID: 235bbd00c9c2
+Revises: ceec8a848b40
+Create Date: 2024-06-13 17:31:25.857888
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '235bbd00c9c2'
+down_revision = 'ceec8a848b40'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('pipelineworkers',
+    sa.Column('cluster_id', sa.Text(), nullable=False),
+    sa.Column('node_id', sa.Text(), nullable=True),
+    sa.Column('nexps', sa.SmallInteger(), nullable=False),
+    sa.Column('lastheartbeat', sa.DateTime(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('modified', sa.DateTime(), nullable=False),
+    sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_pipelineworkers_created_at'), 'pipelineworkers', ['created_at'], unique=False)
+    op.create_index(op.f('ix_pipelineworkers_id'), 'pipelineworkers', ['id'], unique=False)
+    op.create_table('knownexposures',
+    sa.Column('instrument', sa.Text(), nullable=False),
+    sa.Column('identifier', sa.Text(), nullable=False),
+    sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    sa.Column('hold', sa.Boolean(), server_default='false', nullable=False),
+    sa.Column('exposure_id', sa.BigInteger(), nullable=True),
+    sa.Column('mjd', sa.Double(), nullable=True),
+    sa.Column('exp_time', sa.REAL(), nullable=True),
+    sa.Column('filter', sa.Text(), nullable=True),
+    sa.Column('project', sa.Text(), nullable=True),
+    sa.Column('target', sa.Text(), nullable=True),
+    sa.Column('cluster_id', sa.Text(), nullable=True),
+    sa.Column('claim_time', sa.DateTime(), nullable=True),
+    sa.Column('ra', sa.Double(), nullable=True),
+    sa.Column('dec', sa.Double(), nullable=True),
+    sa.Column('gallat', sa.Double(), nullable=True),
+    sa.Column('gallon', sa.Double(), nullable=True),
+    sa.Column('ecllat', sa.Double(), nullable=True),
+    sa.Column('ecllon', sa.Double(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('modified', sa.DateTime(), nullable=False),
+    sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
+    sa.ForeignKeyConstraint(['exposure_id'], ['exposures.id'], name='knownexposure_exposure_id_fkey'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_knownexposures_created_at'), 'knownexposures', ['created_at'], unique=False)
+    op.create_index(op.f('ix_knownexposures_ecllat'), 'knownexposures', ['ecllat'], unique=False)
+    op.create_index(op.f('ix_knownexposures_gallat'), 'knownexposures', ['gallat'], unique=False)
+    op.create_index(op.f('ix_knownexposures_id'), 'knownexposures', ['id'], unique=False)
+    op.create_index(op.f('ix_knownexposures_identifier'), 'knownexposures', ['identifier'], unique=False)
+    op.create_index(op.f('ix_knownexposures_instrument'), 'knownexposures', ['instrument'], unique=False)
+    op.create_index(op.f('ix_knownexposures_mjd'), 'knownexposures', ['mjd'], unique=False)
+    op.create_index('knownexposures_q3c_ang2ipix_idx', 'knownexposures', [sa.text('q3c_ang2ipix(ra, dec)')], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('knownexposures_q3c_ang2ipix_idx', table_name='knownexposures')
+    op.drop_index(op.f('ix_knownexposures_mjd'), table_name='knownexposures')
+    op.drop_index(op.f('ix_knownexposures_instrument'), table_name='knownexposures')
+    op.drop_index(op.f('ix_knownexposures_identifier'), table_name='knownexposures')
+    op.drop_index(op.f('ix_knownexposures_id'), table_name='knownexposures')
+    op.drop_index(op.f('ix_knownexposures_gallat'), table_name='knownexposures')
+    op.drop_index(op.f('ix_knownexposures_ecllat'), table_name='knownexposures')
+    op.drop_index(op.f('ix_knownexposures_created_at'), table_name='knownexposures')
+    op.drop_table('knownexposures')
+    op.drop_index(op.f('ix_pipelineworkers_id'), table_name='pipelineworkers')
+    op.drop_index(op.f('ix_pipelineworkers_created_at'), table_name='pipelineworkers')
+    op.drop_table('pipelineworkers')
+    # ### end Alembic commands ###

alembic/versions/2024_07_01_2122-685ba6bab3f7_modify_calibfile_downloadloack.py

@@ -0,0 +1,32 @@
+"""modify_calibfile_downloadloack
+
+Revision ID: 685ba6bab3f7
+Revises: 235bbd00c9c2
+Create Date: 2024-06-25 15:18:31.400636
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '685ba6bab3f7'
+down_revision = '235bbd00c9c2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('calibfile_downloadlock', 'sensor_section',
+               existing_type=sa.TEXT(),
+               nullable=True)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('calibfile_downloadlock', 'sensor_section',
+               existing_type=sa.TEXT(),
+               nullable=False)
+    # ### end Alembic commands ###

pipeline/data_store.py

@@ -1497,6 +1497,7 @@ def save_and_commit(self, exists_ok=False, overwrite=True, no_archive=False,
             self.products_committed = 'image, sources, psf, wcs, zp, bg'
 
             if self.sub_image is not None:
+                import pdb; pdb.set_trace()
                 if self.reference is not None:
                     self.reference = self.reference.merge_all(session)
                 self.sub_image.new_image = self.image  # update with the now-merged image

tests/fixtures/decam.py

@@ -458,10 +458,9 @@ def decam_elais_e1_two_references( decam_elais_e1_two_refs_datastore, refmaker_f
     with SmartSession() as session:
         maker = refmaker_factory('test_refset_decam', 'DECam')
         maker.make_refset(session=session)
+        prov = maker.refset.provenances[0]
+        prov = session.merge(prov)
         for ds in decam_elais_e1_two_refs_datastore:
-            prov = maker.refset.provenances[0]
-            prov = session.merge(prov)
-
             ref = Reference()
             ref.image = ds.image
             ref.provenance = prov
@@ -474,10 +473,11 @@ def decam_elais_e1_two_references( decam_elais_e1_two_refs_datastore, refmaker_f
 
             ref = ref.merge_all(session=session)
             if not sa.inspect(ref).persistent:
-                ref = session.merge(ref)
-
+                ref = session.merge( ref )
             refs.append( ref )
 
+        session.commit()
+
     yield refs
 
     for ref in refs:

tests/seechange_config_test.yaml

@@ -27,4 +27,4 @@ conductor:
 
 subtraction:
   method: zogy
-
+  refset: test_refset_decam    # This is needed for the pipeline_exposure_launcher test