From 612dc5557c3cf13a489b5222204fc04b0b0e971d Mon Sep 17 00:00:00 2001 From: Rob Knop Date: Tue, 2 Jul 2024 13:46:52 -0700 Subject: [PATCH] my god i hate sqlalchem --- .../2024_07_01_2120-ceec8a848b40_authuser.py | 58 +++++++++++++ .../2024_07_01_2121-235bbd00c9c2_conductor.py | 83 +++++++++++++++++++ ...a6bab3f7_modify_calibfile_downloadloack.py | 32 +++++++ pipeline/data_store.py | 1 + tests/fixtures/decam.py | 10 +-- tests/seechange_config_test.yaml | 2 +- 6 files changed, 180 insertions(+), 6 deletions(-) create mode 100644 alembic/versions/2024_07_01_2120-ceec8a848b40_authuser.py create mode 100644 alembic/versions/2024_07_01_2121-235bbd00c9c2_conductor.py create mode 100644 alembic/versions/2024_07_01_2122-685ba6bab3f7_modify_calibfile_downloadloack.py diff --git a/alembic/versions/2024_07_01_2120-ceec8a848b40_authuser.py b/alembic/versions/2024_07_01_2120-ceec8a848b40_authuser.py new file mode 100644 index 00000000..5d74e2d8 --- /dev/null +++ b/alembic/versions/2024_07_01_2120-ceec8a848b40_authuser.py @@ -0,0 +1,58 @@ +"""authuser + +Revision ID: ceec8a848b40 +Revises: 7384c6d07485 +Create Date: 2024-06-10 17:52:28.527093 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'ceec8a848b40' +down_revision = '7384c6d07485' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('authuser', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('username', sa.Text(), nullable=False), + sa.Column('displayname', sa.Text(), nullable=False), + sa.Column('email', sa.Text(), nullable=False), + sa.Column('pubkey', sa.Text(), nullable=True), + sa.Column('privkey', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('modified', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_authuser_created_at'), 'authuser', ['created_at'], unique=False) + op.create_index(op.f('ix_authuser_email'), 'authuser', ['email'], unique=False) + op.create_index(op.f('ix_authuser_username'), 'authuser', ['username'], unique=True) + op.create_table('passwordlink', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('userid', sa.UUID(), nullable=True), + sa.Column('expires', sa.DateTime(timezone=True), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('modified', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['userid'], ['authuser.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_passwordlink_created_at'), 'passwordlink', ['created_at'], unique=False) + op.create_index(op.f('ix_passwordlink_userid'), 'passwordlink', ['userid'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_passwordlink_userid'), table_name='passwordlink') + op.drop_index(op.f('ix_passwordlink_created_at'), table_name='passwordlink') + op.drop_table('passwordlink') + op.drop_index(op.f('ix_authuser_username'), table_name='authuser') + op.drop_index(op.f('ix_authuser_email'), table_name='authuser') + op.drop_index(op.f('ix_authuser_created_at'), table_name='authuser') + op.drop_table('authuser') + # ### end Alembic commands ### diff --git a/alembic/versions/2024_07_01_2121-235bbd00c9c2_conductor.py b/alembic/versions/2024_07_01_2121-235bbd00c9c2_conductor.py new file mode 100644 index 00000000..d4e9fad7 --- /dev/null +++ b/alembic/versions/2024_07_01_2121-235bbd00c9c2_conductor.py @@ -0,0 +1,83 @@ +"""conductor + +Revision ID: 235bbd00c9c2 +Revises: ceec8a848b40 +Create Date: 2024-06-13 17:31:25.857888 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '235bbd00c9c2' +down_revision = 'ceec8a848b40' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('pipelineworkers', + sa.Column('cluster_id', sa.Text(), nullable=False), + sa.Column('node_id', sa.Text(), nullable=True), + sa.Column('nexps', sa.SmallInteger(), nullable=False), + sa.Column('lastheartbeat', sa.DateTime(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('modified', sa.DateTime(), nullable=False), + sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_pipelineworkers_created_at'), 'pipelineworkers', ['created_at'], unique=False) + op.create_index(op.f('ix_pipelineworkers_id'), 'pipelineworkers', ['id'], unique=False) + op.create_table('knownexposures', + sa.Column('instrument', sa.Text(), nullable=False), + sa.Column('identifier', sa.Text(), nullable=False), + sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('hold', sa.Boolean(), server_default='false', nullable=False), + sa.Column('exposure_id', sa.BigInteger(), nullable=True), + sa.Column('mjd', sa.Double(), nullable=True), + sa.Column('exp_time', sa.REAL(), nullable=True), + sa.Column('filter', sa.Text(), nullable=True), + sa.Column('project', sa.Text(), nullable=True), + sa.Column('target', sa.Text(), nullable=True), + sa.Column('cluster_id', sa.Text(), nullable=True), + sa.Column('claim_time', sa.DateTime(), nullable=True), + sa.Column('ra', sa.Double(), nullable=True), + sa.Column('dec', sa.Double(), nullable=True), + sa.Column('gallat', sa.Double(), nullable=True), + sa.Column('gallon', sa.Double(), nullable=True), + sa.Column('ecllat', sa.Double(), nullable=True), + sa.Column('ecllon', sa.Double(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('modified', sa.DateTime(), nullable=False), + sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False), + sa.ForeignKeyConstraint(['exposure_id'], ['exposures.id'], name='knownexposure_exposure_id_fkey'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_knownexposures_created_at'), 'knownexposures', ['created_at'], unique=False) + op.create_index(op.f('ix_knownexposures_ecllat'), 'knownexposures', ['ecllat'], unique=False) + op.create_index(op.f('ix_knownexposures_gallat'), 'knownexposures', ['gallat'], unique=False) + op.create_index(op.f('ix_knownexposures_id'), 'knownexposures', ['id'], unique=False) + op.create_index(op.f('ix_knownexposures_identifier'), 'knownexposures', ['identifier'], unique=False) + op.create_index(op.f('ix_knownexposures_instrument'), 'knownexposures', ['instrument'], unique=False) + op.create_index(op.f('ix_knownexposures_mjd'), 'knownexposures', ['mjd'], unique=False) + op.create_index('knownexposures_q3c_ang2ipix_idx', 'knownexposures', [sa.text('q3c_ang2ipix(ra, dec)')], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('knownexposures_q3c_ang2ipix_idx', table_name='knownexposures') + op.drop_index(op.f('ix_knownexposures_mjd'), table_name='knownexposures') + op.drop_index(op.f('ix_knownexposures_instrument'), table_name='knownexposures') + op.drop_index(op.f('ix_knownexposures_identifier'), table_name='knownexposures') + op.drop_index(op.f('ix_knownexposures_id'), table_name='knownexposures') + op.drop_index(op.f('ix_knownexposures_gallat'), table_name='knownexposures') + op.drop_index(op.f('ix_knownexposures_ecllat'), table_name='knownexposures') + op.drop_index(op.f('ix_knownexposures_created_at'), table_name='knownexposures') + op.drop_table('knownexposures') + op.drop_index(op.f('ix_pipelineworkers_id'), table_name='pipelineworkers') + op.drop_index(op.f('ix_pipelineworkers_created_at'), table_name='pipelineworkers') + op.drop_table('pipelineworkers') + # ### end Alembic commands ### diff --git a/alembic/versions/2024_07_01_2122-685ba6bab3f7_modify_calibfile_downloadloack.py b/alembic/versions/2024_07_01_2122-685ba6bab3f7_modify_calibfile_downloadloack.py new file mode 100644 index 00000000..d186eeac --- /dev/null +++ b/alembic/versions/2024_07_01_2122-685ba6bab3f7_modify_calibfile_downloadloack.py @@ -0,0 +1,32 @@ +"""modify_calibfile_downloadloack + +Revision ID: 685ba6bab3f7 +Revises: 235bbd00c9c2 +Create Date: 2024-06-25 15:18:31.400636 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '685ba6bab3f7' +down_revision = '235bbd00c9c2' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('calibfile_downloadlock', 'sensor_section', + existing_type=sa.TEXT(), + nullable=True) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('calibfile_downloadlock', 'sensor_section', + existing_type=sa.TEXT(), + nullable=False) + # ### end Alembic commands ### diff --git a/pipeline/data_store.py b/pipeline/data_store.py index 4b3043a2..0ca985f4 100644 --- a/pipeline/data_store.py +++ b/pipeline/data_store.py @@ -1497,6 +1497,7 @@ def save_and_commit(self, exists_ok=False, overwrite=True, no_archive=False, self.products_committed = 'image, sources, psf, wcs, zp, bg' if self.sub_image is not None: + import pdb; pdb.set_trace() if self.reference is not None: self.reference = self.reference.merge_all(session) self.sub_image.new_image = self.image # update with the now-merged image diff --git a/tests/fixtures/decam.py b/tests/fixtures/decam.py index be35b039..3b9eea65 100644 --- a/tests/fixtures/decam.py +++ b/tests/fixtures/decam.py @@ -458,10 +458,9 @@ def decam_elais_e1_two_references( decam_elais_e1_two_refs_datastore, refmaker_f with SmartSession() as session: maker = refmaker_factory('test_refset_decam', 'DECam') maker.make_refset(session=session) + prov = maker.refset.provenances[0] + prov = session.merge(prov) for ds in decam_elais_e1_two_refs_datastore: - prov = maker.refset.provenances[0] - prov = session.merge(prov) - ref = Reference() ref.image = ds.image ref.provenance = prov @@ -474,10 +473,11 @@ def decam_elais_e1_two_references( decam_elais_e1_two_refs_datastore, refmaker_f ref = ref.merge_all(session=session) if not sa.inspect(ref).persistent: - ref = session.merge(ref) - + ref = session.merge( ref ) refs.append( ref ) + session.commit() + yield refs for ref in refs: diff --git a/tests/seechange_config_test.yaml b/tests/seechange_config_test.yaml index 5b64d8ca..bdcfc8f0 100644 --- a/tests/seechange_config_test.yaml +++ b/tests/seechange_config_test.yaml @@ -27,4 +27,4 @@ conductor: subtraction: method: zogy - + refset: test_refset_decam # This is needed for the pipeline_exposure_launcher test