Merge remote-tracking branch 'upstream/main' into 143-limiting-magnitude

.github/workflows/run-pipeline-tests.yml → .github/workflows/run-pipeline-tests-1.yml

@@ -1,4 +1,4 @@
-name: Run Pipeline Tests
+name: Run Pipeline Tests 1
 
 on:
   push:
@@ -59,4 +59,5 @@ jobs:
 
       - name: run test
         run: |
-          TEST_SUBFOLDER=tests/pipeline docker compose run runtests
+          shopt -s nullglob
+          TEST_SUBFOLDER=$(ls tests/pipeline/test_{a..o}*.py) docker compose run runtests

.github/workflows/run-pipeline-tests-2.yml

@@ -0,0 +1,63 @@
+name: Run Pipeline Tests 2
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  tests:
+    name: run tests in docker image
+    runs-on: ubuntu-latest
+    env:
+      REGISTRY: ghcr.io
+      COMPOSE_FILE: tests/docker-compose.yaml
+
+    steps:
+      - name: Dump docker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+
+      - name: checkout code
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: log into github container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: setup docker buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver: docker-container
+
+      - name: bake
+        uses: docker/[email protected]
+        with:
+          workdir: tests
+          load: true
+          files: docker-compose.yaml
+          set: |
+            seechange_postgres.tags=ghcr.io/${{ github.repository_owner }}/seechange-postgres
+            seechange_postgres.cache-from=type=gha,scope=cached-seechange-postgres
+            seechange_postgres.cache-to=type=gha,scope=cached-seechange-postgres,mode=max
+            setuptables.tags=ghcr.io/${{ github.repository_owner }}/runtests
+            setuptables.cache-from=type=gha,scope=cached-seechange
+            setuptables.cache-to=type=gha,scope=cached-seechange,mode=max
+            runtests.tags=ghcr.io/${{ github.repository_owner }}/runtests
+            runtests.cache-from=type=gha,scope=cached-seechange
+            runtests.cache-to=type=gha,scope=cached-seechange,mode=max
+            shell.tags=ghcr.io/${{ github.repository_owner }}/runtests
+            shell.cache-from=type=gha,scope=cached-seechange
+            shell.cache-to=type=gha,scope=cached-seechange,mode=max
+
+      - name: run test
+        run: |
+          shopt -s nullglob
+          TEST_SUBFOLDER=$(ls tests/pipeline/test_{p..z}*.py) docker compose run runtests

.gitignore

@@ -7,11 +7,14 @@ tests/local_config.yaml
 tests/local_overrides.yaml
 tests/local_augments.yaml
 tests/improc/cache/*
-data/cache/*
+data/cache*
 data/DECam_default_calibrators
 .pytest.ini
 tests/plots
 tests/temp_data
+webap/static/rkwebutil.js
+coadd.weight.fits
+tests/coadd.weight.fits
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.gitmodules

@@ -1,6 +1,9 @@
 [submodule "extern/nersc-upload-connector"]
 	path = extern/nersc-upload-connector
 	url = https://github.com/c3-time-domain/nersc-upload-connector
+[submodule "webap/rkwebutil"]
+	path = webap/rkwebutil
+	url = https://github.com/rknop/rkwebutil.git
 [submodule "extern/nersc-desi-gaia-dr3-server"]
 	path = extern/nersc-desi-gaia-dr3-server
 	url = https://github.com/c3-time-domain/nersc-desi-gaia-dr3-server.git

alembic/versions/2024_05_22_2011-2ea9f6f0b790_make_wcs_a_fileondiskmixin.py

@@ -0,0 +1,38 @@
+"""make WCS a FileOnDiskMixin
+
+Revision ID: 2ea9f6f0b790
+Revises: ec64a8fd8cf3
+Create Date: 2024-05-22 20:11:31.195961
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '2ea9f6f0b790'
+down_revision = '9a4097979249'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('world_coordinates', sa.Column('filepath_extensions', postgresql.ARRAY(sa.Text(), zero_indexes=True), nullable=True))
+    op.add_column('world_coordinates', sa.Column('md5sum', sa.UUID(), nullable=True))
+    op.add_column('world_coordinates', sa.Column('md5sum_extensions', postgresql.ARRAY(sa.UUID(), zero_indexes=True), nullable=True))
+    op.add_column('world_coordinates', sa.Column('filepath', sa.Text(), nullable=False))
+    op.create_index(op.f('ix_world_coordinates_filepath'), 'world_coordinates', ['filepath'], unique=True)
+    op.drop_column('world_coordinates', 'header_excerpt')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('world_coordinates', sa.Column('header_excerpt', sa.TEXT(), autoincrement=False, nullable=False))
+    op.drop_index(op.f('ix_world_coordinates_filepath'), table_name='world_coordinates')
+    op.drop_column('world_coordinates', 'filepath')
+    op.drop_column('world_coordinates', 'md5sum_extensions')
+    op.drop_column('world_coordinates', 'md5sum')
+    op.drop_column('world_coordinates', 'filepath_extensions')
+    # ### end Alembic commands ###

alembic/versions/2024_05_23_1652-f36d17393be7_add_bitflag_for_measurements.py

@@ -0,0 +1,36 @@
+"""add bitflag for measurements
+
+Revision ID: f36d17393be7
+Revises: ec64a8fd8cf3
+Create Date: 2024-05-23 16:52:07.448402
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f36d17393be7'
+down_revision = '2ea9f6f0b790'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('measurements', sa.Column('_bitflag', sa.BIGINT(), nullable=False))
+    op.add_column('measurements', sa.Column('description', sa.Text(), nullable=True))
+    op.add_column('measurements', sa.Column('_upstream_bitflag', sa.BIGINT(), nullable=False))
+    op.create_index(op.f('ix_measurements__bitflag'), 'measurements', ['_bitflag'], unique=False)
+    op.create_index(op.f('ix_measurements__upstream_bitflag'), 'measurements', ['_upstream_bitflag'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_measurements__upstream_bitflag'), table_name='measurements')
+    op.drop_index(op.f('ix_measurements__bitflag'), table_name='measurements')
+    op.drop_column('measurements', '_upstream_bitflag')
+    op.drop_column('measurements', 'description')
+    op.drop_column('measurements', '_bitflag')
+    # ### end Alembic commands ###

alembic/versions/2024_05_31_1352-a7dde2327dde_add_is_bad_column_to_objects_and_.py

@@ -0,0 +1,34 @@
+"""Add is_bad column to Objects and Measurements
+
+Revision ID: a7dde2327dde
+Revises: f36d17393be7
+Create Date: 2024-05-31 13:52:26.008896
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a7dde2327dde'
+down_revision = 'f36d17393be7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('measurements', sa.Column('is_bad', sa.Boolean(), nullable=False))
+    op.create_index(op.f('ix_measurements_is_bad'), 'measurements', ['is_bad'], unique=False)
+    op.add_column('objects', sa.Column('is_bad', sa.Boolean(), nullable=False))
+    op.create_index(op.f('ix_objects_is_bad'), 'objects', ['is_bad'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_objects_is_bad'), table_name='objects')
+    op.drop_column('objects', 'is_bad')
+    op.drop_index(op.f('ix_measurements_is_bad'), table_name='measurements')
+    op.drop_column('measurements', 'is_bad')
+    # ### end Alembic commands ###

alembic/versions/2024_06_10_1132-a375526c8260_background_table.py

@@ -0,0 +1,86 @@
+"""background table
+
+Revision ID: a375526c8260
+Revises: a7dde2327dde
+Create Date: 2024-06-10 11:32:39.717922
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'a375526c8260'
+down_revision = 'a7dde2327dde'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('backgrounds',
+    sa.Column('_format', sa.SMALLINT(), nullable=False),
+    sa.Column('_method', sa.SMALLINT(), nullable=False),
+    sa.Column('image_id', sa.BigInteger(), nullable=False),
+    sa.Column('value', sa.Float(), nullable=False),
+    sa.Column('noise', sa.Float(), nullable=False),
+    sa.Column('provenance_id', sa.String(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('modified', sa.DateTime(), nullable=False),
+    sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
+    sa.Column('filepath_extensions', postgresql.ARRAY(sa.Text(), zero_indexes=True), nullable=True),
+    sa.Column('md5sum', sa.UUID(), nullable=True),
+    sa.Column('md5sum_extensions', postgresql.ARRAY(sa.UUID(), zero_indexes=True), nullable=True),
+    sa.Column('filepath', sa.Text(), nullable=False),
+    sa.Column('_bitflag', sa.BIGINT(), nullable=False),
+    sa.Column('description', sa.Text(), nullable=True),
+    sa.Column('_upstream_bitflag', sa.BIGINT(), nullable=False),
+    sa.ForeignKeyConstraint(['image_id'], ['images.id'], name='backgrounds_image_id_fkey', ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['provenance_id'], ['provenances.id'], name='backgrounds_provenance_id_fkey', ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index('backgrounds_image_id_provenance_index', 'backgrounds', ['image_id', 'provenance_id'], unique=True)
+    op.create_index(op.f('ix_backgrounds__bitflag'), 'backgrounds', ['_bitflag'], unique=False)
+    op.create_index(op.f('ix_backgrounds__upstream_bitflag'), 'backgrounds', ['_upstream_bitflag'], unique=False)
+    op.create_index(op.f('ix_backgrounds_created_at'), 'backgrounds', ['created_at'], unique=False)
+    op.create_index(op.f('ix_backgrounds_filepath'), 'backgrounds', ['filepath'], unique=True)
+    op.create_index(op.f('ix_backgrounds_id'), 'backgrounds', ['id'], unique=False)
+    op.create_index(op.f('ix_backgrounds_image_id'), 'backgrounds', ['image_id'], unique=False)
+    op.create_index(op.f('ix_backgrounds_noise'), 'backgrounds', ['noise'], unique=False)
+    op.create_index(op.f('ix_backgrounds_provenance_id'), 'backgrounds', ['provenance_id'], unique=False)
+    op.create_index(op.f('ix_backgrounds_value'), 'backgrounds', ['value'], unique=False)
+    op.add_column('source_lists', sa.Column('inf_aper_num', sa.SMALLINT(), nullable=True))
+    op.add_column('source_lists', sa.Column('best_aper_num', sa.SMALLINT(), nullable=True))
+    op.drop_column('source_lists', '_inf_aper_num')
+
+    op.add_column('measurements', sa.Column('bkg_mean', sa.REAL(), nullable=False))
+    op.add_column('measurements', sa.Column('bkg_std', sa.REAL(), nullable=False))
+    op.add_column('measurements', sa.Column('bkg_pix', sa.REAL(), nullable=False))
+    op.drop_column('measurements', 'background')
+    op.drop_column('measurements', 'background_err')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('measurements', sa.Column('background_err', sa.REAL(), autoincrement=False, nullable=False))
+    op.add_column('measurements', sa.Column('background', sa.REAL(), autoincrement=False, nullable=False))
+    op.drop_column('measurements', 'bkg_pix')
+    op.drop_column('measurements', 'bkg_std')
+    op.drop_column('measurements', 'bkg_mean')
+
+    op.add_column('source_lists', sa.Column('_inf_aper_num', sa.SMALLINT(), autoincrement=False, nullable=True))
+    op.drop_column('source_lists', 'best_aper_num')
+    op.drop_column('source_lists', 'inf_aper_num')
+    op.drop_index(op.f('ix_backgrounds_value'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds_provenance_id'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds_noise'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds_image_id'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds_id'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds_filepath'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds_created_at'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds__upstream_bitflag'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds__bitflag'), table_name='backgrounds')
+    op.drop_index('backgrounds_image_id_provenance_index', table_name='backgrounds')
+    op.drop_table('backgrounds')
+    # ### end Alembic commands ###