Skip to content

Commit

Permalink
End-to-end pipeline tests (c3-time-domain#39)
Browse files Browse the repository at this point in the history
  • Loading branch information
@guynir42
guynir42 authored Jul 12, 2023
1 parent 20d1ece commit 4844eab
Show file tree
Hide file tree
Showing 28 changed files with 1,924 additions and 446 deletions.
2 changes: 1 addition & 1 deletion alembic/versions/2023_05_31_1639-4114e36a2555_image_model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def upgrade() -> None:
op.create_index(op.f('ix_images_telescope'), 'images', ['telescope'], unique=False)
op.create_index(op.f('ix_images_type'), 'images', ['type'], unique=False)
op.create_foreign_key(None, 'images', 'provenances', ['provenance_id'], ['id'], ondelete='CASCADE')
op.create_foreign_key(None, 'images', 'exposures', ['exposure_id'], ['id'])
op.create_foreign_key(None, 'images', 'exposures', ['exposure_id'], ['id'], ondelete='SET NULL')
# ### end Alembic commands ###


Expand Down
213 changes: 213 additions & 0 deletions alembic/versions/2023_06_27_1350-b90b1e3ec58c_reference_table.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
"""reference table
Revision ID: b90b1e3ec58c
Revises: 4114e36a2555
Create Date: 2023-06-27 13:50:00.391100
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = 'b90b1e3ec58c'
down_revision = '4114e36a2555'
branch_labels = None
depends_on = None


def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('reference_images',
sa.Column('image_id', sa.BigInteger(), nullable=False),
sa.Column('target', sa.Text(), nullable=False),
sa.Column('filter', sa.Text(), nullable=False),
sa.Column('section_id', sa.Text(), nullable=False),
sa.Column('validity_start', sa.DateTime(), nullable=False),
sa.Column('validity_end', sa.DateTime(), nullable=False),
sa.Column('is_bad', sa.Boolean(), nullable=False),
sa.Column('bad_reason', sa.Text(), nullable=True),
sa.Column('bad_comment', sa.Text(), nullable=True),
sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=False),
sa.Column('modified', sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(['image_id'], ['images.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_reference_images_created_at'), 'reference_images', ['created_at'], unique=False)
op.create_index(op.f('ix_reference_images_filter'), 'reference_images', ['filter'], unique=False)
op.create_index(op.f('ix_reference_images_id'), 'reference_images', ['id'], unique=False)
op.create_index(op.f('ix_reference_images_image_id'), 'reference_images', ['image_id'], unique=False)
op.create_index(op.f('ix_reference_images_section_id'), 'reference_images', ['section_id'], unique=False)
op.create_index(op.f('ix_reference_images_target'), 'reference_images', ['target'], unique=False)
op.create_index(op.f('ix_reference_images_validity_end'), 'reference_images', ['validity_end'], unique=False)
op.create_index(op.f('ix_reference_images_validity_start'), 'reference_images', ['validity_start'], unique=False)
op.add_column('cutouts', sa.Column('source_list_id', sa.BigInteger(), nullable=False))
op.add_column('cutouts', sa.Column('new_image_id', sa.BigInteger(), nullable=False))
op.add_column('cutouts', sa.Column('ref_image_id', sa.BigInteger(), nullable=False))
op.add_column('cutouts', sa.Column('sub_image_id', sa.BigInteger(), nullable=False))
op.add_column('cutouts', sa.Column('pixel_x', sa.Integer(), nullable=False))
op.add_column('cutouts', sa.Column('pixel_y', sa.Integer(), nullable=False))
op.add_column('cutouts', sa.Column('provenance_id', sa.BigInteger(), nullable=False))
op.add_column('cutouts', sa.Column('filepath', sa.Text(), nullable=False))
op.add_column('cutouts', sa.Column('filepath_extensions', sa.ARRAY(sa.Text()), nullable=True))
op.add_column('cutouts', sa.Column('format', sa.Enum('fits', 'hdf5', name='image_format'), nullable=False))
op.add_column('cutouts', sa.Column('ra', sa.Double(), nullable=False))
op.add_column('cutouts', sa.Column('dec', sa.Double(), nullable=False))
op.add_column('cutouts', sa.Column('gallat', sa.Double(), nullable=True))
op.add_column('cutouts', sa.Column('gallon', sa.Double(), nullable=True))
op.add_column('cutouts', sa.Column('ecllat', sa.Double(), nullable=True))
op.add_column('cutouts', sa.Column('ecllon', sa.Double(), nullable=True))
op.create_index('cutouts_q3c_ang2ipix_idx', 'cutouts', [sa.text('q3c_ang2ipix(ra, dec)')], unique=False)
op.create_index(op.f('ix_cutouts_ecllat'), 'cutouts', ['ecllat'], unique=False)
op.create_index(op.f('ix_cutouts_filepath'), 'cutouts', ['filepath'], unique=True)
op.create_index(op.f('ix_cutouts_gallat'), 'cutouts', ['gallat'], unique=False)
op.create_index(op.f('ix_cutouts_new_image_id'), 'cutouts', ['new_image_id'], unique=False)
op.create_index(op.f('ix_cutouts_provenance_id'), 'cutouts', ['provenance_id'], unique=False)
op.create_index(op.f('ix_cutouts_ref_image_id'), 'cutouts', ['ref_image_id'], unique=False)
op.create_index(op.f('ix_cutouts_source_list_id'), 'cutouts', ['source_list_id'], unique=False)
op.create_index(op.f('ix_cutouts_sub_image_id'), 'cutouts', ['sub_image_id'], unique=False)
op.create_foreign_key(None, 'cutouts', 'images', ['ref_image_id'], ['id'])
op.create_foreign_key(None, 'cutouts', 'provenances', ['provenance_id'], ['id'], ondelete='CASCADE')
op.create_foreign_key(None, 'cutouts', 'images', ['new_image_id'], ['id'])
op.create_foreign_key(None, 'cutouts', 'images', ['sub_image_id'], ['id'])
op.create_foreign_key(None, 'cutouts', 'source_lists', ['source_list_id'], ['id'])
op.drop_index('exposure_q3c_ang2ipix_idx', table_name='exposures')
op.add_column('images', sa.Column('ref_image_id', sa.BigInteger(), nullable=True))
op.add_column('images', sa.Column('new_image_id', sa.BigInteger(), nullable=True))
op.drop_index('ix_images_combine_method', table_name='images')
op.create_index(op.f('ix_images_new_image_id'), 'images', ['new_image_id'], unique=False)
op.create_index(op.f('ix_images_ref_image_id'), 'images', ['ref_image_id'], unique=False)
op.create_foreign_key(None, 'images', 'images', ['new_image_id'], ['id'], ondelete='CASCADE')
op.create_foreign_key(None, 'images', 'images', ['ref_image_id'], ['id'], ondelete='CASCADE')
op.drop_column('images', 'combine_method')
op.add_column('measurements', sa.Column('cutouts_id', sa.BigInteger(), nullable=False))
op.add_column('measurements', sa.Column('provenance_id', sa.BigInteger(), nullable=False))
op.add_column('measurements', sa.Column('ra', sa.Double(), nullable=False))
op.add_column('measurements', sa.Column('dec', sa.Double(), nullable=False))
op.add_column('measurements', sa.Column('gallat', sa.Double(), nullable=True))
op.add_column('measurements', sa.Column('gallon', sa.Double(), nullable=True))
op.add_column('measurements', sa.Column('ecllat', sa.Double(), nullable=True))
op.add_column('measurements', sa.Column('ecllon', sa.Double(), nullable=True))
op.create_index(op.f('ix_measurements_cutouts_id'), 'measurements', ['cutouts_id'], unique=False)
op.create_index(op.f('ix_measurements_ecllat'), 'measurements', ['ecllat'], unique=False)
op.create_index(op.f('ix_measurements_gallat'), 'measurements', ['gallat'], unique=False)
op.create_index(op.f('ix_measurements_provenance_id'), 'measurements', ['provenance_id'], unique=False)
op.create_index('measurements_q3c_ang2ipix_idx', 'measurements', [sa.text('q3c_ang2ipix(ra, dec)')], unique=False)
op.create_foreign_key(None, 'measurements', 'provenances', ['provenance_id'], ['id'], ondelete='CASCADE')
op.create_foreign_key(None, 'measurements', 'cutouts', ['cutouts_id'], ['id'])
op.add_column('source_lists', sa.Column('image_id', sa.BigInteger(), nullable=False))
op.add_column('source_lists', sa.Column('is_sub', sa.Boolean(), nullable=False))
op.add_column('source_lists', sa.Column('provenance_id', sa.BigInteger(), nullable=False))
op.add_column('source_lists', sa.Column('filepath', sa.Text(), nullable=False))
op.add_column('source_lists', sa.Column('filepath_extensions', sa.ARRAY(sa.Text()), nullable=True))
op.add_column('source_lists', sa.Column('format', sa.Enum('fits', 'hdf5', name='image_format'), nullable=False))
op.create_index(op.f('ix_source_lists_filepath'), 'source_lists', ['filepath'], unique=True)
op.create_index(op.f('ix_source_lists_image_id'), 'source_lists', ['image_id'], unique=False)
op.create_index(op.f('ix_source_lists_provenance_id'), 'source_lists', ['provenance_id'], unique=False)
op.create_foreign_key(None, 'source_lists', 'provenances', ['provenance_id'], ['id'], ondelete='CASCADE')
op.create_foreign_key(None, 'source_lists', 'images', ['image_id'], ['id'])
op.add_column('world_coordinates', sa.Column('source_list_id', sa.BigInteger(), nullable=False))
op.add_column('world_coordinates', sa.Column('provenance_id', sa.BigInteger(), nullable=False))
op.create_index(op.f('ix_world_coordinates_provenance_id'), 'world_coordinates', ['provenance_id'], unique=False)
op.create_index(op.f('ix_world_coordinates_source_list_id'), 'world_coordinates', ['source_list_id'], unique=False)
op.create_foreign_key(None, 'world_coordinates', 'source_lists', ['source_list_id'], ['id'])
op.create_foreign_key(None, 'world_coordinates', 'provenances', ['provenance_id'], ['id'], ondelete='CASCADE')
op.add_column('zero_points', sa.Column('source_list_id', sa.BigInteger(), nullable=False))
op.add_column('zero_points', sa.Column('provenance_id', sa.BigInteger(), nullable=False))
op.create_index(op.f('ix_zero_points_provenance_id'), 'zero_points', ['provenance_id'], unique=False)
op.create_index(op.f('ix_zero_points_source_list_id'), 'zero_points', ['source_list_id'], unique=False)
op.create_foreign_key(None, 'zero_points', 'provenances', ['provenance_id'], ['id'], ondelete='CASCADE')
op.create_foreign_key(None, 'zero_points', 'source_lists', ['source_list_id'], ['id'])
# ### end Alembic commands ###


def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint(None, 'zero_points', type_='foreignkey')
op.drop_constraint(None, 'zero_points', type_='foreignkey')
op.drop_index(op.f('ix_zero_points_source_list_id'), table_name='zero_points')
op.drop_index(op.f('ix_zero_points_provenance_id'), table_name='zero_points')
op.drop_column('zero_points', 'provenance_id')
op.drop_column('zero_points', 'source_list_id')
op.drop_constraint(None, 'world_coordinates', type_='foreignkey')
op.drop_constraint(None, 'world_coordinates', type_='foreignkey')
op.drop_index(op.f('ix_world_coordinates_source_list_id'), table_name='world_coordinates')
op.drop_index(op.f('ix_world_coordinates_provenance_id'), table_name='world_coordinates')
op.drop_column('world_coordinates', 'provenance_id')
op.drop_column('world_coordinates', 'source_list_id')
op.drop_constraint(None, 'source_lists', type_='foreignkey')
op.drop_constraint(None, 'source_lists', type_='foreignkey')
op.drop_index(op.f('ix_source_lists_provenance_id'), table_name='source_lists')
op.drop_index(op.f('ix_source_lists_image_id'), table_name='source_lists')
op.drop_index(op.f('ix_source_lists_filepath'), table_name='source_lists')
op.drop_column('source_lists', 'format')
op.drop_column('source_lists', 'filepath_extensions')
op.drop_column('source_lists', 'filepath')
op.drop_column('source_lists', 'provenance_id')
op.drop_column('source_lists', 'is_sub')
op.drop_column('source_lists', 'image_id')
op.drop_constraint(None, 'measurements', type_='foreignkey')
op.drop_constraint(None, 'measurements', type_='foreignkey')
op.drop_index('measurements_q3c_ang2ipix_idx', table_name='measurements')
op.drop_index(op.f('ix_measurements_provenance_id'), table_name='measurements')
op.drop_index(op.f('ix_measurements_gallat'), table_name='measurements')
op.drop_index(op.f('ix_measurements_ecllat'), table_name='measurements')
op.drop_index(op.f('ix_measurements_cutouts_id'), table_name='measurements')
op.drop_column('measurements', 'ecllon')
op.drop_column('measurements', 'ecllat')
op.drop_column('measurements', 'gallon')
op.drop_column('measurements', 'gallat')
op.drop_column('measurements', 'dec')
op.drop_column('measurements', 'ra')
op.drop_column('measurements', 'provenance_id')
op.drop_column('measurements', 'cutouts_id')
op.add_column('images', sa.Column('combine_method', postgresql.ENUM('coadd', 'subtraction', name='image_combine_method'), autoincrement=False, nullable=True))
op.drop_constraint(None, 'images', type_='foreignkey')
op.drop_constraint(None, 'images', type_='foreignkey')
op.drop_index(op.f('ix_images_ref_image_id'), table_name='images')
op.drop_index(op.f('ix_images_new_image_id'), table_name='images')
op.create_index('ix_images_combine_method', 'images', ['combine_method'], unique=False)
op.drop_column('images', 'new_image_id')
op.drop_column('images', 'ref_image_id')
op.create_index('exposure_q3c_ang2ipix_idx', 'exposures', [sa.text('q3c_ang2ipix(ra, "dec")')], unique=False)
op.drop_constraint(None, 'cutouts', type_='foreignkey')
op.drop_constraint(None, 'cutouts', type_='foreignkey')
op.drop_constraint(None, 'cutouts', type_='foreignkey')
op.drop_constraint(None, 'cutouts', type_='foreignkey')
op.drop_constraint(None, 'cutouts', type_='foreignkey')
op.drop_index(op.f('ix_cutouts_sub_image_id'), table_name='cutouts')
op.drop_index(op.f('ix_cutouts_source_list_id'), table_name='cutouts')
op.drop_index(op.f('ix_cutouts_ref_image_id'), table_name='cutouts')
op.drop_index(op.f('ix_cutouts_provenance_id'), table_name='cutouts')
op.drop_index(op.f('ix_cutouts_new_image_id'), table_name='cutouts')
op.drop_index(op.f('ix_cutouts_gallat'), table_name='cutouts')
op.drop_index(op.f('ix_cutouts_filepath'), table_name='cutouts')
op.drop_index(op.f('ix_cutouts_ecllat'), table_name='cutouts')
op.drop_index('cutouts_q3c_ang2ipix_idx', table_name='cutouts')
op.drop_column('cutouts', 'ecllon')
op.drop_column('cutouts', 'ecllat')
op.drop_column('cutouts', 'gallon')
op.drop_column('cutouts', 'gallat')
op.drop_column('cutouts', 'dec')
op.drop_column('cutouts', 'ra')
op.drop_column('cutouts', 'format')
op.drop_column('cutouts', 'filepath_extensions')
op.drop_column('cutouts', 'filepath')
op.drop_column('cutouts', 'provenance_id')
op.drop_column('cutouts', 'pixel_y')
op.drop_column('cutouts', 'pixel_x')
op.drop_column('cutouts', 'sub_image_id')
op.drop_column('cutouts', 'ref_image_id')
op.drop_column('cutouts', 'new_image_id')
op.drop_column('cutouts', 'source_list_id')
op.drop_index(op.f('ix_reference_images_validity_start'), table_name='reference_images')
op.drop_index(op.f('ix_reference_images_validity_end'), table_name='reference_images')
op.drop_index(op.f('ix_reference_images_target'), table_name='reference_images')
op.drop_index(op.f('ix_reference_images_section_id'), table_name='reference_images')
op.drop_index(op.f('ix_reference_images_image_id'), table_name='reference_images')
op.drop_index(op.f('ix_reference_images_id'), table_name='reference_images')
op.drop_index(op.f('ix_reference_images_filter'), table_name='reference_images')
op.drop_index(op.f('ix_reference_images_created_at'), table_name='reference_images')
op.drop_table('reference_images')
# ### end Alembic commands ###
5 changes: 3 additions & 2 deletions default_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ storage:
# so you can use e.g., {ra_int:03d} to get a 3 digit zero padded right ascension.
# The name convention can also include subfolders (e.g., using {ra_int}/...).
# The minimal set of fields to make the filenames unique include:
# short_name (instrument name), date, time, section_id, prov_id (the unique provenance ID)
name_convention: "{ra_int:03d}/{short_name}_{date}_{time}_{section_id:02d}_{filter}_{prov_id:03d}"
# short_name (instrument name), date, time, section_id, prov_hash
# (in this example, the first six characters of the provenance unique hash)
name_convention: "{ra_int:03d}/{short_name}_{date}_{time}_{section_id}_{filter}_{prov_hash:.6s}"

Loading

0 comments on commit 4844eab

Please sign in to comment.