diff --git a/models/base.py b/models/base.py index 1ddec466..de0d5586 100644 --- a/models/base.py +++ b/models/base.py @@ -1957,7 +1957,7 @@ def _find_possibly_containing_temptable( cls, ra, dec, session, prov_id=None ): " ( maxra >= :ra AND minra <= :ra ) )" " OR " " ( ( maxra < minra ) AND " - " ( ( maxra >= :ra OR :ra > 180. ) AND ( minra <= :ra OR ra <= 180. ) ) )" + " ( ( maxra >= :ra OR :ra > 180. ) AND ( minra <= :ra OR :ra <= 180. ) ) )" " )" ")" ) diff --git a/models/reference.py b/models/reference.py index 0ee35cc5..82930e8f 100644 --- a/models/reference.py +++ b/models/reference.py @@ -1,4 +1,4 @@ -import shapely.geometry +from uuid import UUID import sqlalchemy as sa from sqlalchemy import orm @@ -163,6 +163,12 @@ def get_references( cls, ra=None, dec=None, + minra=None, + maxra=None, + mindec=None, + maxdec=None, + image=None, + overlapfrac=None, target=None, section_id=None, instrument=None, @@ -174,15 +180,42 @@ def get_references( """Find all references in the specified part of the sky, with the given filter. Can also match specific provenances and will (by default) not return bad references. + Operates in three modes: + + * References tagged for a given target and sectionid + + * References that include a given point on the sky. Specify ra and dec, do not + pass any of minra, maxra, mindec, maxdec, or target. + + * References that overlap an area on the sky. Specify either + minra/maxra/mindec/maxdec or image. If overlapfrac is None, + will return references that overlap the area at all; this is + usually not what you want. + Parameters ---------- - ra: float or string, optional - Right ascension in degrees, or a hexagesimal string (in hours!). - If given, must also give the declination. + ra: float, optional + Right ascension in degrees. If given, must also give the declination. + + dec: float, optional + Declination in degrees. If given, must also give the right ascension. - dec: float or string, optional - Declination in degrees, or a hexagesimal string (in degrees). - If given, must also give the right ascension. + minra, maxra, mindec, maxdec: float, optional + Rectangle on the sky, in degrees. Will find references whose + bounding rectangle overlaps this rectangle on the sky. + minra is the W edge, maxra is the E edge, so if the center of + a 2° wide retange is at RA=0, then minra=359 and maxra=1. + + image: Image, optional + If specified, minra/maxra/mindec/maxdec will be pulled from + this Image (or any other type of object that inherits from + FourCorners). + + overlapfrac: float, default None + If minra/maxra/mindec/maxdec or image is not None, then only + return references whose bounding rectangle overlaps the + passed bounding rectangle by at least this much. Ignored if + ra/dec or target/section_id is specified. target: string, optional Name of the target object or field id. Will only match @@ -219,79 +252,200 @@ def get_references( list of Reference, list of Image """ - if ( ( ( ra is None ) or ( dec is None ) ) and - ( ( target is None ) or ( section_id is None ) ) + + radecgiven = ( ra is not None ) or ( dec is not None ) + areagiven = ( image is not None ) or any( i is not None for i in [ minra, maxra, mindec, maxdec ] ) + targetgiven = ( target is not None ) or ( section_id is not None ) + if ( ( radecgiven and ( areagiven or targetgiven ) ) or + ( areagiven and ( radecgiven or targetgiven ) ) or + ( targetgiven and ( radecgiven or areagiven ) ) ): - raise ValueError( "Must provide at least ra/dec or target/section_id" ) - - if ( ra is None ) != ( dec is None ): - raise ValueError( "Must provide both or neither of ra/dec" ) - - if ra is None: - stmt = ( sa.select( Reference, Image ) - .where( Reference.target == target ) - .where( Reference.section_id == section_id ) - ) - else: - # Not using FourCorners.containing here, because - # that doesn't actually use the q3c indices, - # so will be slow. minra, maxra, mindec, maxdec - # have classic indices, so this is a good first pass. - # Below, we'll crop the list down. - stmt = ( sa.select( Reference, Image ) - .where( Image._id==Reference.image_id ) - .where( Image.minra<=ra ) - .where( Image.maxra>=ra ) - .where( Image.mindec<=dec ) - .where( Image.maxdec>=dec ) - ) - if target is not None: - stmt = stmt.where( Reference.target==target ) - if section_id is not None: - stmt = stmt.where( Reference.section_id==str(section_id) ) - - if instrument is not None: - stmt = stmt.where( Reference.instrument==instrument ) - - if filter is not None: - stmt = stmt.where( Reference.filter==filter ) - - if skip_bad: - stmt = stmt.where( Reference.is_bad.is_( False ) ) - - provenance_ids = listify(provenance_ids) - if provenance_ids is not None: - for i, prov in enumerate(provenance_ids): - if isinstance(prov, Provenance): - provenance_ids[i] = prov.id - elif not isinstance(prov, str): - raise ValueError(f"Provenance ID must be a string or a Provenance object, not {type(prov)}.") - - stmt = stmt.where( Reference.provenance_id.in_(provenance_ids) ) + raise ValueError( "Specify only one of ( target/section_id, ra/dec, minra/maxra/mindec/maxdec or image )" ) + + fcobj = None with SmartSession( session ) as sess: - refs = sess.execute( stmt ).all() - imgs = [ r[1] for r in refs ] - refs = [ r[0] for r in refs ] - - if ra is not None: - # Have to crop down the things found to things that actually include - # the ra/dec - croprefs = [] - cropimgs = [] - for ref, img in zip( refs, imgs ): - poly = shapely.geometry.Polygon( [ ( img.ra_corner_00, img.dec_corner_00 ), - ( img.ra_corner_01, img.dec_corner_01 ), - ( img.ra_corner_11, img.dec_corner_11 ), - ( img.ra_corner_10, img.dec_corner_10 ), - ( img.ra_corner_00, img.dec_corner_00 ) ] ) - if poly.contains( shapely.geometry.Point( ra, dec ) ): - croprefs.append( ref ) - cropimgs.append( img ) - refs = croprefs - imgs = cropimgs - - return refs, imgs + # Mode 1 : target / section_id + + if ( ( target is not None ) or ( section_id is not None ) ): + if ( target is None ) or (section_id is None ): + raise ValueError( "Must give both target and section_id" ) + if overlapfrac is not None: + raise ValueError( "Can't give overlapfrac with target/section_id" ) + + q = "SELECT r.* FROM refs r WHERE target=:target AND section_id=:section_id " + subdict = { 'target': target, 'section_id': section_id } + + # Mode 2 : ra/dec + + elif ( ( ra is not None ) or ( dec is not None ) ): + if ( ra is None ) or ( dec is None ): + raise ValueError( "Must give both ra and dec" ) + if overlapfrac is not None: + raise ValueError( "Can't give overlapfrac with ra/dec" ) + + # Bobby Tables + ra = float(ra) if isinstance( ra, int ) else ra + dec = float(dec) if isinstance( dec, int ) else dec + if ( not isinstance( ra, float ) ) or ( not isinstance( dec, float ) ): + raise TypeError( f"(ra, dec) must be floats, got ({type(ra)}, {type(dec)})" ) + + # This code is kind of redundant with the code in + # FourCorners._find_possibly_containing_temptable and + # FourCorners.find_containing, but we can't just use + # that because Reference isn't a FourCorners, and we + # have to join Reference to Image + + q = ( "SELECT r.*, i.ra_corner_00, i.ra_corner_01, i.ra_corner_10, i.ra_corner_11, " + " i.dec_corner_00, i.dec_corner_01, i.dec_corner_10, i.dec_corner_11 " + " INTO TEMP TABLE temp_find_containing_ref " + " FROM refs r INNER JOIN images i ON r.image_id=i._id " + "WHERE ( " + " ( i.maxdec >= :dec AND i.mindec <= :dec ) " + " AND ( " + " ( ( i.maxra > i.minra ) AND " + " ( i.maxra >= :ra AND i.minra <= :ra ) )" + " OR " + " ( ( i.maxra < i.minra ) AND " + " ( ( i.maxra >= :ra OR :ra > 180. ) AND ( i.minra <= :ra OR :ra <= 180. ) ) )" + " )" + ")" + ) + subdict = { "ra": ra, "dec": dec } + sess.execute( sa.text(q), subdict ) + + q = ( "SELECT r.* FROM refs r INNER JOIN temp_find_containing_ref t ON r._id=t._id " + f"WHERE q3c_poly_query( :ra, :dec, ARRAY[ t.ra_corner_00, t.dec_corner_00, " + " t.ra_corner_01, t.dec_corner_01, " + " t.ra_corner_11, t.dec_corner_11, " + " t.ra_corner_10, t.dec_corner_10 ] ) " ) + + # Mode 3 : overlapping area + + elif ( image is not None ) or any( i is not None for i in [ minra, maxra, mindec, maxdec ] ): + if image is not None: + if any( i is not None for i in [ minra, maxra, mindec, maxdec ] ): + raise ValueError( "Specify either image or minra/maxra/mindec/maxdec, not both" ) + minra = image.minra + maxra = image.maxra + mindec = image.mindec + maxdec = image.maxdec + fcobj = image + else: + if any( i is None for i in [ minra, maxra, mindec, maxdec ] ): + raise ValueError( "Must give all of minra, maxra, mindec, maxdec" ) + fcobj = FourCorners() + fcobj.ra = (minra + maxra) / 2. + fcobj.dec = (mindec + maxdec) / 2. + fcobj.ra_corner_00 = minra + fcobj.ra_corner_01 = minra + fcobj.minra = minra + fcobj.ra_corner_10 = maxra + fcobj.ra_corner_11 = maxra + fcobj.maxra = maxra + fcobj.dec_corner_00 = mindec + fcobj.dec_corner_10 = mindec + fcobj.mindec = mindec + fcobj.dec_corner_01 = maxdec + fcobj.dec_corner_11 = maxdec + fcobj.maxdec = maxdec + + # Sort of redundant code from FourCorners._find_potential_overlapping_temptable, + # but we can't just use that because Reference isn't a FourCorners and + # we have to do the refs/images join. + + q = ( "SELECT r.* FROM refs r INNER JOIN images i ON r.image_id=i._id " + "WHERE ( " + " ( i.maxdec >= :mindec AND i.mindec <= :maxdec ) " + " AND " + " ( ( ( i.maxra >= i.minra AND :maxra >= :minra ) AND " + " i.maxra >= :minra AND i.minra <= :maxra ) " + " OR " + " ( i.maxra < i.minra AND :maxra < :minra ) " # both include RA=0, will overlap in RA + " OR " + " ( ( i.maxra < i.minra AND :maxra >= :minra AND :minra <= 180. ) AND " + " i.maxra >= :minra ) " + " OR " + " ( ( i.maxra < i.minra AND :maxra >= :minra AND :minra > 180. ) AND " + " i.minra <= :maxra ) " + " OR " + " ( ( i.maxra >= i.minra AND :maxra < :minra AND i.maxra <= 180. ) AND " + " i.minra <= :maxra ) " + " OR " + " ( ( i.maxra >= i.minra AND :maxra < :minra AND i.maxra > 180. ) AND " + " i.maxra >= :minra ) " + " )" + ") " ) + subdict = { 'minra': minra, 'maxra': maxra, 'mindec': mindec, 'maxdec': maxdec } + + else: + raise ValueError( "Must give one of target/section_id, ra/dec, or minra/maxra/mindec/maxdec or image" ) + + # Additional criteria + + if provenance_ids is not None: + if isinstance( prov_id, str ) or isinstance( prov_id, UUID ): + q += " AND r.provenance_id=:prov" + subdict['prov'] = provenance_ids + elif isinstance( prov_id, Provenance ): + q += " AND r.provenance_id=:prov" + subdict['prov'] = provenance_ids.id + elif isinstance( prov_id, list ): + q += " AND r.provenance_id IN :provs" + subdict['provs'] = [] + for pid in provenance_ids: + subdict['provs'].append( pid if isinstance( pid, str ) or isinstance( pid, UUID ) + else pid.id ) + subdict['provs'] = tuple( subdict['provs'] ) + + if instrument is not None: + q += " AND r.instrument=:instrument " + subdict['instrument'] = instrument + + if filter is not None: + q += " AND r.filter=:filter " + subdict['filter'] = filter + + if skip_bad: + q += " AND NOT r.is_bad " + + # Get the Reference objects + + references = list( sess.scalars( sa.select( Reference ) + .from_statement( sa.text(q).bindparams(**subdict) ) + ).all() ) + + # Get the image objects + + imset = set( sess.scalars( sa.select( Image ) + .where( Image._id.in_( r.image_id for r in references ) ) + ).all() ) + + # Make sure they're sorted right + + if len( references ) != len( imset ): + raise RuntimeError( "Database corruption: number of returned references and images didn't match!" ) + if not all( r.image_id in imset for r in references ): + raise RuntimeError( "Didn't get back the images expected; this should not happen!" ) + imdict = { i._id : i for i in imset } + images = [ imdict[r.image_id] for r in references ] + + # Deal with overlapfrac if relevant + + if overlapfrac is not None: + retref = [] + retim = [] + for r, i in zip( references, images ): + if FourCorners.get_overlap_frac( fcobj, i ) >= overlapfrac: + retref.append( r ) + retim.append( i ) + references = retref + images = rtim + + # Done! + + return references, images + # ====================================================================== # The fields below are things that we've deprecated; these definitions diff --git a/pipeline/ref_maker.py b/pipeline/ref_maker.py index eb3ebf2f..1657acba 100644 --- a/pipeline/ref_maker.py +++ b/pipeline/ref_maker.py @@ -96,6 +96,16 @@ def __init__(self, **kwargs): critical=True, ) + self.coadd_overlap_fraction = self.add_par( + 'coadd_overlap_fraction', + 0.1, + (None, float), + ( "When looking for images to coadd into a new reference, only consider images whose " + "min/max ra/dec overlap the sky rectangle of the target by at least this much. " + "Ignored when corner_distance is None." ), + critical=True, + ) + self.instruments = self.add_par( 'instruments', None, @@ -147,7 +157,18 @@ def __init__(self, **kwargs): 1, int, ( 'Construct a reference only if there are at least this many images that pass all other criteria ' - ' ' ), + 'If corner_distance is not None, then this applies to all test positions on the image unless ' + 'min_only_center is True.' ), + critical=True, + ) + + self.min_only_center = self.add_par( + 'min_only_center', + False, + bool, + ( 'If True, then min_number only applies to the center position of the target area. Otherwise, ' + 'every test position on the image must have at least min_number references for the construction ' + 'not to fail. Ignored if corner_distance is None.' ), critical=True, ) diff --git a/tests/conftest.py b/tests/conftest.py index bb6423f5..dd466265 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,8 +37,8 @@ # at the end of tests. In general, we want this to be True, so we can make sure # that our tests are properly cleaning up after themselves. However, the errors # from this can hide other errors and failures, so when debugging, set it to False. -verify_archive_database_empty = True -# verify_archive_database_empty = False +# verify_archive_database_empty = True +verify_archive_database_empty = False pytest_plugins = [ diff --git a/tests/pipeline/test_making_references.py b/tests/pipeline/test_making_references.py index 3573e1e0..525b0d64 100644 --- a/tests/pipeline/test_making_references.py +++ b/tests/pipeline/test_making_references.py @@ -9,13 +9,14 @@ from pipeline.ref_maker import RefMaker -from models.base import SmartSession +from models.base import SmartSession, FourCorners from models.provenance import Provenance from models.image import Image from models.reference import Reference from models.refset import RefSet from util.util import env_as_bool +from util.logger import SCLogger def add_test_parameters(maker): @@ -31,41 +32,270 @@ def add_test_parameters(maker): obj.pars._enforce_no_new_attrs = True -def test_finding_references(ptf_ref): - with pytest.raises(ValueError, match='Must provide at least ra/dec or target/section_id'): - ref, img = Reference.get_references(ra=188) - with pytest.raises(ValueError, match='Must provide at least ra/dec or target/section_id'): - ref, img = Reference.get_references(dec=4.5) - with pytest.raises(ValueError, match='Must provide at least ra/dec or target/section_id'): - ref, img = Reference.get_references(target='foo') - with pytest.raises(ValueError, match='Must provide at least ra/dec or target/section_id'): - ref, img = Reference.get_references(section_id='bar') - with pytest.raises(ValueError, match='Must provide at least ra/dec or target/section_id'): - ref,img = Reference.get_references(ra=188, section_id='bar') - with pytest.raises(ValueError, match='Must provide at least ra/dec or target/section_id'): - ref, img = Reference.get_references(dec=4.5, target='foo') - with pytest.raises(ValueError, match='Must provide at least ra/dec or target/section_id'): - ref, img = Reference.get_references() - - ref, img = Reference.get_references(ra=188, dec=4.5) - assert len(ref) == 1 - assert ref[0].id == ptf_ref.id - - ref, img = Reference.get_references(ra=188, dec=4.5, provenance_ids=ptf_ref.provenance_id) - assert len(ref) == 1 - assert ref[0].id == ptf_ref.id - - ref, img = Reference.get_references(ra=0, dec=0) - assert len(ref) == 0 - - ref, img = Reference.get_references(target='foo', section_id='bar') - assert len(ref) == 0 - - ref, img = Reference.get_references(ra=180, dec=4.5, provenance_ids=['foo', 'bar']) - assert len(ref) == 0 - - # TODO : test target/section filter on ra/dec search, test - # instrument and filter filters, test provenance_ids, test skip_bad +def test_finding_references( provenance_base, provenance_extra ): + refstodel = set() + imgstodel = set() + + try: + # Create ourselves some fake images and references to use as test fodder + + reuseimgkw = { 'provenance_id': provenance_base.id, + 'mjd': 60000., + 'end_mjd': 60000.000694, + 'exp_time': 60., + 'fwhm_estimate': 1.1, + 'zero_point_estimate': 24., + 'bkg_mean_estimate': 0., + 'bkg_rms_estimate': 1., + 'md5sum': uuid.uuid4(), + 'format': 'fits', + 'telescope': 'testscope', + 'instrument': 'DECam', + 'project': 'Mercator' + } + + # Something somewhere, 0.2° on a side, in r and g, and one with a different provenance + img1 = Image( ra=20., dec=45., + minra=19.8586, maxra=20.1414, mindec=44.9, maxdec=45.1, + ra_corner_00=19.8586, ra_corner_01=19.8586, ra_corner_10=20.1414, ra_corner_11=20.1414, + dec_corner_00=44.9, dec_corner_10=44.9, dec_corner_01=45.1, dec_corner_11=44.1, + target='target1', section_id='1', filter='r', filepath='testimage1.fits', **reuseimgkw ) + img1.calculate_coordinates() + img1.insert() + imgstodel.add( img1.id ) + ref1 = Reference( provenance_id=provenance_base.id, image_id=img1.id, target=img1.target, + filter=img1.filter, section_id=img1.section_id, instrument=img1.instrument ) + ref1.insert() + refstodel.add( ref1.id ) + + img2 = Image( ra=20., dec=45., + minra=19.8586, maxra=20.1414, mindec=44.9, maxdec=45.1, + ra_corner_00=19.8586, ra_corner_01=19.8586, ra_corner_10=20.1414, ra_corner_11=20.1414, + dec_corner_00=44.9, dec_corner_10=44.9, dec_corner_01=45.1, dec_corner_11=44.1, + target='target1', section_id='1', filter='g', filepath='testimage2.fits', **reuseimgkw ) + img2.calculate_coordinates() + img2.insert() + imgstodel.add( img2.id ) + ref2 = Reference( provenance_id=provenance_base.id, image_id=img2.id, target=img2.target, + filter=img2.filter, section_id=img2.section_id, instrument=img2.instrument ) + ref2.insert() + refstodel.add( ref2.id ) + + refp = Reference( provenance_id=provenance_extra.id, image_id=img1.id, target=img1.target, + filter=img1.filter, section_id=img1.section_id, instrument=img1.instrument ) + refp.insert() + refstodel.add( refp.id ) + + # Offset by 0.15° in both ra and dec + img3 = Image( ra=20.2121, dec=45.15, + minra=20.0707, maxra=20.3536, mindec=45.05, maxdec=45.25, + ra_corner_00=20.0707, ra_corner_01=20.0707, ra_corner_10=20.3536, ra_corner_11=20.3536, + dec_corner_00=45.05, dec_corner_10=45.05, dec_corner_01=45.25, dec_corner_11=4525, + target='target2', section_id='1', filter='r', filepath='testimage3.fits', **reuseimgkw ) + img3.calculate_coordinates() + img3.insert() + imgstodel.add( img3.id ) + ref3 = Reference( provenance_id=provenance_base.id, image_id=img3.id, target=img3.target, + filter=img3.filter, section_id=img3.section_id, instrument=img3.instrument ) + ref3.insert() + refstodel.add( ref3.id ) + + #Offset, but also rotated by 45° + img4 = Image( ra=20.2121, dec=45.15, + minra=20.0121, maxra=20.4121, mindec=45.0086, maxdec=45.2914, + ra_corner_00=20.0121, ra_corner_01=20.0121, ra_corner_11=20.4121, ra_corner_10=20.4121, + dec_corner_00=45.0086, dec_corner_01=45.0086, dec_corner_11=45.2914, dec_corner_10=44.2914, + target='target2', section_id='1', filter='r', filepath='testimage4.fits', **reuseimgkw ) + img4.calculate_coordinates() + img4.insert() + imgstodel.add( img4.id ) + ref4 = Reference( provenance_id=provenance_base.id, image_id=img4.id, target=img4.target, + filter=img4.filter, section_id=img4.section_id, instrument=img4.instrument ) + ref4.insert() + refstodel.add( ref4.id ) + + # At 0 ra + img5 = Image( ra=0.02, dec=0., + minra=359.92, maxra=0.12, mindec=-0.1, maxdec=0.1, + ra_corner_00=359.92, ra_corner_01=359.92, ra_corner_10=0.12, ra_corner_11=0.12, + dec_corner_00=-0.1, dec_corner_10=-0.1, dec_corner_01=0.1, dec_corner_11=0.1, + target='target3', section_id='1', filter='r', filepath='testimage5.fits', **reuseimgkw ) + img5.calculate_coordinates() + img5.insert() + imgstodel.add( img5.id ) + ref5 = Reference( provenance_id=provenance_base.id, image_id=img5.id, target=img5.target, + filter=img5.filter, section_id=img5.section_id, instrument=img5.instrument ) + ref5.insert() + refstodel.add( ref5.id ) + + # Test bad parameters + with pytest.raises( ValueError, match="Must give one of target.*or image" ): + ref, img = Reference.get_references() + for kws in [ { 'ra': 20., 'minra': 19. }, + { 'ra': 20., 'target': 'foo' }, + { 'minra': 19., 'target': 'foo' }, + { 'image': img5, 'ra': 20. }, + { 'image': img5, 'target': 'foo' } + ]: + with pytest.raises( ValueError, match="Specify only one of" ): + ref, img = Reference.get_references( **kws ) + for kws in [ { 'target': 'foo' }, { 'section_id': '1' } ]: + with pytest.raises( ValueError, match="Must give both target and section_id" ): + ref, img = Reference.get_references( **kws ) + for kws in [ { 'ra': 20.}, { 'dec': 45. } ]: + with pytest.raises( ValueError, match="Must give both ra and dec" ): + ref, img = Reference.get_references( **kws ) + with pytest.raises( ValueError, match="Specify either image or minra/maxra/mindec/maxdec" ): + ref, img = Reference.get_references( image=img5, minra=19. ) + # TODO : write clever for loops to test all possibly combinations of minra/maxra/mindec/maxdec + # that are missing one or more. For now, just test a few + for kws in [ { 'minra': 19.8 }, + { 'maxra': 20.2, 'mindec': 44.9 }, + { 'minra': 19.8, 'mindec': 44.9, 'maxdec': 45.1 } ]: + with pytest.raises( ValueError, match="Must give all of minra, maxra, mindec, maxdec" ): + ref, img = Reference.get_references( **kws ) + with pytest.raises( ValueError, match="Can't give overlapfrac with target/section_id" ): + ref, img = Reference.get_references( target='foo', section_id='1', overlapfrac=0.5 ) + with pytest.raises( ValueError, match="Can't give overlapfrac with ra/dec" ): + ref, img = Reference.get_references( ra=20., dec=45., overlapfrac=0.5 ) + + # Get point at center of img1, all filters, all provenances + import pdb; pdb.set_trace() + refs, imgs = Reference.get_references( ra=20., dec=45. ) + assert len(imgs) == len(refs) + assert all( r.image_id == i.id for r, i in zip( refs, imgs ) ) + assert len(refs) == 3 + assert set( r.id for r in refs ) == set( ref1.id, ref2.id, refp.id ) + assert set( i.id for i in imgs ) == set( img1.id, img2.id ) + + # Get point at center of img1, all filters, only one provenance + for provarg in [ provenance_base.id, provenance_base, [ provenance_base.id ], [ provenance_base ] ]: + refs, imgs = Reference.get_references( ra=20., dec=45., provenance_ids=provarg ) + assert len(imgs) == len(refs) + assert all( r.image_id == i.id for r, i in zip( refs, imgs ) ) + assert len(refs) == 2 + assert set( r.id for r in refs ) == set( ref1.id, ref2.id ) + assert set( i.id for i in imgs ) == set( img1.id, img2.id ) + + # Get point at center of img1, all provenances, only one filter + refs, imgs = Reference.get_references( ra=20., dec=45., filter='r' ) + assert len(imgs) == len(refs) + assert all( r.image_id == i.id for r, i in zip( refs, imgs ) ) + assert len(refs) == 2 + assert set( r.id for r in refs ) == set( ref1.id, refp.id ) + assert set( i.id for i in imgs ) == set( img1.id ) + + # Get point at center of img1, one provenance, one filter + refs, imgs = Reference.get_references( ra=20., dec=45., filter='r', provenance_ids=provenance_base.id ) + assert len(imgs) == len(refs) + assert all( r.image_id == i.id for r, i in zip( refs, imgs ) ) + assert len(refs) == 1 + assert refs[0].id == ref1.id + assert imgs[0].id == img1.id + + refs, imgs = Reference.get_references( ra=20., dec=45., filter='g', provenance_ids=provenance_extra.id ) + assert len(refs) == 0 + assert len(imgs) == 0 + + # TODO : test limiting on other things like instrument, skip_bad + + # For the rest of the tests, we're going to do filter r and provenance provenance_base + kwargs = { 'filter': 'r', 'provenance_ids': provenance_base.id } + + # Get point at upper-left of img1 + refs, imgs = Reference.get_references( ra=20.1273, dec=45.09, **kwargs ) + assert len(imgs) == len(refs) + assert all( r.image_id == i.id for r, i in zip( refs, imgs ) ) + assert len(refs) == 3 + assert set( r.id for r in refs ) == set( ref1.id, ref3.id, ref4.id ) + assert set( i.id for i in imgs ) == set( img1.id, img3.id, img4.id ) + + # Get point included in img3 but not img4 + refs, imgs = Reference.get_references( ra=20.+0.16/numpy.sqrt(2.), dec=45.16, **kwargs ) + assert len(imgs) == len(refs) + assert all( r.image_id == i.id for r, i in zip( refs, imgs ) ) + assert len(refs) == 2 + assert set( r.id for r in refs ) == set( ref1.id, ref3.id ) + assert set( i.id for i in imgs ) == set( img1.id, img3.id ) + + # Get point included in img3 and img4 but not img1 (center of img3) + refs, imgs = References.get_references( ra=img3.ra, dec=img3.dec, **kwargs ) + assert len(imgs) == len(refs) + assert all( r.image_id == i.id for r, i in zip( refs, imgs ) ) + assert len(refs) == 2 + assert set( r.id for r in refs ) == set( ref3.id, ref4.id ) + assert set( i.id for i in imgs ) == set( img3.id, img4.id ) + + # Get points around RA 0 + ras = [ 0., 0.05, 364.95 ] + decs = [ 0., -0.05, 0.05 ] + ramess, decmess = numpy.meshgrid( ras, decs ) + for messdex in range( len(ramess) ): + for ra, dec in zip( ramess[messdex], decmess[messdex] ): + refs, imgs = References.get_references( ra=ra, dec=dec, **kwargs ) + assert len(imgs) == len(refs) + assert all( r.image_id == i.id for r, i in zip( refs, imgs ) ) + assert len( refs ) == 1 + assert refs[0].id == ref5.id + assert imgs[0].id == img5.id + + + # Overlapping -- overlaps img1 at all + refs, imgs = Reference.get_references( image=img1, **kwargs ) + assert len(imgs) == len(refs) + assert all( r.image_id == i.id for r, i in zip( refs, imgs ) ) + assert len(refs) == 3 + assert set( r.id for r in refs ) == set( ref1.id, ref3.id, ref4.id ) + assert set( i.id for i in imgs ) == set( img1.id, img3.id, img4.id ) + + refs, imgs == Reference.get_refrences( minra=img1.minra, maxra=img1.maxra, + mindec=img1.mindec, maxdec=img1.maxdec, + **kwargs ) + assert len(imgs) == len(refs) + assert all( r.image_id == i.id for r, i in zip( refs, imgs ) ) + assert len(refs) == 3 + assert set( r.id for r in refs ) == set( ref1.id, ref3.id, ref4.id ) + assert set( i.id for i in imgs ) == set( img1.id, img3.id, img4.id ) + + # Overlapping -- overlaps img1 by at least x% + # ROB TODO BASED ON CALCULATIONS YOU GET + import pdb; pdb.set_trace() + + # Overlapping -- overlapping around RA 0 + for ctrra, ctrdec in zip( [ 0., 0., 0.08, 0.08, -0.08, -0.08 ], + [ 0., 0.05, 0., 0.05, 0., 0.05 ] ): + refs, imgs == Reference.get_references( minra=ctrra-0.1, maxra=ctrra+0.1, + mindec=ctrdec-0.1, maxdec=ctrdec+0.1, + **kwargs ) + assert len(imgs) == len(refs) + assert all( r.image_id == i.id for r, i in zip( refs, imgs ) ) + assert len(refs) == 1 + assert refs[0].id == ref5.id + assert imgs[0].id == img5.id + # **** + fcobj = FourCorners() + fcobj.ra = ctrra + fcobj.dec = ctrdec + fcobj.ra_corner_00 = ctrra-0.1 + rcobj.ra_corner_01 = ctrra-0.1 + fcobj.minra = ctrra-0.1 + fcobj.ra_corner_10 = ctrra+0.1 + fcobj.ra_corner_11 = ctrra+0.1 + fcobj.maxra = ctrra+0.1 + fcobj.dec_corner_00 = ctrdec-0.1 + fcobj.dec_corner_10 = ctrdec-0.1 + fcobj.mindec = ctrdec-0.1 + fcobj.dec_corner_01 = ctrdec+0.1 + fcobj.dec_corner_11 = ctrdec+0.1 + fcob.maxdec = ctrdec+0.1 + SCLogger.info( f"For ctrra,ctrdec = (ctrra,ctrdec), frac=" + "f{FourCorners.get_voerlap_frac( fcobj, img5 )}" ) + + finally: + # Clean up images and refs we made + with SmartSession() as session: + session.execute( sa.delete( Reference ).where( Reference._id.in_( refstodel ) ) ) + session.execute( sa.delete( Image ).where( Image._id.in_( imgstodel ) ) ) def test_make_refset():