From 3a5c475114f7af0facc2315fe00b28ffd40c65e3 Mon Sep 17 00:00:00 2001 From: Rob Knop Date: Thu, 18 Jul 2024 14:30:16 -0700 Subject: [PATCH] iteration is soooo slooooow --- models/decam.py | 5 +++-- models/provenance.py | 18 ------------------ pipeline/pipeline_exposure_launcher.py | 2 +- pipeline/top_level.py | 2 -- tests/fixtures/ptf.py | 2 +- tests/pipeline/test_extraction.py | 5 +---- 6 files changed, 6 insertions(+), 28 deletions(-) diff --git a/models/decam.py b/models/decam.py index a5bdc4c4..e7ae27c3 100644 --- a/models/decam.py +++ b/models/decam.py @@ -654,9 +654,10 @@ def _commit_exposure( self, origin_identifier, expfile, obs_type='Sci', proc_typ .filter( Exposure.instrument == 'DECam' ) .filter( Exposure.origin_identifier == origin_identifier ) ) + if q.count() > 1: + raise RuntimeError( f"Database error: got more than one Exposure " + f"with origin_identifier {origin_identifier}" ) existing = q.first() - # Maybe check that q.count() isn't >1; if it is, throw an exception - # about database corruption? if existing is not None: raise FileExistsError( f"Exposure with origin identifier {origin_identifier} " f"already exists in the database. ({existing.filepath})" ) diff --git a/models/provenance.py b/models/provenance.py index 930bda70..13e24843 100644 --- a/models/provenance.py +++ b/models/provenance.py @@ -357,24 +357,6 @@ def merge_concurrent(self, session=None, commit=True): If that happens, we simply begin again, checking for the provenance and merging it. """ return models.base.merge_concurrent( self, session=session, commit=commit ) - # output = None - # with SmartSession(session) as session: - # for i in range(5): - # try: - # output = session.merge(self) - # if commit: - # session.commit() - # break - # except IntegrityError as e: - # if 'duplicate key value violates unique constraint "pk_provenances"' in str(e): - # session.rollback() - # time.sleep(0.1 * 2 ** i) # exponential sleep - # else: - # raise e - # else: # if we didn't break out of the loop, there must have been some integrity error - # raise e - - # return output @event.listens_for(Provenance, "before_insert") diff --git a/pipeline/pipeline_exposure_launcher.py b/pipeline/pipeline_exposure_launcher.py index dcd0ba99..9adb2cb8 100644 --- a/pipeline/pipeline_exposure_launcher.py +++ b/pipeline/pipeline_exposure_launcher.py @@ -47,7 +47,7 @@ def __init__( self, instrument, identifier, params, numprocs, onlychips=None, numprocs: int Number of worker processes (not including the manager process) - to run at once. 0 = do all work in the main manager process. + to run at once. 0 or 1 = do all work in the main manager process. onlychips : list, default None If not None, will only process the sensor sections whose names diff --git a/pipeline/top_level.py b/pipeline/top_level.py index 6667cda9..dfa9ddf6 100644 --- a/pipeline/top_level.py +++ b/pipeline/top_level.py @@ -239,8 +239,6 @@ def setup_datastore(self, *args, **kwargs): ) ).all() report.num_prev_reports = len(prev_rep) - # report = dbsession.merge(report) - # dbsession.commit() report = merge_concurrent( report, dbsession, True ) if report.exposure_id is None: diff --git a/tests/fixtures/ptf.py b/tests/fixtures/ptf.py index fd951866..b4d03b84 100644 --- a/tests/fixtures/ptf.py +++ b/tests/fixtures/ptf.py @@ -442,7 +442,7 @@ def ptf_aligned_images(request, ptf_cache_dir, data_dir, code_version): with SmartSession() as session: expsrs = session.query( Exposure ).filter( - Exposure.filepath.in_( [ i.exposure.filepath for i in images ] ) ).all() + Exposure.filepath.in_( [ i.exposure.filepath for i in ptf_reference_images ] ) ).all() for expsr in expsrs: expsr.delete_from_disk_and_database( commit=True, remove_downstreams=True ) diff --git a/tests/pipeline/test_extraction.py b/tests/pipeline/test_extraction.py index 0337e128..bd99cb2f 100644 --- a/tests/pipeline/test_extraction.py +++ b/tests/pipeline/test_extraction.py @@ -292,10 +292,7 @@ def test_extract_sources_sextractor( decam_datastore, extractor, provenance_base assert sources.apfluxadu()[0].std() == pytest.approx( 90700, rel=0.01 ) assert sources.good.sum() == pytest.approx(530, rel=0.01) - # This value is what you get using the SPREAD_MODEL parameter - # assert sources.is_star.sum() == ??? - # assert ( sources.good & sources.is_star ).sum() == ??? - # This is what you get with CLASS_STAR + # This is what you get with CLASS_STAR; you'll get different values with SPREAD_MODEL assert sources.is_star.sum() == pytest.approx(43, rel=0.01) assert ( sources.good & sources.is_star ).sum() == pytest.approx(15, abs=5)