iteration is soooo slooooow

models/decam.py

@@ -654,9 +654,10 @@ def _commit_exposure( self, origin_identifier, expfile, obs_type='Sci', proc_typ
                   .filter( Exposure.instrument == 'DECam' )
                   .filter( Exposure.origin_identifier == origin_identifier )
                  )
+            if q.count() > 1:
+                raise RuntimeError( f"Database error: got more than one Exposure "
+                                    f"with origin_identifier {origin_identifier}" )
             existing = q.first()
-            # Maybe check that q.count() isn't >1; if it is, throw an exception
-            #  about database corruption?
             if existing is not None:
                 raise FileExistsError( f"Exposure with origin identifier {origin_identifier} "
                                        f"already exists in the database. ({existing.filepath})" )

models/provenance.py

@@ -357,24 +357,6 @@ def merge_concurrent(self, session=None, commit=True):
         If that happens, we simply begin again, checking for the provenance and merging it.
         """
         return models.base.merge_concurrent( self, session=session, commit=commit )
-        # output = None
-        # with SmartSession(session) as session:
-        #     for i in range(5):
-        #         try:
-        #             output = session.merge(self)
-        #             if commit:
-        #                 session.commit()
-        #             break
-        #         except IntegrityError as e:
-        #             if 'duplicate key value violates unique constraint "pk_provenances"' in str(e):
-        #                 session.rollback()
-        #                 time.sleep(0.1 * 2 ** i)  # exponential sleep
-        #             else:
-        #                 raise e
-        #     else:  # if we didn't break out of the loop, there must have been some integrity error
-        #         raise e
-
-        # return output
 
 
 @event.listens_for(Provenance, "before_insert")

pipeline/pipeline_exposure_launcher.py

@@ -47,7 +47,7 @@ def __init__( self, instrument, identifier, params, numprocs, onlychips=None,
 
         numprocs: int
           Number of worker processes (not including the manager process)
-          to run at once.  0 = do all work in the main manager process.
+          to run at once.  0 or 1 = do all work in the main manager process.
 
         onlychips : list, default None
           If not None, will only process the sensor sections whose names

pipeline/top_level.py

@@ -239,8 +239,6 @@ def setup_datastore(self, *args, **kwargs):
                     )
                 ).all()
                 report.num_prev_reports = len(prev_rep)
-                # report = dbsession.merge(report)
-                # dbsession.commit()
                 report = merge_concurrent( report, dbsession, True )
 
             if report.exposure_id is None:

tests/fixtures/ptf.py

@@ -442,7 +442,7 @@ def ptf_aligned_images(request, ptf_cache_dir, data_dir, code_version):
 
             with SmartSession() as session:
                 expsrs = session.query( Exposure ).filter(
-                    Exposure.filepath.in_( [ i.exposure.filepath for i in images ] ) ).all()
+                    Exposure.filepath.in_( [ i.exposure.filepath for i in ptf_reference_images ] ) ).all()
             for expsr in expsrs:
                 expsr.delete_from_disk_and_database( commit=True, remove_downstreams=True )
 

tests/pipeline/test_extraction.py

@@ -292,10 +292,7 @@ def test_extract_sources_sextractor( decam_datastore, extractor, provenance_base
     assert sources.apfluxadu()[0].std() == pytest.approx( 90700, rel=0.01 )
 
     assert sources.good.sum() == pytest.approx(530, rel=0.01)
-    # This value is what you get using the SPREAD_MODEL parameter
-    # assert sources.is_star.sum() == ???
-    # assert ( sources.good & sources.is_star ).sum() == ???
-    # This is what you get with CLASS_STAR
+    # This is what you get with CLASS_STAR; you'll get different values with SPREAD_MODEL
     assert sources.is_star.sum() == pytest.approx(43, rel=0.01)
     assert ( sources.good & sources.is_star ).sum() == pytest.approx(15, abs=5)