From a47eb3451d62eb4c005ae071c83392641966f46e Mon Sep 17 00:00:00 2001
From: whohensee <106775295+whohensee@users.noreply.github.com>
Date: Fri, 24 May 2024 11:12:39 -0700
Subject: [PATCH 01/11] make WCS a FileOnDiskMixin (#167)

WCS stores a filepath in the database used to save and load
This filepath points to a .txt file on disk with information to create
an astropy wcs object
---
 ...2ea9f6f0b790_make_wcs_a_fileondiskmixin.py |  38 ++++++
 models/world_coordinates.py                   | 120 ++++++++++++------
 tests/fixtures/pipeline_objects.py            |  18 +--
 tests/models/test_image.py                    |   2 +-
 tests/models/test_world_coordinates.py        |  66 +++++++++-
 5 files changed, 188 insertions(+), 56 deletions(-)
 create mode 100644 alembic/versions/2024_05_22_2011-2ea9f6f0b790_make_wcs_a_fileondiskmixin.py

diff --git a/alembic/versions/2024_05_22_2011-2ea9f6f0b790_make_wcs_a_fileondiskmixin.py b/alembic/versions/2024_05_22_2011-2ea9f6f0b790_make_wcs_a_fileondiskmixin.py
new file mode 100644
index 00000000..ea3d5338
--- /dev/null
+++ b/alembic/versions/2024_05_22_2011-2ea9f6f0b790_make_wcs_a_fileondiskmixin.py
@@ -0,0 +1,38 @@
+"""make WCS a FileOnDiskMixin
+
+Revision ID: 2ea9f6f0b790
+Revises: ec64a8fd8cf3
+Create Date: 2024-05-22 20:11:31.195961
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '2ea9f6f0b790'
+down_revision = '9a4097979249'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('world_coordinates', sa.Column('filepath_extensions', postgresql.ARRAY(sa.Text(), zero_indexes=True), nullable=True))
+    op.add_column('world_coordinates', sa.Column('md5sum', sa.UUID(), nullable=True))
+    op.add_column('world_coordinates', sa.Column('md5sum_extensions', postgresql.ARRAY(sa.UUID(), zero_indexes=True), nullable=True))
+    op.add_column('world_coordinates', sa.Column('filepath', sa.Text(), nullable=False))
+    op.create_index(op.f('ix_world_coordinates_filepath'), 'world_coordinates', ['filepath'], unique=True)
+    op.drop_column('world_coordinates', 'header_excerpt')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('world_coordinates', sa.Column('header_excerpt', sa.TEXT(), autoincrement=False, nullable=False))
+    op.drop_index(op.f('ix_world_coordinates_filepath'), table_name='world_coordinates')
+    op.drop_column('world_coordinates', 'filepath')
+    op.drop_column('world_coordinates', 'md5sum_extensions')
+    op.drop_column('world_coordinates', 'md5sum')
+    op.drop_column('world_coordinates', 'filepath_extensions')
+    # ### end Alembic commands ###
diff --git a/models/world_coordinates.py b/models/world_coordinates.py
index fb7508ec..5e5ad91f 100644
--- a/models/world_coordinates.py
+++ b/models/world_coordinates.py
@@ -1,4 +1,6 @@
+import pathlib
 import numpy as np
+import os
 
 import sqlalchemy as sa
 from sqlalchemy import orm
@@ -14,44 +16,13 @@
 from models.source_list import SourceList
 
 
-class WorldCoordinates(Base, AutoIDMixin, HasBitFlagBadness):
+class WorldCoordinates(Base, AutoIDMixin, FileOnDiskMixin, HasBitFlagBadness):
     __tablename__ = 'world_coordinates'
 
     __table_args__ = (
         UniqueConstraint('sources_id', 'provenance_id', name='_wcs_sources_provenance_uc'),
     )
 
-    # This is a little profligate.  There will eventually be millions of
-    # images, which means that there will be gigabytes of header data
-    # stored in the relational database.  (One header excerpt is about
-    # 4k.)  It's not safe to assume we know exactly what keywords
-    # astropy.wcs.WCS will produce, as there may be new FITS standard
-    # extensions etc., and astropy doesn't document the keywords.
-    #
-    # Another option would be to parse all the keywords into a dict of {
-    # string: (float or string) } and store them as a JSONB; that would
-    # reduce the size pretty substantially, but it would still be
-    # roughly a KB for each header, so the consideration is similar.
-    # (It's also more work to implement....)
-    #
-    # Yet another option is to store the WCS in an external file, but
-    # now we're talking something awfully small (a few kB) for this HPC
-    # filesystems.
-    #
-    # Even yet another option that we won't do short term because it's
-    # WAY too much effort is to have an additional nosql database of
-    # some sort that is designed for document storage (which really is
-    # what this is here).
-    #
-    # For now, we'll be profliate with the database, and hope we don't
-    # regret it later.
-    header_excerpt = sa.Column(
-        sa.Text,
-        nullable=False,
-        index=False,
-        doc="Text that containts FITS header cards (ASCII, \n-separated) with the header that defines this WCS"
-    )
-
     sources_id = sa.Column(
         sa.ForeignKey('source_lists.id', ondelete='CASCADE', name='world_coordinates_source_list_id_fkey'),
         nullable=False,
@@ -63,6 +34,7 @@ class WorldCoordinates(Base, AutoIDMixin, HasBitFlagBadness):
         'SourceList',
         cascade='save-update, merge, refresh-expire, expunge',
         passive_deletes=True,
+        lazy='selectin',
         doc="The source list this world coordinate system is associated with. "
     )
 
@@ -92,33 +64,30 @@ class WorldCoordinates(Base, AutoIDMixin, HasBitFlagBadness):
 
     @property
     def wcs( self ):
-        if self._wcs is None:
-            self._wcs = WCS( fits.Header.fromstring( self.header_excerpt, sep='\n' ) )
+        if self._wcs is None and self.filepath is not None:
+            self.load()
         return self._wcs
 
     @wcs.setter
     def wcs( self, value ):
         self._wcs = value
-        self.header_excerpt = value.to_header().tostring( sep='\n', padding=False )
 
     def __init__(self, *args, **kwargs):
-        SeeChangeBase.__init__(self)  # don't pass kwargs as they could contain non-column key-values
+        FileOnDiskMixin.__init__( self, **kwargs )
+        SeeChangeBase.__init__( self )
         self._wcs = None
 
         # manually set all properties (columns or not)
         self.set_attributes_from_dict(kwargs)
 
-    @orm.reconstructor
-    def init_on_load(self):
-        SeeChangeBase.init_on_load(self)
-
     def _get_inverse_badness(self):
         """Get a dict with the allowed values of badness that can be assigned to this object"""
         return catalog_match_badness_inverse
 
     @orm.reconstructor
     def init_on_load( self ):
-        Base.init_on_load( self )
+        SeeChangeBase.init_on_load( self )
+        FileOnDiskMixin.init_on_load( self )
         self._wcs = None
 
     def get_pixel_scale(self):
@@ -152,4 +121,73 @@ def get_downstreams(self, session=None):
 
         downstreams = zps + subs
         return downstreams
+
+    def save( self, filename=None, **kwargs ):
+        """Write the WCS data to disk.
+        Updates self.filepath
+        Parameters
+        ----------
+          filename: str or path
+             The path to the file to write, relative to the local store
+             root.  Do not include the extension (e.g. '.psf') at the
+             end of the name; that will be added automatically.
+             If None, will call image.invent_filepath() to get a
+             filestore-standard filename and directory.
+          Additional arguments are passed on to FileOnDiskMixin.save
+        """ 
+
+        # ----- Make sure we have a path ----- #
+        # if filename already exists, check it is correct and use
+
+        if filename is not None:
+            if not filename.endswith('.txt'):
+                filename += '.txt'
+            self.filepath = filename
+
+        # if not, generate one
+        else:
+            if self.provenance is None:
+                raise RuntimeError("Can't invent a filepath for the WCS without a provenance")
+            
+            if self.image.filepath is not None:
+                self.filepath = self.image.filepath
+            else:
+                self.filepath = self.image.invent_filepath()
+
+            self.filepath += f'.wcs_{self.provenance.id[:6]}.txt'
+
+        txtpath = pathlib.Path( self.local_path ) / self.filepath
+
+        # ----- Get the header string to save and save ----- #
+        header_txt = self.wcs.to_header().tostring(padding=False, sep='\\n' )
+
+        if txtpath.exists():
+            if not kwargs.get('overwrite', True):
+                # raise the error if overwrite is explicitly set False
+                raise FileExistsError( f"{txtpath} already exists, cannot save." )
+
+        with open( txtpath, "w") as ofp:
+            ofp.write( header_txt )
+
+        # ----- Write to the archive ----- #
+        FileOnDiskMixin.save( self, txtpath, **kwargs )
+
+    def load( self, download=True, always_verify_md5=False, txtpath=None ):
+        """Load this wcs from the file.
+        updates self.wcs.
+        Parameters
+        ----------
+        txtpath: str, Path, or None
+            File to read. If None, will load the file returned by self.get_fullpath()
+        """
+
+        if txtpath is None:
+            txtpath = self.get_fullpath( download=download, always_verify_md5=always_verify_md5)
+
+        if not os.path.isfile(txtpath):
+            raise OSError(f'WCS file is missing at {txtpath}')
+
+        with open( txtpath ) as ifp:
+            headertxt = ifp.read()
+            self.wcs = WCS( fits.Header.fromstring( headertxt , sep='\\n' ))
     
diff --git a/tests/fixtures/pipeline_objects.py b/tests/fixtures/pipeline_objects.py
index 52c6bc5d..04a8ab86 100644
--- a/tests/fixtures/pipeline_objects.py
+++ b/tests/fixtures/pipeline_objects.py
@@ -466,18 +466,18 @@ def make_datastore(
 
             ############## astro_cal to create wcs ################
             if cache_dir is not None and cache_base_name is not None:
-                cache_name = cache_base_name + '.wcs.json'
-                cache_path = os.path.join(cache_dir, cache_name)
-                if os.path.isfile(cache_path):
-                    SCLogger.debug('loading WCS from cache. ')
-                    ds.wcs = WorldCoordinates.copy_from_cache(cache_dir, cache_name)
-                    prov = Provenance(
+                prov = Provenance(
                         code_version=code_version,
                         process='astro_cal',
                         upstreams=[ds.sources.provenance],
                         parameters=p.astro_cal.pars.get_critical_pars(),
                         is_testing=True,
                     )
+                cache_name = f'{cache_base_name}.wcs_{prov.id[:6]}.txt.json'
+                cache_path = os.path.join(cache_dir, cache_name)
+                if os.path.isfile(cache_path):
+                    SCLogger.debug('loading WCS from cache. ')
+                    ds.wcs = WorldCoordinates.copy_from_cache(cache_dir, cache_name)
                     prov = session.merge(prov)
 
                     # check if WCS already exists on the database
@@ -500,13 +500,15 @@ def make_datastore(
 
                     ds.wcs.provenance = prov
                     ds.wcs.sources = ds.sources
+                    # make sure this is saved to the archive as well
+                    ds.wcs.save(verify_md5=False, overwrite=True)
 
             if ds.wcs is None:  # make the WCS
                 SCLogger.debug('Running astrometric calibration')
                 ds = p.astro_cal.run(ds)
+                ds.wcs.save()
                 if cache_dir is not None and cache_base_name is not None:
-                    # must provide a name because this one isn't a FileOnDiskMixin
-                    output_path = ds.wcs.copy_to_cache(cache_dir, cache_name)
+                    output_path = ds.wcs.copy_to_cache(cache_dir)
                     if output_path != cache_path:
                         warnings.warn(f'cache path {cache_path} does not match output path {output_path}')
 
diff --git a/tests/models/test_image.py b/tests/models/test_image.py
index 6e60959a..c665bc7d 100644
--- a/tests/models/test_image.py
+++ b/tests/models/test_image.py
@@ -1365,7 +1365,7 @@ def test_image_products_are_deleted(ptf_datastore, data_dir, archive):
     # make sure the files are there
     local_files = []
     archive_files = []
-    for obj in [im, im.psf, im.sources]:  # TODO: add WCS when it becomes a FileOnDiskMixin
+    for obj in [im, im.psf, im.sources, im.wcs]:
         for file in obj.get_fullpath(as_list=True):
             archive_file = file[len(obj.local_path)+1:]  # grap the end of the path only
             archive_file = os.path.join(archive.test_folder_path, archive_file)  # prepend the archive path
diff --git a/tests/models/test_world_coordinates.py b/tests/models/test_world_coordinates.py
index 52990dd9..7d6821d8 100644
--- a/tests/models/test_world_coordinates.py
+++ b/tests/models/test_world_coordinates.py
@@ -1,5 +1,7 @@
 import pytest
 import hashlib
+import os
+import pathlib
 
 import sqlalchemy as sa
 from sqlalchemy.exc import IntegrityError
@@ -14,6 +16,7 @@
 
 def test_world_coordinates( ztf_datastore_uncommitted, provenance_base, provenance_extra ):
     image = ztf_datastore_uncommitted.image
+    image.instrument = 'DECam' # hack - otherwise invent_filepath will not work as 'ZTF' is not an Instrument
     hdr = image.header
 
     origwcs = WCS( hdr )
@@ -23,14 +26,15 @@ def test_world_coordinates( ztf_datastore_uncommitted, provenance_base, provenan
 
     wcobj = WorldCoordinates()
     wcobj.wcs = origwcs
-    md5 = hashlib.md5( wcobj.header_excerpt.encode('ascii') )
+    header_excerpt = wcobj.wcs.to_header().tostring( sep='\n', padding=False)
+    md5 = hashlib.md5( header_excerpt.encode('ascii') )
     assert md5.hexdigest() == 'a13d6bdd520c5a0314dc751025a62619'
 
     # Make sure that we can construct a WCS from a WorldCoordinates
 
-    hdrkws = wcobj.header_excerpt
+    old_wcs = wcobj.wcs
     wcobj = WorldCoordinates()
-    wcobj.header_excerpt = hdrkws
+    wcobj.wcs = old_wcs
     scs = wcobj.wcs.pixel_to_world( [ 0, 0, 1024, 1024 ], [ 0, 1024, 0, 1024 ] )
     for sc, origsc in zip( scs, origscs ):
         assert sc.ra.value == pytest.approx( origsc.ra.value, abs=0.01/3600. )
@@ -58,17 +62,17 @@ def test_world_coordinates( ztf_datastore_uncommitted, provenance_base, provenan
                 upstreams=[provenance_extra],
                 is_testing=True,
             )
+            wcobj.save()
 
-            # TODO: will need to save the WCS object if we turn it into a FileOnDiskMixin
             session.add(wcobj)
-
             session.commit()
 
             # add a second WCS object and make sure we cannot accidentally commit it, too
             wcobj2 = WorldCoordinates()
-            wcobj2.header_excerpt = hdrkws
+            wcobj2.wcs = old_wcs
             wcobj2.sources = image.sources
             wcobj2.provenance = wcobj.provenance
+            wcobj2.save() # overwrite the save of wcobj
 
             with pytest.raises(
                     IntegrityError,
@@ -78,6 +82,10 @@ def test_world_coordinates( ztf_datastore_uncommitted, provenance_base, provenan
                 session.commit()
             session.rollback()
 
+            # ensure you cannot overwrite when explicitly setting overwrite=False
+            with pytest.raises( OSError, match=".txt already exists" ):
+                wcobj2.save(overwrite=False)
+
             # if we change any of the provenance parameters we should be able to save it
             wcobj2.provenance = Provenance(
                 process='test_world_coordinates',
@@ -86,6 +94,8 @@ def test_world_coordinates( ztf_datastore_uncommitted, provenance_base, provenan
                 upstreams=[provenance_extra],
                 is_testing=True,
             )
+            wcobj2.save(overwrite=False)
+
             session.add(wcobj2)
             session.commit()
 
@@ -107,3 +117,47 @@ def test_world_coordinates( ztf_datastore_uncommitted, provenance_base, provenan
 
             if 'image' in locals():
                 image.delete_from_disk_and_database(session=session, commit=True)
+
+
+def test_save_and_load_wcs(ztf_datastore_uncommitted, provenance_base, provenance_extra):
+    image = ztf_datastore_uncommitted.image
+    image.instrument = 'DECam' # otherwise invent_filepath will not work as 'ZTF' is not an Instrument
+    hdr = image.header
+
+    origwcs = WCS( hdr )
+    wcobj = WorldCoordinates()
+    wcobj.wcs = origwcs
+    wcobj.sources = image.sources
+    wcobj.provenance = Provenance(
+                process='test_world_coordinates',
+                code_version=provenance_base.code_version,
+                parameters={'test_parameter': 'test_value'},
+                upstreams=[provenance_extra],
+                is_testing=True,
+            )
+
+    with SmartSession() as session:
+        try:
+            wcobj.save()
+
+            txtpath = pathlib.Path( wcobj.local_path ) / f'{wcobj.filepath}'
+
+            # check for an error if the file is not found when loading
+            os.remove(txtpath)
+            with pytest.raises( OSError, match="file is missing" ):
+                wcobj.load()
+            
+            # ensure you can create an identical wcs from a saved one
+            wcobj.save()
+            wcobj2 = WorldCoordinates()
+            wcobj2.load( txtpath=txtpath )
+
+            assert wcobj2.wcs.to_header() == wcobj.wcs.to_header()
+
+            session.commit()
+
+        finally:
+            if "wcobj" in locals():
+                wcobj.delete_from_disk_and_database(session=session)
+            if "wcobj2" in locals():
+                wcobj2.delete_from_disk_and_database(session=session)

From 420f56129040b8eea8453cdc7c36a373e61e9f76 Mon Sep 17 00:00:00 2001
From: Guy Nir <37179063+guynir42@users.noreply.github.com>
Date: Wed, 29 May 2024 05:11:08 -0700
Subject: [PATCH 02/11] Replace binary dilate structure from square to circular
 (no diagonal) (#280)

---
 improc/bitmask_tools.py | 20 +++++++++++++++-----
 improc/inpainting.py    | 11 ++++++-----
 pipeline/detection.py   |  6 ++++--
 3 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/improc/bitmask_tools.py b/improc/bitmask_tools.py
index 26465829..d4209c08 100644
--- a/improc/bitmask_tools.py
+++ b/improc/bitmask_tools.py
@@ -13,14 +13,19 @@ def dilate_bitflag(array, iterations=1, structure=None):
         The number of iterations to dilate. Default is 1.
     structure : ndarray of booleans
         The kernel to use for dilation.
-        If None (default), will use a structure with square
-        connectivity equal to one.
+        If None (default), will use a structure with non-diagonal
+        connectivity (one pixel in each direction horizontally and vertically).
 
     Returns
     -------
     output : ndarray of integers
         The dilated array, same shape and type as the input array.
     """
+    if structure is None:
+        structure = np.zeros((3, 3), dtype=bool)
+        structure[1, :] = True
+        structure[:, 1] = True
+
     output = np.zeros_like(array)
     b = array.max()
     while b:
@@ -47,8 +52,8 @@ def make_saturated_flag(imdata, saturation=50000, iterations=2, structure=None):
         The number of iterations to dilate. Default is 1.
     structure : ndarray of booleans
         The kernel to use for dilation.
-        If None (default), will use a structure with square
-        connectivity equal to one.
+        If None (default), will use a structure with non-diagonal
+        connectivity (one pixel in each direction horizontally and vertically).
 
     Returns
     -------
@@ -56,4 +61,9 @@ def make_saturated_flag(imdata, saturation=50000, iterations=2, structure=None):
     """
     mask = imdata >= saturation
 
-    return ndimage.binary_dilation(mask, iterations=iterations, structure=structure)
\ No newline at end of file
+    if structure is None:
+        structure = np.zeros((3, 3), dtype=bool)
+        structure[1, :] = True
+        structure[:, 1] = True
+
+    return ndimage.binary_dilation(mask, iterations=iterations, structure=structure)
diff --git a/improc/inpainting.py b/improc/inpainting.py
index 91cd281d..668dc4f4 100644
--- a/improc/inpainting.py
+++ b/improc/inpainting.py
@@ -157,11 +157,12 @@ def inpaint_cube(self):
                 square = np.zeros((2 * w + 3, 2 * w + 3))
                 square[1:-1, 1:-1] = 1.0
                 k = convolve(square, square, mode='same')  # linear tapering kernel
-                pos_tapered = convolve(
-                    binary_dilation(np.pad(positions, 2 * w), iterations=w * 2, structure=np.ones((3, 3))),
-                    k,
-                    mode='same'
-                )[2 * w:-2 * w, 2 * w:-2 * w]
+
+                struc = np.zeros((3, 3), dtype=bool)
+                struc[1, :] = True
+                struc[:, 1] = True
+                dilat = binary_dilation(np.pad(positions, 2 * w), iterations=w * 2, structure=struc)
+                pos_tapered = convolve(dilat, k, mode='same')[2 * w:-2 * w, 2 * w:-2 * w]
                 if np.max(pos_tapered) > 0:
                     pos_tapered /= np.max(pos_tapered)
                 pos_flipped = 1 - pos_tapered
diff --git a/pipeline/detection.py b/pipeline/detection.py
index f77fcbdf..884bc8aa 100644
--- a/pipeline/detection.py
+++ b/pipeline/detection.py
@@ -915,8 +915,10 @@ def extract_sources_filter(self, image):
         det_map = abs(score) > self.pars.threshold  # catch negative peaks too (can get rid of them later)
 
         # dilate the map to merge nearby peaks
-        struct = np.ones((1 + 2 * fwhm_pixels, 1 + 2 * fwhm_pixels))
-        det_map = ndimage.binary_dilation(det_map, structure=struct).astype(det_map.dtype)
+        struc = np.zeros((3, 3), dtype=bool)
+        struc[1, :] = True
+        struc[:, 1] = True
+        det_map = ndimage.binary_dilation(det_map, iterations=fwhm_pixels, structure=struc).astype(det_map.dtype)
 
         # label the map to get the number of sources
         labels, num_sources = ndimage.label(det_map)

From 256a0687252a610593e350e857395f0d89e25f92 Mon Sep 17 00:00:00 2001
From: dryczanowski <85491059+dryczanowski@users.noreply.github.com>
Date: Wed, 29 May 2024 13:46:53 +0100
Subject: [PATCH 03/11] =?UTF-8?q?#248:=20Change=20location=20of=20fits=20t?=
 =?UTF-8?q?ests=20from=20test=5Futil=20to=20test=5Ffits=5Fopera=E2=80=A6?=
 =?UTF-8?q?=20(#293)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Change location of fits tests from test_util to test_fits_operations
---
 tests/util/test_fits_operations.py | 227 ++++++++++++++++++++++++++++-
 tests/util/test_util.py            | 227 +----------------------------
 2 files changed, 226 insertions(+), 228 deletions(-)

diff --git a/tests/util/test_fits_operations.py b/tests/util/test_fits_operations.py
index 951a1568..4be9a793 100644
--- a/tests/util/test_fits_operations.py
+++ b/tests/util/test_fits_operations.py
@@ -2,8 +2,15 @@
 
 import numpy as np
 
-from models.base import CODE_ROOT
-from util.util import read_fits_image
+import pytest
+import pathlib
+import random
+
+from astropy.io import fits
+
+from util.util import read_fits_image, save_fits_image_file
+
+from models.base import CODE_ROOT, FileOnDiskMixin
 
 
 def test_read_fits_image(decam_fits_image_filename, cache_dir):
@@ -29,3 +36,219 @@ def test_read_fits_image(decam_fits_image_filename, cache_dir):
 
     assert header['LMT_MG'] == 25.37038556706342
 
+@pytest.fixture
+def fits_file():
+    filename = ( ''.join( random.choices( 'abcdefghijklmnopqrstuvwxyz', k=10 ) ) )
+    filepath = pathlib.Path( FileOnDiskMixin.temp_path ) / filename
+
+    data = np.zeros( (64, 32), dtype=np.float32 )
+    hdr = fits.Header()
+    hdr[ 'TEST1' ] = 'testing 1'
+    hdr[ 'TEST2' ] = 'testing 2'
+
+    savedpath = pathlib.Path( save_fits_image_file( str(filepath), data, hdr ) )
+
+    yield filepath, savedpath
+
+    savedpath.unlink()
+
+@pytest.fixture
+def fits_single_file():
+    filename = ( ''.join( random.choices( 'abcdefghijklmnopqrstuvwxyz', k=10 ) ) )
+    filepath = pathlib.Path( FileOnDiskMixin.temp_path ) / filename
+
+    data = np.zeros( (64, 32), dtype=np.float32 )
+    hdr = fits.Header()
+    hdr[ 'TEST1' ] = 'testing 1'
+    hdr[ 'TEST2' ] = 'testing 2'
+
+    savedpath = pathlib.Path( save_fits_image_file( str(filepath), data, hdr,
+                                                                   extname='image', single_file=True ) )
+
+    yield filepath, savedpath
+
+    savedpath.unlink()
+
+@pytest.fixture
+def two_extension_fits_file():
+    filename = ( ''.join( random.choices( 'abcdefghijklmnopqrstuvwxyz', k=10 ) ) )
+    filepath = pathlib.Path( FileOnDiskMixin.temp_path ) / filename
+
+    data = np.full( (64, 32), 3.141, dtype=np.float32 )
+    hdr = fits.Header()
+    hdr[ 'TEST1' ] = 'testing 64'
+    hdr[ 'TEST2' ] = 'testing 128'
+
+    savedpath1 = save_fits_image_file( str(filepath), data, hdr, extname='image', single_file=True )
+
+    data = np.full( (64, 32), 2.718, dtype=np.float32 )
+    hdr[ 'TEST1' ] = 'Rosencrantz'
+    hdr[ 'TEST2' ] = 'Guildenstern'
+
+    savedpath2 = save_fits_image_file( str(filepath), data, hdr, extname='weight', single_file=True )
+
+    assert savedpath1 == savedpath2
+
+    savedpath = pathlib.Path( savedpath1 )
+    yield str(savedpath)
+
+    savedpath.unlink()
+
+def test_basic_save_fits_file( fits_file ):
+    filepath, fullpath = fits_file
+    with fits.open( fullpath ) as ifp:
+        assert ifp[0].header['BITPIX'] == -32
+        assert ifp[0].header['NAXIS'] == 2
+        assert ifp[0].header['NAXIS1'] == 32
+        assert ifp[0].header['NAXIS2'] == 64
+        assert 'BSCALE' not in ifp[0].header
+        assert 'BZERO' not in ifp[0].header
+        assert ifp[0].header['TEST1'] == 'testing 1'
+        assert ifp[0].header['TEST2'] == 'testing 2'
+        assert ifp[0].data.dtype == np.dtype('>f4')
+        assert ( ifp[0].data == np.zeros( ( 64, 32 ) ) ).all()
+
+def test_save_separate_extension( fits_file ):
+    filepath, fullpath = fits_file
+    nextpath = filepath.parent / f'{filepath.name}.next.fits'
+
+    data = np.full( (64, 32), 1., dtype=np.float32 )
+    hdr = fits.Header()
+    hdr[ 'EXTTEST1' ] = 'extension testing 1'
+    hdr[ 'EXTTEST2' ] = 'extension testing 2'
+
+    try:
+        save_fits_image_file( str(filepath), data, hdr, extname='next' )
+
+        with fits.open( fullpath ) as ifp:
+            assert ifp[0].header['TEST1'] == 'testing 1'
+            assert ifp[0].header['TEST2'] == 'testing 2'
+            assert ( ifp[0].data == np.zeros( ( 64, 32 ) ) ).all()
+
+        with fits.open( nextpath ) as ifp:
+            assert ifp[0].header['EXTTEST1'] == 'extension testing 1'
+            assert ifp[0].header['EXTTEST2'] == 'extension testing 2'
+            assert ( ifp[0].data == np.full( ( 64, 32 ), 1., dtype=np.float32 ) ).all()
+    finally:
+        nextpath.unlink( missing_ok=True )
+
+
+def test_save_extension( fits_single_file ):
+    filepath, fullpath = fits_single_file
+
+    data = np.full( (64, 32), 1., dtype=np.float32 )
+    hdr = fits.Header()
+    hdr[ 'EXTTEST1' ] = 'extension testing 1'
+    hdr[ 'EXTTEST2' ] = 'extension testing 2'
+
+    save_fits_image_file( str(filepath), data, hdr, extname='next', single_file=True )
+
+    with fits.open( fullpath ) as ifp:
+        assert ifp[1].header['TEST1'] == 'testing 1'
+        assert ifp[1].header['TEST2'] == 'testing 2'
+        assert ( ifp[1].data == np.zeros( ( 64, 32 ) ) ).all()
+        assert ifp['image'].header['TEST1'] == 'testing 1'
+        assert ifp['image'].header['TEST2'] == 'testing 2'
+        assert ( ifp['image'].data == np.zeros( ( 64, 32 ) ) ).all()
+        assert ifp[2].header['EXTTEST1'] == 'extension testing 1'
+        assert ifp[2].header['EXTTEST2'] == 'extension testing 2'
+        assert ( ifp[2].data == np.full( ( 64, 32 ), 1., dtype=np.float32 ) ).all()
+        assert ifp['next'].header['EXTTEST1'] == 'extension testing 1'
+        assert ifp['next'].header['EXTTEST2'] == 'extension testing 2'
+        assert ( ifp['next'].data == np.full( ( 64, 32 ), 1., dtype=np.float32 ) ).all()
+
+def test_no_overwrite( fits_file ):
+    filepath, fullpath = fits_file
+
+    data = np.full( (64, 32), 3., dtype=np.float32 )
+    hdr = fits.Header()
+    hdr[ 'TEST1' ] = 'testing 42'
+    hdr[ 'TEST2' ] = 'testing 64738'
+
+    with pytest.raises( OSError, match='File.*already exists' ):
+        savedpath = pathlib.Path( save_fits_image_file( str(filepath), data, hdr, overwrite=False ) )
+    with fits.open( fullpath ) as ifp:
+        assert ifp[0].header['TEST1'] == 'testing 1'
+        assert ifp[0].header['TEST2'] == 'testing 2'
+        assert ( ifp[0].data == np.zeros( ( 64, 32 ) ) ).all()
+
+def test_overwrite( fits_file ):
+    filepath, fullpath = fits_file
+
+    data = np.full( (64, 32), 3., dtype=np.float32 )
+    hdr = fits.Header()
+    hdr[ 'TEST1' ] = 'testing 42'
+    hdr[ 'TEST2' ] = 'testing 64738'
+
+    savedpath = pathlib.Path( save_fits_image_file( str(filepath), data, hdr, overwrite=True ) )
+    assert savedpath == fullpath
+    with fits.open( fullpath ) as ifp:
+        assert ifp[0].header['TEST1'] == 'testing 42'
+        assert ifp[0].header['TEST2'] == 'testing 64738'
+        assert( ifp[0].data == np.full( (64, 32), 3., dtype=np.float32 ) ).all()
+
+
+def test_basic_read( fits_file ):
+    filename, fullpath = fits_file
+
+    hdr = read_fits_image( fullpath, output='header' )
+    assert isinstance( hdr, fits.Header )
+    assert hdr['TEST1'] == 'testing 1'
+
+    data = read_fits_image( fullpath, output='data' )
+    assert data.dtype == np.float32
+    assert ( data == np.zeros( ( 64, 32 ) ) ).all()
+
+    data, hdr = read_fits_image( fullpath, output='both' )
+    assert isinstance( hdr, fits.Header )
+    assert hdr['TEST1'] == 'testing 1'
+    assert ( data == np.zeros( ( 64, 32 ) ) ).all()
+
+
+def test_read_extension( two_extension_fits_file ):
+    hdr = read_fits_image( two_extension_fits_file, ext='image', output='header' )
+    assert isinstance( hdr, fits.Header )
+    assert hdr['TEST1'] == 'testing 64'
+    assert hdr['TEST2'] == 'testing 128'
+
+    hdr = read_fits_image( two_extension_fits_file, ext='weight', output='header' )
+    assert isinstance( hdr, fits.Header )
+    assert hdr['TEST1'] == 'Rosencrantz'
+    assert hdr['TEST2'] == 'Guildenstern'
+
+    data = read_fits_image( two_extension_fits_file, ext='image', output='data' )
+    assert data.dtype == np.float32
+    assert ( data == np.full( (64, 32), 3.141, dtype=np.float32 ) ).all()
+
+    data = read_fits_image( two_extension_fits_file, ext='weight', output='data' )
+    assert data.dtype == np.float32
+    assert ( data == np.full( (64, 32), 2.718, dtype=np.float32 ) ).all()
+
+    data, hdr = read_fits_image( two_extension_fits_file, ext='image', output='both' )
+    assert isinstance( hdr, fits.Header )
+    assert hdr['TEST1'] == 'testing 64'
+    assert hdr['TEST2'] == 'testing 128'
+    assert ( data == np.full( (64, 32), 3.141,dtype=np.float32 ) ).all()
+
+    data, hdr = read_fits_image( two_extension_fits_file, ext='weight', output='both' )
+    assert isinstance( hdr, fits.Header )
+    assert hdr['TEST1'] == 'Rosencrantz'
+    assert hdr['TEST2'] == 'Guildenstern'
+    assert ( data == np.full( (64, 32), 2.718, dtype=np.float32 ) ).all()
+
+def test_just_update_header( fits_file ):
+    filepath, fullpath = fits_file
+
+    with fits.open( fullpath ) as ifp:
+        header = ifp[0].header
+        data = ifp[0].data
+
+    header['TEST3'] = 'added'
+    data = np.full( (64, 32), 1.414, dtype=np.float32 )
+
+    savedpath = save_fits_image_file( str(filepath), data, header, just_update_header=True )
+    assert pathlib.Path( savedpath ) == fullpath
+
+    with fits.open( fullpath) as ifp:
+        assert ifp[0].header['TEST3'] == 'added'
+        assert ( ifp[0].data == np.zeros( (64, 32), dtype=np.float32 ) ).all()
diff --git a/tests/util/test_util.py b/tests/util/test_util.py
index 1aa62df4..f7bc1d78 100644
--- a/tests/util/test_util.py
+++ b/tests/util/test_util.py
@@ -2,13 +2,7 @@
 import pathlib
 import random
 
-import numpy as np
-
-from astropy.io import fits
-
-from models.base import FileOnDiskMixin
-
-from util.util import listify, ensure_file_does_not_exist, read_fits_image, save_fits_image_file
+from util.util import listify, ensure_file_does_not_exist
 
 # TODO : tests other than fits file reading/saving
 
@@ -71,222 +65,3 @@ def test_ensure_file_does_not_exist():
             else:
                 fpath.rmdir()
 
-@pytest.fixture
-def fits_file():
-    filename = ( ''.join( random.choices( 'abcdefghijklmnopqrstuvwxyz', k=10 ) ) )
-    filepath = pathlib.Path( FileOnDiskMixin.temp_path ) / filename
-
-    data = np.zeros( (64, 32), dtype=np.float32 )
-    hdr = fits.Header()
-    hdr[ 'TEST1' ] = 'testing 1'
-    hdr[ 'TEST2' ] = 'testing 2'
-
-    savedpath = pathlib.Path( save_fits_image_file( str(filepath), data, hdr ) )
-
-    yield filepath, savedpath
-
-    savedpath.unlink()
-
-@pytest.fixture
-def fits_single_file():
-    filename = ( ''.join( random.choices( 'abcdefghijklmnopqrstuvwxyz', k=10 ) ) )
-    filepath = pathlib.Path( FileOnDiskMixin.temp_path ) / filename
-
-    data = np.zeros( (64, 32), dtype=np.float32 )
-    hdr = fits.Header()
-    hdr[ 'TEST1' ] = 'testing 1'
-    hdr[ 'TEST2' ] = 'testing 2'
-
-    savedpath = pathlib.Path( save_fits_image_file( str(filepath), data, hdr,
-                                                                   extname='image', single_file=True ) )
-
-    yield filepath, savedpath
-
-    savedpath.unlink()
-
-@pytest.fixture
-def two_extension_fits_file():
-    filename = ( ''.join( random.choices( 'abcdefghijklmnopqrstuvwxyz', k=10 ) ) )
-    filepath = pathlib.Path( FileOnDiskMixin.temp_path ) / filename
-
-    data = np.full( (64, 32), 3.141, dtype=np.float32 )
-    hdr = fits.Header()
-    hdr[ 'TEST1' ] = 'testing 64'
-    hdr[ 'TEST2' ] = 'testing 128'
-
-    savedpath1 = save_fits_image_file( str(filepath), data, hdr, extname='image', single_file=True )
-
-    data = np.full( (64, 32), 2.718, dtype=np.float32 )
-    hdr[ 'TEST1' ] = 'Rosencrantz'
-    hdr[ 'TEST2' ] = 'Guildenstern'
-
-    savedpath2 = save_fits_image_file( str(filepath), data, hdr, extname='weight', single_file=True )
-
-    assert savedpath1 == savedpath2
-
-    savedpath = pathlib.Path( savedpath1 )
-    yield str(savedpath)
-
-    savedpath.unlink()
-
-def test_basic_save_fits_file( fits_file ):
-    filepath, fullpath = fits_file
-    with fits.open( fullpath ) as ifp:
-        assert ifp[0].header['BITPIX'] == -32
-        assert ifp[0].header['NAXIS'] == 2
-        assert ifp[0].header['NAXIS1'] == 32
-        assert ifp[0].header['NAXIS2'] == 64
-        assert 'BSCALE' not in ifp[0].header
-        assert 'BZERO' not in ifp[0].header
-        assert ifp[0].header['TEST1'] == 'testing 1'
-        assert ifp[0].header['TEST2'] == 'testing 2'
-        assert ifp[0].data.dtype == np.dtype('>f4')
-        assert ( ifp[0].data == np.zeros( ( 64, 32 ) ) ).all()
-
-def test_save_separate_extension( fits_file ):
-    filepath, fullpath = fits_file
-    nextpath = filepath.parent / f'{filepath.name}.next.fits'
-
-    data = np.full( (64, 32), 1., dtype=np.float32 )
-    hdr = fits.Header()
-    hdr[ 'EXTTEST1' ] = 'extension testing 1'
-    hdr[ 'EXTTEST2' ] = 'extension testing 2'
-
-    try:
-        save_fits_image_file( str(filepath), data, hdr, extname='next' )
-
-        with fits.open( fullpath ) as ifp:
-            assert ifp[0].header['TEST1'] == 'testing 1'
-            assert ifp[0].header['TEST2'] == 'testing 2'
-            assert ( ifp[0].data == np.zeros( ( 64, 32 ) ) ).all()
-
-        with fits.open( nextpath ) as ifp:
-            assert ifp[0].header['EXTTEST1'] == 'extension testing 1'
-            assert ifp[0].header['EXTTEST2'] == 'extension testing 2'
-            assert ( ifp[0].data == np.full( ( 64, 32 ), 1., dtype=np.float32 ) ).all()
-    finally:
-        nextpath.unlink( missing_ok=True )
-
-
-def test_save_extension( fits_single_file ):
-    filepath, fullpath = fits_single_file
-
-    data = np.full( (64, 32), 1., dtype=np.float32 )
-    hdr = fits.Header()
-    hdr[ 'EXTTEST1' ] = 'extension testing 1'
-    hdr[ 'EXTTEST2' ] = 'extension testing 2'
-
-    save_fits_image_file( str(filepath), data, hdr, extname='next', single_file=True )
-
-    with fits.open( fullpath ) as ifp:
-        assert ifp[1].header['TEST1'] == 'testing 1'
-        assert ifp[1].header['TEST2'] == 'testing 2'
-        assert ( ifp[1].data == np.zeros( ( 64, 32 ) ) ).all()
-        assert ifp['image'].header['TEST1'] == 'testing 1'
-        assert ifp['image'].header['TEST2'] == 'testing 2'
-        assert ( ifp['image'].data == np.zeros( ( 64, 32 ) ) ).all()
-        assert ifp[2].header['EXTTEST1'] == 'extension testing 1'
-        assert ifp[2].header['EXTTEST2'] == 'extension testing 2'
-        assert ( ifp[2].data == np.full( ( 64, 32 ), 1., dtype=np.float32 ) ).all()
-        assert ifp['next'].header['EXTTEST1'] == 'extension testing 1'
-        assert ifp['next'].header['EXTTEST2'] == 'extension testing 2'
-        assert ( ifp['next'].data == np.full( ( 64, 32 ), 1., dtype=np.float32 ) ).all()
-
-def test_no_overwrite( fits_file ):
-    filepath, fullpath = fits_file
-
-    data = np.full( (64, 32), 3., dtype=np.float32 )
-    hdr = fits.Header()
-    hdr[ 'TEST1' ] = 'testing 42'
-    hdr[ 'TEST2' ] = 'testing 64738'
-
-    with pytest.raises( OSError, match='File.*already exists' ):
-        savedpath = pathlib.Path( save_fits_image_file( str(filepath), data, hdr, overwrite=False ) )
-    with fits.open( fullpath ) as ifp:
-        assert ifp[0].header['TEST1'] == 'testing 1'
-        assert ifp[0].header['TEST2'] == 'testing 2'
-        assert ( ifp[0].data == np.zeros( ( 64, 32 ) ) ).all()
-
-def test_overwrite( fits_file ):
-    filepath, fullpath = fits_file
-
-    data = np.full( (64, 32), 3., dtype=np.float32 )
-    hdr = fits.Header()
-    hdr[ 'TEST1' ] = 'testing 42'
-    hdr[ 'TEST2' ] = 'testing 64738'
-
-    savedpath = pathlib.Path( save_fits_image_file( str(filepath), data, hdr, overwrite=True ) )
-    assert savedpath == fullpath
-    with fits.open( fullpath ) as ifp:
-        assert ifp[0].header['TEST1'] == 'testing 42'
-        assert ifp[0].header['TEST2'] == 'testing 64738'
-        assert( ifp[0].data == np.full( (64, 32), 3., dtype=np.float32 ) ).all()
-
-
-def test_basic_read( fits_file ):
-    filename, fullpath = fits_file
-
-    hdr = read_fits_image( fullpath, output='header' )
-    assert isinstance( hdr, fits.Header )
-    assert hdr['TEST1'] == 'testing 1'
-
-    data = read_fits_image( fullpath, output='data' )
-    assert data.dtype == np.float32
-    assert ( data == np.zeros( ( 64, 32 ) ) ).all()
-
-    data, hdr = read_fits_image( fullpath, output='both' )
-    assert isinstance( hdr, fits.Header )
-    assert hdr['TEST1'] == 'testing 1'
-    assert ( data == np.zeros( ( 64, 32 ) ) ).all()
-
-
-def test_read_extension( two_extension_fits_file ):
-    hdr = read_fits_image( two_extension_fits_file, ext='image', output='header' )
-    assert isinstance( hdr, fits.Header )
-    assert hdr['TEST1'] == 'testing 64'
-    assert hdr['TEST2'] == 'testing 128'
-
-    hdr = read_fits_image( two_extension_fits_file, ext='weight', output='header' )
-    assert isinstance( hdr, fits.Header )
-    assert hdr['TEST1'] == 'Rosencrantz'
-    assert hdr['TEST2'] == 'Guildenstern'
-
-    data = read_fits_image( two_extension_fits_file, ext='image', output='data' )
-    assert data.dtype == np.float32
-    assert ( data == np.full( (64, 32), 3.141, dtype=np.float32 ) ).all()
-
-    data = read_fits_image( two_extension_fits_file, ext='weight', output='data' )
-    assert data.dtype == np.float32
-    assert ( data == np.full( (64, 32), 2.718, dtype=np.float32 ) ).all()
-
-    data, hdr = read_fits_image( two_extension_fits_file, ext='image', output='both' )
-    assert isinstance( hdr, fits.Header )
-    assert hdr['TEST1'] == 'testing 64'
-    assert hdr['TEST2'] == 'testing 128'
-    assert ( data == np.full( (64, 32), 3.141,dtype=np.float32 ) ).all()
-
-    data, hdr = read_fits_image( two_extension_fits_file, ext='weight', output='both' )
-    assert isinstance( hdr, fits.Header )
-    assert hdr['TEST1'] == 'Rosencrantz'
-    assert hdr['TEST2'] == 'Guildenstern'
-    assert ( data == np.full( (64, 32), 2.718, dtype=np.float32 ) ).all()
-
-def test_just_update_header( fits_file ):
-    filepath, fullpath = fits_file
-
-    with fits.open( fullpath ) as ifp:
-        header = ifp[0].header
-        data = ifp[0].data
-
-    header['TEST3'] = 'added'
-    data = np.full( (64, 32), 1.414, dtype=np.float32 )
-
-    savedpath = save_fits_image_file( str(filepath), data, header, just_update_header=True )
-    assert pathlib.Path( savedpath ) == fullpath
-
-    with fits.open( fullpath) as ifp:
-        assert ifp[0].header['TEST3'] == 'added'
-        assert ( ifp[0].data == np.zeros( (64, 32), dtype=np.float32 ) ).all()
-
-
-

From 08789038aaa85585176f59b36c58f91554c1848f Mon Sep 17 00:00:00 2001
From: whohensee <106775295+whohensee@users.noreply.github.com>
Date: Wed, 29 May 2024 11:07:33 -0700
Subject: [PATCH 04/11] Add time.sleeps to memory free tests (#263)

---
 tests/models/test_image.py       | 7 +++++++
 tests/models/test_psf.py         | 5 +++++
 tests/models/test_source_list.py | 5 +++++
 3 files changed, 17 insertions(+)

diff --git a/tests/models/test_image.py b/tests/models/test_image.py
index c665bc7d..2a5fa02b 100644
--- a/tests/models/test_image.py
+++ b/tests/models/test_image.py
@@ -6,6 +6,7 @@
 import hashlib
 import pathlib
 import uuid
+import time
 
 import numpy as np
 
@@ -1390,9 +1391,12 @@ def test_free( decam_exposure, decam_raw_image, ptf_ref ):
     proc = psutil.Process()
     origmem = proc.memory_info()
 
+    sleeptime = 0.5 # in seconds
+
     # Make sure that only_free behaves as expected
     decam_raw_image._weight = 'placeholder'
     decam_raw_image.free( only_free={'weight'} )
+    time.sleep(sleeptime)
     assert decam_raw_image._weight is None
     assert decam_raw_image._data is not None
     assert decam_raw_image.raw_data is not None
@@ -1404,6 +1408,7 @@ def test_free( decam_exposure, decam_raw_image, ptf_ref ):
     # when we free
 
     decam_raw_image.free( )
+    time.sleep(sleeptime)
     assert decam_raw_image._data is None
     # The image is ~4k by 2k, data is 32-bit
     # so expect to free ~( 4000*2000 ) *4 >~ 30MiB of data
@@ -1418,6 +1423,7 @@ def test_free( decam_exposure, decam_raw_image, ptf_ref ):
     assert decam_raw_image.raw_data is None
     decam_exposure.data.clear_cache()
     decam_exposure.section_headers.clear_cache()
+    time.sleep(sleeptime)
     gc.collect()
     freemem = proc.memory_info()
     assert origmem.rss - freemem.rss > 45 * 1024 * 1024
@@ -1449,6 +1455,7 @@ def test_free( decam_exposure, decam_raw_image, ptf_ref ):
         # Free the image and all the refs.  Expected savings: 6 4k × 2k
         # 32-bit images =~ 6 * (4000*2000) * 4 >~ 180MiB.
         ptf_ref.image.free( free_aligned=True )
+        time.sleep(sleeptime)
         gc.collect()
         freemem = proc.memory_info()
         assert origmem.rss - freemem.rss > 180 * 1024 * 1024
diff --git a/tests/models/test_psf.py b/tests/models/test_psf.py
index f79cff57..2138f2f0 100644
--- a/tests/models/test_psf.py
+++ b/tests/models/test_psf.py
@@ -3,6 +3,7 @@
 import os
 import psutil
 import gc
+import time
 import uuid
 import random
 import math
@@ -349,6 +350,8 @@ def test_free( decam_datastore ):
     ds.get_psf()
     proc = psutil.Process()
 
+    sleeptime = 0.5 # in seconds
+
     # Make sure memory is loaded
     _ = ds.image.data
     _ = ds.psf.data
@@ -361,6 +364,7 @@ def test_free( decam_datastore ):
 
     origmem = proc.memory_info()
     ds.psf.free()
+    time.sleep(sleeptime)
     assert ds.psf._data is None
     assert ds.psf._info is None
     assert ds.psf._header is None
@@ -378,6 +382,7 @@ def test_free( decam_datastore ):
 
     origmem = proc.memory_info()
     ds.image.free( free_derived_products=True )
+    time.sleep(sleeptime)
     assert ds.psf._data is None
     assert ds.psf._info is None
     assert ds.psf._header is None
diff --git a/tests/models/test_source_list.py b/tests/models/test_source_list.py
index 0a189d7d..7a72dbbb 100644
--- a/tests/models/test_source_list.py
+++ b/tests/models/test_source_list.py
@@ -4,6 +4,7 @@
 import gc
 import pathlib
 import numpy as np
+import time
 
 import sqlalchemy as sa
 
@@ -275,6 +276,8 @@ def test_free( decam_datastore ):
     ds.get_sources()
     proc = psutil.Process()
 
+    sleeptime = 0.5 # in seconds
+
     # Make sure image and source data is loaded into memory,
     #  then try freeing just the source data
     _ = ds.image.data
@@ -287,6 +290,7 @@ def test_free( decam_datastore ):
     origmem = proc.memory_info()
 
     ds.sources.free()
+    time.sleep(sleeptime)
     assert ds.sources._data is None
     assert ds.sources._info is None
     gc.collect()
@@ -326,6 +330,7 @@ def test_free( decam_datastore ):
     origmem = proc.memory_info()
 
     ds.image.free( free_derived_products=True )
+    time.sleep(sleeptime)
     assert ds.image._data is None
     assert ds.image._weight is None
     assert ds.image._flags is None

From efab173fd0dd9d6cb26ca26d9bcc481d59401b58 Mon Sep 17 00:00:00 2001
From: Guy Nir <37179063+guynir42@users.noreply.github.com>
Date: Thu, 30 May 2024 09:50:16 -0700
Subject: [PATCH 05/11] Measurements Upgrades: coordinates, bitflag, psf
 photometry (#288)

---
 .gitignore                                    |   4 +-
 ...6d17393be7_add_bitflag_for_measurements.py |  36 ++
 default_config.yaml                           |   1 -
 improc/photometry.py                          | 384 ++++++++++++------
 models/base.py                                |  21 +-
 models/cutouts.py                             |  49 ++-
 models/enums_and_bitflags.py                  |   9 +-
 models/measurements.py                        | 112 ++++-
 models/{objects.py => object.py}              |  70 ++++
 models/psf.py                                 |  10 +-
 pipeline/cutting.py                           |  18 +
 pipeline/measuring.py                         |  71 +++-
 pipeline/subtraction.py                       |   2 +-
 tests/conftest.py                             |   2 +-
 tests/fixtures/simulated.py                   |  22 +-
 tests/improc/test_photometry.py               |   7 +-
 tests/models/test_cutouts.py                  |   2 +-
 tests/models/test_enums.py                    |   1 -
 tests/models/test_image.py                    |   1 +
 tests/models/test_measurements.py             |  22 +
 tests/models/test_objects.py                  |   6 +-
 tests/pipeline/test_measuring.py              |  74 +++-
 22 files changed, 719 insertions(+), 205 deletions(-)
 create mode 100644 alembic/versions/2024_05_23_1652-f36d17393be7_add_bitflag_for_measurements.py
 rename models/{objects.py => object.py} (83%)

diff --git a/.gitignore b/.gitignore
index 9280bd59..c794ff55 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,11 +7,13 @@ tests/local_config.yaml
 tests/local_overrides.yaml
 tests/local_augments.yaml
 tests/improc/cache/*
-data/cache/*
+data/cache*
 data/DECam_default_calibrators
 .pytest.ini
 tests/plots
 tests/temp_data
+coadd.weight.fits
+tests/coadd.weight.fits
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
diff --git a/alembic/versions/2024_05_23_1652-f36d17393be7_add_bitflag_for_measurements.py b/alembic/versions/2024_05_23_1652-f36d17393be7_add_bitflag_for_measurements.py
new file mode 100644
index 00000000..1f4e9a83
--- /dev/null
+++ b/alembic/versions/2024_05_23_1652-f36d17393be7_add_bitflag_for_measurements.py
@@ -0,0 +1,36 @@
+"""add bitflag for measurements
+
+Revision ID: f36d17393be7
+Revises: ec64a8fd8cf3
+Create Date: 2024-05-23 16:52:07.448402
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f36d17393be7'
+down_revision = '2ea9f6f0b790'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('measurements', sa.Column('_bitflag', sa.BIGINT(), nullable=False))
+    op.add_column('measurements', sa.Column('description', sa.Text(), nullable=True))
+    op.add_column('measurements', sa.Column('_upstream_bitflag', sa.BIGINT(), nullable=False))
+    op.create_index(op.f('ix_measurements__bitflag'), 'measurements', ['_bitflag'], unique=False)
+    op.create_index(op.f('ix_measurements__upstream_bitflag'), 'measurements', ['_upstream_bitflag'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_measurements__upstream_bitflag'), table_name='measurements')
+    op.drop_index(op.f('ix_measurements__bitflag'), table_name='measurements')
+    op.drop_column('measurements', '_upstream_bitflag')
+    op.drop_column('measurements', 'description')
+    op.drop_column('measurements', '_bitflag')
+    # ### end Alembic commands ###
diff --git a/default_config.yaml b/default_config.yaml
index 27ef20ef..bac2205b 100644
--- a/default_config.yaml
+++ b/default_config.yaml
@@ -100,7 +100,6 @@ photo_cal:
   mag_range_catalog: 4.0
   min_catalog_stars: 50
 
-
 subtraction:
   method: zogy
   alignment:
diff --git a/improc/photometry.py b/improc/photometry.py
index 23b9ac68..94fbb004 100644
--- a/improc/photometry.py
+++ b/improc/photometry.py
@@ -49,11 +49,12 @@ class Circle:
     def __init__(self, radius, imsize=15, oversampling=100, soft=True):
         self.radius = radius
         self.imsize = imsize
+        self.datasize = max(imsize, 1 + 2 * int(radius + 1))
         self.oversampling = oversampling
         self.soft = soft
 
         # these include the circle, after being moved by sub-pixel shifts for all possible positions in x and y
-        self.datacube = np.zeros((oversampling ** 2, imsize, imsize))
+        self.datacube = np.zeros((oversampling ** 2, self.datasize, self.datasize))
 
         for i in range(oversampling):
             for j in range(oversampling):
@@ -68,11 +69,11 @@ def _make_circle(self, x, y):
             raise ValueError("x and y must be between 0 and 1")
 
         # Create the circle
-        xgrid, ygrid = np.meshgrid(np.arange(self.imsize), np.arange(self.imsize))
-        xgrid = xgrid - self.imsize // 2 - x
-        ygrid = ygrid - self.imsize // 2 - y
+        xgrid, ygrid = np.meshgrid(np.arange(self.datasize), np.arange(self.datasize))
+        xgrid = xgrid - self.datasize // 2 - x
+        ygrid = ygrid - self.datasize // 2 - y
         r = np.sqrt(xgrid ** 2 + ygrid ** 2)
-        if self.soft==True:
+        if self.soft:
             im = 1 + self.radius - r
             im[r <= self.radius] = 1
             im[r > self.radius + 1] = 0
@@ -80,7 +81,7 @@ def _make_circle(self, x, y):
             im = r
             im[r <= self.radius] = 1
             im[r > self.radius] = 0
-        
+
         # TODO: improve this with a better soft-edge function
 
         return im
@@ -130,17 +131,22 @@ def get_image(self, dx, dy):
         else:
             im[iy:, :] = 0
 
+        if self.imsize != self.datasize:  # crop the image to the correct size
+            im = im[
+                (self.datasize - self.imsize) // 2 : (self.datasize + self.imsize) // 2,
+                (self.datasize - self.imsize) // 2 : (self.datasize + self.imsize) // 2,
+            ]
+
         return im
 
 
 def iterative_cutouts_photometry(
-        image, weight, flags, psf, radii=[3.0, 5.0, 7.0], annulus=[7.5, 10.0], iterations=3, verbose=False
+        image, weight, flags, radii=[3.0, 5.0, 7.0], annulus=[7.5, 10.0], iterations=2, local_bg=True
 ):
-    """Perform aperture and PSF photometry on an image, at positions, using a list of apertures.
+    """Perform aperture photometry on an image, at slowly updating positions, using a list of apertures.
 
     The "iterative" part means that it will use the starting positions but move the aperture centers
-    around based on the centroid found using the PSF. The centroid will be used as the new position
-    for the aperture and PSF photometry, and the new centroid will be updated.
+    around based on the centroid found using the last aperture.
 
     Parameters
     ----------
@@ -150,22 +156,29 @@ def iterative_cutouts_photometry(
         The weight map for the image.
     flags: np.ndarray
         The flags for the image.
-    psf: np.ndarray or float scalar
-        The PSF to use for photometry.
-        If given as a float, will interpret that as a Gaussian
-        with that FWHM, in units of pixels.
     radii: list or 1D array
         The apertures to use for photometry.
         Must be a list of positive numbers.
         In units of pixels!
         Default is [3, 5, 7].
+    annulus: list or 1D array
+        The inner and outer radii of the annulus in pixels.
     iterations: int
-        The number of iterations to perform.
-        Each iteration will refine the position of the aperture.
-        Default is 3.
-    verbose: bool
-        If True, print out information about the progress.
-        Default is False.
+        The number of repositioning iterations to perform.
+        For each aperture, will measure and reposition the centroid
+        this many times before moving on to the next aperture.
+        After the final centroid is found, will measure the flux
+        and second moments using the best centroid, over all apertures.
+        Default is 2.
+    local_bg: bool
+        Toggle the use of a local background estimate.
+        When True, will use the measured background in the annulus
+        when calculating the centroids. If the background is really
+        well subtracted before sending the cutout into this function,
+        the results will be a little more accurate with this set to False.
+        If the area in the annulus is very crowded,
+        it's better to set this to False as well.
+        Default is True.
 
     Returns
     -------
@@ -185,14 +198,6 @@ def iterative_cutouts_photometry(
     if len(flags.shape) != 2:
         raise ValueError("Flags must be a 2D array")
 
-    # Make sure the PSF is a 2D array
-    if np.isscalar(psf):
-        psf = make_gaussian(psf, imsize=image.shape)
-    else:
-        if len(psf.shape) != 2:
-            raise ValueError("PSF must be a 2D array")
-    # TODO: still need to figure out how to actually use the PSF for photometry!
-
     # Make sure the apertures are a list or 1D array
     radii = np.atleast_1d(radii)
     if not np.all(radii > 0):
@@ -209,34 +214,41 @@ def iterative_cutouts_photometry(
 
     if np.all(nandata == 0 | np.isnan(nandata)):
         cx = cy = cxx = cyy = cxy = 0.0
-        iterations = 0  # skip the iterative mode if there's no data
+        need_break = True  # skip the iterative mode if there's no data
     else:
-        # find a rough estimate of the centroid using non-tapered cutout
-        bkg_estimate = np.nanmedian(nandata)
-        normalization = np.nansum(nandata - bkg_estimate)
-        if normalization == 0:
-            normalization = 1.0
-        elif abs(normalization) < 1.0:
-            normalization = 1.0 * np.sign(normalization)  # prevent division by zero and other rare cases
-        cx = np.nansum(xgrid * (nandata - bkg_estimate)) / normalization
-        cy = np.nansum(ygrid * (nandata - bkg_estimate)) / normalization
-        cxx = np.nansum((xgrid - cx) ** 2 * (nandata - bkg_estimate)) / normalization
-        cyy = np.nansum((ygrid - cy) ** 2 * (nandata - bkg_estimate)) / normalization
-        cxy = np.nansum((xgrid - cx) * (ygrid - cy) * (nandata - bkg_estimate)) / normalization
+        need_break = False
+        # find a rough estimate of the centroid using an unmasked cutout
+        if local_bg:
+            bkg_estimate = np.nanmedian(nandata)
+        else:
+            bkg_estimate = 0.0
+
+        denominator = np.nansum(nandata - bkg_estimate)
+        epsilon = 0.01
+        if denominator == 0:
+            denominator = epsilon
+        elif abs(denominator) < epsilon:
+            denominator = epsilon * np.sign(denominator)  # prevent division by zero and other rare cases
+
+        cx = np.nansum(xgrid * (nandata - bkg_estimate)) / denominator
+        cy = np.nansum(ygrid * (nandata - bkg_estimate)) / denominator
+        cxx = np.nansum((xgrid - cx) ** 2 * (nandata - bkg_estimate)) / denominator
+        cyy = np.nansum((ygrid - cy) ** 2 * (nandata - bkg_estimate)) / denominator
+        cxy = np.nansum((xgrid - cx) * (ygrid - cy) * (nandata - bkg_estimate)) / denominator
 
     # get some very rough estimates just so we have something in case of immediate failure of the loop
     fluxes = [np.nansum((nandata - bkg_estimate))] * len(radii)
     areas = [float(np.nansum(~np.isnan(nandata)))] * len(radii)
+    norms = [float(np.nansum(~np.isnan(nandata)))] * len(radii)
+
     background = 0.0
     variance = np.nanvar(nandata)
 
     photometry = dict(
-        psf_flux=0.0,  # TODO: update this!
-        psf_err=0.0,  # TODO: update this!
-        psf_area=0.0,  # TODO: update this!
         radii=radii,
         fluxes=fluxes,
         areas=areas,
+        normalizations=norms,
         background=background,
         variance=variance,
         offset_x=cx,
@@ -247,97 +259,76 @@ def iterative_cutouts_photometry(
     )
 
     if abs(cx) > nandata.shape[1] or abs(cy) > nandata.shape[0]:
-        iterations = 0  # skip iterations if the centroid measurement is outside the cutouts
-
-    # Loop over the iterations
-    for i in range(iterations):
-        fluxes = np.zeros(len(radii))
-        areas = np.zeros(len(radii))
-        need_break = False
+        need_break = True  # skip iterations if the centroid measurement is outside the cutouts
 
-        # reposition based on the last centroids
-        # TODO: move the reposition into the aperture loop?
-        #  That would mean we close in on the best position, but is that the right thing to do?
-        reposition_cx = cx
-        reposition_cy = cy
-        for j, r in enumerate(radii):  # go over radii in order (from large to small!)
-            # make a circle-mask based on the centroid position
-            if not np.isfinite(reposition_cx) or not np.isfinite(reposition_cy):
-                raise ValueError("Centroid is not finite, cannot proceed with photometry")
-            mask = get_circle(radius=r, imsize=nandata.shape[0]).get_image(reposition_cx, reposition_cy)
-
-            # mask the data and get the flux
-            masked_data = nandata * mask
-            fluxes[j] = np.nansum(masked_data)  # total flux, not per pixel!
-            areas[j] = np.nansum(mask)  # save the number of pixels in the aperture
-
-            # get an offset annulus to get a local background estimate
-            inner = get_circle(radius=annulus[0], imsize=nandata.shape[0], soft=False).get_image(reposition_cx, reposition_cy)
-            outer = get_circle(radius=annulus[1], imsize=nandata.shape[0], soft=False).get_image(reposition_cx, reposition_cy)
-            annulus_map = outer - inner
-            annulus_map[annulus_map == 0.] = np.nan # flag pixels outside annulus as nan
-
-            # background and variance only need to be calculated once (they are the same for all apertures)
-            # but moments/centroids can be calculated for each aperture, but we will only want to save one
-            # so how about we use the smallest one?
-            if j == 0:  # largest aperture only
-                # TODO: if we move the reposition into the aperture loop, this will need to be updated!
-                #  We would have to calculate the background/variance on the last positions, or all positions?
-                annulus_map_sum = np.nansum(annulus_map)
-                if annulus_map_sum == 0:  # this should only happen in tests or if the annulus is way too large
-                    background = 0
-                    variance = 0
-                else:
-                    # b/g mean and variance (per pixel)
-                    background, standard_dev = sigma_clipping(nandata * annulus_map, nsigma=5.0, median=True)
-                    variance = standard_dev ** 2
-
-                normalization = (fluxes[j] - background * areas[j])
-                masked_data_bg = (nandata - background) * mask
-
-                if normalization == 0:  # this should only happen in pathological cases
-                    cx = cy = cxx = cyy = cxy = 0
-                    need_break = True
-                    break
-
-                # update the centroids
-                cx = np.nansum(xgrid * masked_data_bg) / normalization
-                cy = np.nansum(ygrid * masked_data_bg) / normalization
-
-                # update the second moments
-                cxx = np.nansum((xgrid - cx) ** 2 * masked_data_bg) / normalization
-                cyy = np.nansum((ygrid - cy) ** 2 * masked_data_bg) / normalization
-                cxy = np.nansum((xgrid - cx) * (ygrid - cy) * masked_data_bg) / normalization
-
-                # TODO: how to do PSF photometry with offsets and a given PSF? and get the error, too!
-
-                # check that we got reasonable values! If not, break and keep the current values
-                if np.isnan(cx) or cx > nandata.shape[1] / 2 or cx < -nandata.shape[1] / 2:
-                    need_break = True
-                    break  # there's no point doing more radii if we are not going to save the results!
-                if np.isnan(cy) or cy > nandata.shape[0] / 2 or cy < -nandata.shape[0] / 2:
-                    need_break = True
-                    break  # there's no point doing more radii if we are not going to save the results!
-                if np.nansum(mask) == 0 or np.nansum(annulus_map) == 0:
-                    need_break = True
-                    break  # there's no point doing more radii if we are not going to save the results!
+    # in case any of the iterations fail, go back to the last centroid
+    prev_cx = cx
+    prev_cy = cy
 
+    for j, r in enumerate(radii):  # go over radii in order (from large to small!)
+        # short circuit if one of the measurements failed
         if need_break:
             break
 
-        photometry['psf_flux'] = 0.0  # TODO: update this!
-        photometry['psf_err'] = 0.0  # TODO: update this!
-        photometry['psf_area'] = 0.0  # TODO: update this!
-        photometry['radii'] = radii[::-1]  # return radii and fluxes in increasing order
-        photometry['fluxes'] = fluxes[::-1]  # return radii and fluxes in increasing order
-        photometry['areas'] = areas[::-1]  # return radii and fluxes in increasing order
-        photometry['background'] = background
-        photometry['variance'] = variance
-        photometry['offset_x'] = cx
-        photometry['offset_y'] = cy
-        photometry['moment_xx'] = cxx
-        photometry['moment_yy'] = cyy
-        photometry['moment_xy'] = cxy
+        # for each radius, do 1-3 rounds of repositioning the centroid
+        for i in range(iterations):
+            flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, failure = calc_at_position(
+                nandata, r, annulus, xgrid, ygrid, cx, cy, local_bg=local_bg, full=False  # reposition only!
+            )
+
+            if failure:
+                need_break = True
+                cx = prev_cx
+                cy = prev_cy
+                break
+
+            # keep this in case any of the iterations fail
+            prev_cx = cx
+            prev_cy = cy
+
+    fluxes = np.full(len(radii), np.nan)
+    areas = np.full(len(radii), np.nan)
+    norms = np.full(len(radii), np.nan)
+
+    # no more updating of the centroids!
+    best_cx = cx
+    best_cy = cy
+
+    # go over each radius again and this time get all outputs (e.g., cxx) using the best centroid
+    for j, r in enumerate(radii):
+        flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, failure = calc_at_position(
+            nandata,
+            r,
+            annulus,
+            xgrid,
+            ygrid,
+            best_cx,
+            best_cy,
+            local_bg=local_bg,
+            soft=True,
+            full=True,
+            fixed=True,
+        )
+
+        if failure:
+            break
+
+        fluxes[j] = flux
+        areas[j] = area
+        norms[j] = norm
+
+    # update the output dictionary
+    photometry['radii'] = radii[::-1]  # return radii and fluxes in increasing order
+    photometry['fluxes'] = fluxes[::-1]  # return radii and fluxes in increasing order
+    photometry['areas'] = areas[::-1]  # return radii and areas in increasing order
+    photometry['background'] = background
+    photometry['variance'] = variance
+    photometry['normalizations'] = norms[::-1]  # return radii and areas in increasing order
+    photometry['offset_x'] = best_cx
+    photometry['offset_y'] = best_cy
+    photometry['moment_xx'] = cxx
+    photometry['moment_yy'] = cyy
+    photometry['moment_xy'] = cxy
 
     # calculate from 2nd moments the width, ratio and angle of the source
     # ref: https://en.wikipedia.org/wiki/Image_moment
@@ -357,6 +348,139 @@ def iterative_cutouts_photometry(
     return photometry
 
 
+def calc_at_position(data, radius, annulus, xgrid, ygrid, cx, cy, local_bg=True, soft=True, full=True, fixed=False):
+    """Calculate the photometry at a given position.
+
+    Parameters
+    ----------
+    data: np.ndarray
+        The image to perform photometry on.
+        Any bad pixels in the image are replaced by NaN.
+    radius: float
+        The radius of the aperture in pixels.
+    annulus: list or 1D array
+        The inner and outer radii of the annulus in pixels.
+    xgrid: np.ndarray
+        The x grid for the image.
+    ygrid: np.ndarray
+        The y grid for the image.
+    cx: float
+        The x position of the aperture center.
+    cy: float
+        The y position of the aperture center.
+    local_bg: bool
+        Toggle the use of a local background estimate.
+        When True, will use the measured background in the annulus
+        when calculating the centroids. If the background is really
+        well subtracted before sending the cutout into this function,
+        the results will be a little more accurate with this set to False.
+        If the area in the annulus is very crowded,
+        it's better to set this to False as well.
+        Default is True.
+    soft: bool
+        Toggle the use of a soft-edged aperture.
+        Default is True.
+    full: bool
+        Toggle the calculation of the fluxes and second moments.
+        If set to False, will only calculate the centroids.
+        Default is True.
+    fixed: bool
+        If True, do not update the centroid position (assume it is fixed).
+        Default is False.
+
+    Returns
+    -------
+    flux: float
+        The flux in the aperture.
+    area: float
+        The area of the aperture.
+    background: float
+        The background level.
+    variance: float
+        The variance of the background.
+    norm: float
+        The normalization factor for the flux error
+        (this is the sqrt of the sum of squares of the aperture mask).
+    cx: float
+        The x position of the centroid.
+    cy: float
+        The y position of the centroid.
+    cxx: float
+        The second moment in x.
+    cyy: float
+        The second moment in y.
+    cxy: float
+        The cross moment.
+    failure: bool
+        A flag to indicate if the calculation failed.
+        This means the centroid is outside the cutout,
+        or the aperture is empty, or things like that.
+        If True, it flags to the outer scope to stop
+        the iterative process.
+    """
+    flux = area = background = variance = norm = cxx = cyy = cxy = 0
+
+    # make a circle-mask based on the centroid position
+    if not np.isfinite(cx) or not np.isfinite(cy):
+        raise ValueError("Centroid is not finite, cannot proceed with photometry")
+
+    # get a circular mask
+    mask = get_circle(radius=radius, imsize=data.shape[0], soft=soft).get_image(cx, cy)
+    if np.nansum(mask) == 0:
+        return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
+
+    masked_data = data * mask
+
+    flux = np.nansum(masked_data)  # total flux, not per pixel!
+    area = np.nansum(mask)  # save the number of pixels in the aperture
+    denominator = flux
+    masked_data_bg = masked_data
+
+    # get an offset annulus to get a local background estimate
+    if full or local_bg:
+        inner = get_circle(radius=annulus[0], imsize=data.shape[0], soft=False).get_image(cx, cy)
+        outer = get_circle(radius=annulus[1], imsize=data.shape[0], soft=False).get_image(cx, cy)
+        annulus_map = outer - inner
+        annulus_map[annulus_map == 0.] = np.nan  # flag pixels outside annulus as nan
+
+        if np.nansum(annulus_map) == 0:  # this can happen if annulus is too large
+            return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
+
+        annulus_map_sum = np.nansum(annulus_map)
+        if annulus_map_sum == 0:  # this should only happen in tests or if the annulus is way too large
+            background = 0
+            variance = 0
+            norm = 0
+        else:
+            # b/g mean and variance (per pixel)
+            background, standard_dev = sigma_clipping(data * annulus_map, nsigma=5.0, median=True)
+            variance = standard_dev ** 2
+            norm = np.sqrt(np.nansum(mask ** 2))
+
+        if local_bg:  # update these to use the local background
+            denominator = (flux - background * area)
+            masked_data_bg = (data - background) * mask
+
+    if denominator == 0:  # this should only happen in pathological cases
+        return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
+
+    if not fixed:  # update the centroids
+        cx = np.nansum(xgrid * masked_data_bg) / denominator
+        cy = np.nansum(ygrid * masked_data_bg) / denominator
+
+        # check that we got reasonable values!
+        if np.isnan(cx) or abs(cx) > data.shape[1] / 2 or np.isnan(cy) or abs(cy) > data.shape[0] / 2:
+            return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
+
+    if full:
+        # update the second moments
+        cxx = np.nansum((xgrid - cx) ** 2 * masked_data_bg) / denominator
+        cyy = np.nansum((ygrid - cy) ** 2 * masked_data_bg) / denominator
+        cxy = np.nansum((xgrid - cx) * (ygrid - cy) * masked_data_bg) / denominator
+
+    return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, False
+
+
 if __name__ == '__main__':
     import matplotlib
     matplotlib.use('TkAgg')
diff --git a/models/base.py b/models/base.py
index fae85231..a9fd7a62 100644
--- a/models/base.py
+++ b/models/base.py
@@ -181,7 +181,7 @@ def get_all_database_objects(display=False, session=None):
     from models.zero_point import ZeroPoint
     from models.cutouts import Cutouts
     from models.measurements import Measurements
-    from models.objects import Object
+    from models.object import Object
     from models.calibratorfile import CalibratorFile
     from models.catalog_excerpt import CatalogExcerpt
     from models.reference import Reference
@@ -431,8 +431,15 @@ def to_dict(self):
             if isinstance(value, np.ndarray) and key in [
                 'aper_rads', 'aper_radii', 'aper_cors', 'aper_cor_radii',
                 'flux_apertures', 'flux_apertures_err', 'area_apertures',
+                'ra', 'dec',
             ]:
-                value = list(value)
+                if len(value.shape) > 0:
+                    value = list(value)
+                else:
+                    value = float(value)
+
+            if isinstance(value, np.number):
+                value = value.item()
 
             if key in ['modified', 'created_at'] and isinstance(value, datetime.datetime):
                 value = value.isoformat()
@@ -1933,17 +1940,17 @@ def update_downstream_badness(self, session=None, commit=True):
         """
         # make sure this object is current:
         with SmartSession(session) as session:
+            merged_self = session.merge(self)
             new_bitflag = 0  # start from scratch, in case some upstreams have lost badness
-            for upstream in self.get_upstreams(session):
+            for upstream in merged_self.get_upstreams(session):
                 if hasattr(upstream, '_bitflag'):
                     new_bitflag |= upstream.bitflag
 
-            if hasattr(self, '_upstream_bitflag'):
-                self._upstream_bitflag = new_bitflag
-                session.add(self)
+            if hasattr(merged_self, '_upstream_bitflag'):
+                merged_self._upstream_bitflag = new_bitflag
 
             # recursively do this for all the other objects
-            for downstream in self.get_downstreams(session):
+            for downstream in merged_self.get_downstreams(session):
                 if hasattr(downstream, 'update_downstream_badness') and callable(downstream.update_downstream_badness):
                     downstream.update_downstream_badness(session=session, commit=False)
 
diff --git a/models/cutouts.py b/models/cutouts.py
index 8b64cf5e..59e323ab 100644
--- a/models/cutouts.py
+++ b/models/cutouts.py
@@ -20,7 +20,7 @@
     SpatiallyIndexed,
     HasBitFlagBadness,
 )
-from models.enums_and_bitflags import CutoutsFormatConverter
+from models.enums_and_bitflags import CutoutsFormatConverter, cutouts_badness_inverse
 from models.source_list import SourceList
 
 
@@ -136,12 +136,17 @@ def __init__(self, *args, **kwargs):
         self.format = 'hdf5'  # the default should match the column-defined default above!
 
         self._source_row = None
+
         self._sub_data = None
         self._sub_weight = None
         self._sub_flags = None
+        self._sub_psfflux = None
+        self._sub_psffluxerr = None
+
         self._ref_data = None
         self._ref_weight = None
         self._ref_flags = None
+
         self._new_data = None
         self._new_weight = None
         self._new_flags = None
@@ -157,13 +162,19 @@ def __init__(self, *args, **kwargs):
     def init_on_load(self):
         Base.init_on_load(self)
         FileOnDiskMixin.init_on_load(self)
+
         self._source_row = None
+
         self._sub_data = None
         self._sub_weight = None
         self._sub_flags = None
+        self._sub_psfflux = None
+        self._sub_psffluxerr = None
+
         self._ref_data = None
         self._ref_weight = None
         self._ref_flags = None
+
         self._new_data = None
         self._new_weight = None
         self._new_flags = None
@@ -184,16 +195,20 @@ def __setattr__(self, key, value):
         super().__setattr__(key, value)
 
     @staticmethod
-    def get_data_attributes():
+    def get_data_attributes(include_optional=True):
         names = ['source_row']
         for im in ['sub', 'ref', 'new']:
             for att in ['data', 'weight', 'flags']:
                 names.append(f'{im}_{att}')
+
+        if include_optional:
+            names += ['sub_psfflux', 'sub_psffluxerr']
+
         return names
 
     @property
     def has_data(self):
-        for att in self.get_data_attributes():
+        for att in self.get_data_attributes(include_optional=False):
             if getattr(self, att) is None:
                 return False
         return True
@@ -310,14 +325,15 @@ def _save_dataset_to_hdf5(self, file, groupname):
             if att == 'source_row':
                 continue
 
-            data = getattr(self, att)
-            file.create_dataset(
-                f'{groupname}/{att}',
-                data=data,
-                shape=data.shape,
-                dtype=data.dtype,
-                compression='gzip'
-            )
+            data = getattr(self, f'_{att}')  # get the private attribute so as not to trigger a load upon hitting None
+            if data is not None:
+                file.create_dataset(
+                    f'{groupname}/{att}',
+                    data=data,
+                    shape=data.shape,
+                    dtype=data.dtype,
+                    compression='gzip'
+                )
 
         # handle the source_row dictionary
         target = file[groupname].attrs
@@ -432,8 +448,8 @@ def _load_dataset_from_hdf5(self, file, groupname):
         """
         for att in self.get_data_attributes():
             if att == 'source_row':
-                self.source_row = dict(file[f'{groupname}'].attrs)
-            else:
+                self.source_row = dict(file[groupname].attrs)
+            elif att in file[groupname]:
                 setattr(self, att, np.array(file[f'{groupname}/{att}']))
 
         self.format = 'hdf5'
@@ -659,9 +675,9 @@ def get_upstreams(self, session=None):
             return session.scalars(sa.select(SourceList).where(SourceList.id == self.sources_id)).all()
         
     def get_downstreams(self, session=None):
-        """Get the downstream Measurements that were made from this Cutouts. """
+        """Get the downstream Measurements that were made from this Cutouts object. """
         from models.measurements import Measurements
-        from models.objects import Object
+
         with SmartSession(session) as session:
             return session.scalars(sa.select(Measurements).where(Measurements.cutouts_id == self.id)).all()
 
@@ -747,6 +763,9 @@ def check_equals(self, other):
 
         return True
 
+    def _get_inverse_badness(self):
+        return cutouts_badness_inverse
+
 
 # use these two functions to quickly add the "property" accessor methods
 def load_attribute(object, att):
diff --git a/models/enums_and_bitflags.py b/models/enums_and_bitflags.py
index ff64148a..c928c0de 100644
--- a/models/enums_and_bitflags.py
+++ b/models/enums_and_bitflags.py
@@ -374,7 +374,7 @@ def string_to_bitflag(value, dictionary):
 
 
 # join the badness:
-data_badness_dict = {0: 'good'}
+data_badness_dict = {}
 data_badness_dict.update(image_badness_dict)
 data_badness_dict.update(cutouts_badness_dict)
 data_badness_dict.update(source_list_badness_dict)
@@ -382,6 +382,13 @@ def string_to_bitflag(value, dictionary):
 if 0 in data_badness_inverse:
     raise ValueError('Cannot have a badness bitflag of zero. This is reserved for good data.')
 
+
+class BadnessConverter( EnumConverter ):
+    _dict = data_badness_dict
+    _allowed_values = data_badness_dict
+    _dict_filtered = None
+    _dict_inverse = None
+
 # bitflag for image preprocessing steps that have been done
 image_preprocessing_dict = {
     0: 'overscan',
diff --git a/models/measurements.py b/models/measurements.py
index 5010ad36..f800cb59 100644
--- a/models/measurements.py
+++ b/models/measurements.py
@@ -8,11 +8,13 @@
 from sqlalchemy.dialects.postgresql import JSONB, ARRAY
 from sqlalchemy.ext.associationproxy import association_proxy
 
-from models.base import Base, SeeChangeBase, SmartSession, AutoIDMixin, SpatiallyIndexed
+from models.base import Base, SeeChangeBase, SmartSession, AutoIDMixin, SpatiallyIndexed, HasBitFlagBadness
 from models.cutouts import Cutouts
 
+from improc.photometry import get_circle
 
-class Measurements(Base, AutoIDMixin, SpatiallyIndexed):
+
+class Measurements(Base, AutoIDMixin, SpatiallyIndexed, HasBitFlagBadness):
 
     __tablename__ = 'measurements'
 
@@ -111,6 +113,24 @@ class Measurements(Base, AutoIDMixin, SpatiallyIndexed):
 
     filter = association_proxy('cutouts', 'sources.image.filter')
 
+    @property
+    def flux(self):
+        """The background subtracted aperture flux in the "best" aperture. """
+        if self.best_aperture == -1:
+            return self.flux_psf - self.background * self.area_psf
+        else:
+            return self.flux_apertures[self.best_aperture] - self.background * self.area_apertures[self.best_aperture]
+
+    @property
+    def flux_err(self):
+        """The error on the background subtracted aperture flux in the "best" aperture. """
+        if self.best_aperture == -1:
+            return np.sqrt(self.flux_psf_err ** 2 + self.background_err ** 2 * self.area_psf)
+        else:
+            err = self.flux_apertures_err[self.best_aperture]
+            err += self.background_err ** 2 * self.area_apertures[self.best_aperture]
+            return np.sqrt(err)
+
     @property
     def mag_psf(self):
         if self.flux_psf <= 0:
@@ -286,11 +306,6 @@ def __setattr__(self, key, value):
         if key in ['flux_apertures', 'flux_apertures_err', 'aper_radii']:
             value = np.array(value)
 
-        if key == 'cutouts':
-            super().__setattr__('cutouts_id', value.id)
-            for att in ['ra', 'dec', 'gallon', 'gallat', 'ecllon', 'ecllat']:
-                super().__setattr__(att, getattr(value, att))
-
         super().__setattr__(key, value)
 
     def get_filter_description(self, number=None):
@@ -366,7 +381,7 @@ def associate_object(self, session=None):
         This should only be done for measurements that have passed all preliminary cuts,
         which mostly rules out obvious artefacts.
         """
-        from models.objects import Object  # avoid circular import
+        from models.object import Object  # avoid circular import
 
         with SmartSession(session) as session:
             obj = session.scalars(sa.select(Object).where(
@@ -388,6 +403,87 @@ def associate_object(self, session=None):
 
             self.object = obj
 
+    def get_flux_at_point(self, ra, dec, aperture=None):
+        """Use the given coordinates to find the flux, assuming it is inside the cutout.
+
+        Parameters
+        ----------
+        ra: float
+            The right ascension of the point in degrees.
+        dec: float
+            The declination of the point in degrees.
+        aperture: int, optional
+            Use this aperture index in the list of aperture radii to choose
+            which aperture to use. Set -1 to get PSF photometry.
+            Leave None to use the best_aperture.
+            Can also specify "best" or "psf".
+
+        Returns
+        -------
+        flux: float
+            The flux in the aperture.
+        fluxerr: float
+            The error on the flux.
+        area: float
+            The area of the aperture.
+        """
+        if aperture is None:
+            aperture = self.best_aperture
+        if aperture == 'best':
+            aperture = self.best_aperture
+        if aperture == 'psf':
+            aperture = -1
+
+        im = self.cutouts.sub_nandata  # the cutouts image we are working with (includes NaNs for bad pixels)
+
+        wcs = self.cutouts.sources.image.new_image.wcs.wcs
+        # these are the coordinates relative to the center of the cutouts
+        image_pixel_x = wcs.world_to_pixel_values(ra, dec)[0]
+        image_pixel_y = wcs.world_to_pixel_values(ra, dec)[1]
+
+        offset_x = image_pixel_x - self.cutouts.x
+        offset_y = image_pixel_y - self.cutouts.y
+
+        if abs(offset_x) > im.shape[1] / 2 or abs(offset_y) > im.shape[0] / 2:
+            return np.nan, np.nan, np.nan  # quietly return NaNs for large offsets, they will fail the cuts anyway...
+
+        if np.isnan(image_pixel_x) or np.isnan(image_pixel_y):
+            return np.nan, np.nan, np.nan  # if we can't use the WCS for some reason, need to fail gracefully
+
+        if aperture == -1:
+            # get the subtraction PSF or (if unavailable) the new image PSF
+            psf = self.cutouts.sources.image.get_psf()
+            psf_clip = psf.get_clip(x=image_pixel_x, y=image_pixel_y)
+            offset_ix = int(np.round(offset_x))
+            offset_iy = int(np.round(offset_y))
+            # shift the psf_clip by the offset and multiply by the cutouts sub_flux
+            # the corner offset between the pixel coordinates of the cutout to that of the psf_clip:
+            dx = psf_clip.shape[1] // 2 - im.shape[1] // 2 - offset_ix
+            dy = psf_clip.shape[0] // 2 - im.shape[0] // 2 - offset_iy
+            start_x = max(0, -dx)  # where (in cutout coordinates) do we start counting the pixels
+            end_x = min(im.shape[1], psf_clip.shape[1] - dx)  # where do we stop counting the pixels
+            start_y = max(0, -dy)
+            end_y = min(im.shape[0], psf_clip.shape[0] - dy)
+
+            # make a mask the same size as the cutout, with the offset PSF and zeros where it is not overlapping
+            # before clipping the non overlapping and removing bad pixels, the PSF clip was normalized to 1
+            mask = np.zeros_like(im, dtype=float)
+            mask[start_y:end_y, start_x:end_x] = psf_clip[start_y + dy:end_y + dy, start_x + dx:end_x + dx]
+            mask[np.isnan(im)] = 0  # exclude bad pixels from the mask
+            flux = np.nansum(im * mask) / np.nansum(mask ** 2)
+            fluxerr = self.background_err / np.sqrt(np.nansum(mask ** 2))
+            area = np.nansum(mask) / (np.nansum(mask ** 2))
+        else:
+            radius = self.aper_radii[aperture]
+            # get the aperture mask
+            mask = get_circle(radius=radius, imsize=im.shape[0], soft=True).get_image(offset_x, offset_y)
+            # for aperture photometry we don't normalize, just assume the PSF is in the aperture
+            flux = np.nansum(im * mask)
+            fluxerr = self.background_err * np.sqrt(np.nansum(mask ** 2))
+            area = np.nansum(mask)
+
+        return flux, fluxerr, area
+
     def get_upstreams(self, session=None):
         """Get the image that was used to make this source list. """
         with SmartSession(session) as session:
diff --git a/models/objects.py b/models/object.py
similarity index 83%
rename from models/objects.py
rename to models/object.py
index edd0b43c..c164a6b7 100644
--- a/models/objects.py
+++ b/models/object.py
@@ -8,6 +8,7 @@
 from sqlalchemy import orm
 
 from astropy.time import Time
+from astropy.coordinates import SkyCoord
 
 from models.base import Base, SeeChangeBase, SmartSession, AutoIDMixin, SpatiallyIndexed
 from models.measurements import Measurements
@@ -185,6 +186,75 @@ def get_measurements_list(
 
         return output
 
+    def get_mean_coordinates(self, sigma=3.0, iterations=3, measurement_list_kwargs=None):
+        """Get the mean coordinates of the object.
+
+        Uses the measurements that are loaded using the get_measurements_list method.
+        From these, central ra/dec are calculated, using an aperture flux weighted mean.
+        Outliers are removed based on the sigma/iterations parameters.
+
+        Parameters
+        ----------
+        sigma: float, optional
+            The sigma to use for the clipping of the measurements. Default is 3.0.
+        iterations: int, optional
+            The number of iterations to use for the clipping of the measurements. Default is 3.
+        measurement_list_kwargs: dict, optional
+            The keyword arguments to pass to the get_measurements_list method.
+
+        Returns
+        -------
+        float, float
+            The mean RA and Dec of the object.
+        """
+        measurements = self.get_measurements_list(**(measurement_list_kwargs or {}))
+
+        ra = np.array([m.ra for m in measurements])
+        dec = np.array([m.dec for m in measurements])
+        flux = np.array([m.flux for m in measurements])
+        fluxerr = np.array([m.flux_err for m in measurements])
+
+        good = np.isfinite(ra) & np.isfinite(dec) & np.isfinite(flux) & np.isfinite(fluxerr)
+        good &= flux > fluxerr * 3.0  # require a 3-sigma detection
+        # make sure that if one of these is bad, all are bad
+        ra[~good] = np.nan
+        dec[~good] = np.nan
+        flux[~good] = np.nan
+
+        points = SkyCoord(ra, dec, unit='deg')
+
+        ra_mean = np.nansum(ra * flux) / np.nansum(flux[good])
+        dec_mean = np.nansum(dec * flux) / np.nansum(flux[good])
+        center = SkyCoord(ra_mean, dec_mean, unit='deg')
+
+        num_good = np.sum(good)
+        if num_good < 3:
+            iterations = 0  # skip iterative step if too few points
+
+        # clip the measurements
+        for i in range(iterations):
+            # the 2D distance from the center
+            offsets = points.separation(center).arcsec
+
+            scatter = np.nansum(flux * offsets ** 2) / np.nansum(flux)
+            scatter *= num_good / (num_good - 1)
+            scatter = np.sqrt(scatter)
+
+            bad_idx = np.where(offsets > sigma * scatter)[0]
+            ra[bad_idx] = np.nan
+            dec[bad_idx] = np.nan
+            flux[bad_idx] = np.nan
+
+            num_good = np.sum(np.isfinite(flux))
+            if num_good < 3:
+                break
+
+            ra_mean = np.nansum(ra * flux) / np.nansum(flux)
+            dec_mean = np.nansum(dec * flux) / np.nansum(flux)
+            center = SkyCoord(ra_mean, dec_mean, unit='deg')
+
+        return ra_mean, dec_mean
+
     @staticmethod
     def make_naming_function(format_string):
         """Generate a function that will translate a serial number into a name.
diff --git a/models/psf.py b/models/psf.py
index eaa933b6..8e4fc2ac 100644
--- a/models/psf.py
+++ b/models/psf.py
@@ -305,7 +305,6 @@ def load( self, download=True, always_verify_md5=False, psfpath=None, psfxmlpath
         with open( psfxmlpath ) as ifp:
             self._info = ifp.read()
 
-
     def free( self ):
         """Free loaded world coordinates memory.
 
@@ -318,7 +317,6 @@ def free( self ):
         self._info = None
         self._header = None
 
-
     def get_resampled_psf( self, x, y, dtype=np.float64 ):
         """Return an image fragment with the PSF at the underlying sampling of the PSF model.
 
@@ -428,8 +426,8 @@ def get_clip( self, x=None, y=None, flux=1.0, norm=True, noisy=False, gain=1., r
 
         psfwid, psfsamp, stampwid, psfdex1d = self._get_clip_info()
 
-        xc = int( np.floor(x + 0.5) )
-        yc = int( np.floor(y + 0.5) )
+        xc = int( np.round(x) )
+        yc = int( np.round(y) )
 
         # See Chapter 5, "How PSFEx Works", of the PSFEx manual
         #   https://psfex.readthedocs.io/en/latest/Working.html
@@ -491,8 +489,8 @@ def add_psf_to_image( self, image, x, y, flux, norm=True, noisy=False, weight=No
         if ( x < 0 ) or ( x >= image.shape[1] ) or ( y < 0 ) or ( y >= image.shape[0] ):
             SCLogger.warn( "Center of psf to be added to image is off of edge of image" )
 
-        xc = int( np.floor(x + 0.5) )
-        yc = int( np.floor(y + 0.5) )
+        xc = int( np.round(x) )
+        yc = int( np.round(y) )
         clip = self.get_clip( x, y, flux, norm=norm, noisy=noisy, gain=gain, rng=rng )
         stampwid = clip.shape[1]
 
diff --git a/pipeline/cutting.py b/pipeline/cutting.py
index 17afc428..651dfa5e 100644
--- a/pipeline/cutting.py
+++ b/pipeline/cutting.py
@@ -93,9 +93,19 @@ def run(self, *args, **kwargs):
                 sub_stamps_data = make_cutouts(ds.sub_image.data, x, y, sz)
                 sub_stamps_weight = make_cutouts(ds.sub_image.weight, x, y, sz, fillvalue=0)
                 sub_stamps_flags = make_cutouts(ds.sub_image.flags, x, y, sz, fillvalue=0)
+
+                # TODO: figure out if we can actually use this flux (maybe renormalize it)
+                # if ds.sub_image.psfflux is not None and ds.sub_image.psffluxerr is not None:
+                #     sub_stamps_psfflux = make_cutouts(ds.sub_image.psfflux, x, y, sz, fillvalue=0)
+                #     sub_stamps_psffluxerr = make_cutouts(ds.sub_image.psffluxerr, x, y, sz, fillvalue=0)
+                # else:
+                #     sub_stamps_psfflux = None
+                #     sub_stamps_psffluxerr = None
+
                 ref_stamps_data = make_cutouts(ds.sub_image.ref_aligned_image.data, x, y, sz)
                 ref_stamps_weight = make_cutouts(ds.sub_image.ref_aligned_image.weight, x, y, sz, fillvalue=0)
                 ref_stamps_flags = make_cutouts(ds.sub_image.ref_aligned_image.flags, x, y, sz, fillvalue=0)
+
                 new_stamps_data = make_cutouts(ds.sub_image.new_aligned_image.data, x, y, sz)
                 new_stamps_weight = make_cutouts(ds.sub_image.new_aligned_image.weight, x, y, sz, fillvalue=0)
                 new_stamps_flags = make_cutouts(ds.sub_image.new_aligned_image.flags, x, y, sz, fillvalue=0)
@@ -106,14 +116,22 @@ def run(self, *args, **kwargs):
                     cutout.sub_data = sub_stamps_data[i]
                     cutout.sub_weight = sub_stamps_weight[i]
                     cutout.sub_flags = sub_stamps_flags[i]
+                    # TODO: figure out if we can actually use this flux (maybe renormalize it)
+                    # if sub_stamps_psfflux is not None and sub_stamps_psffluxerr is not None:
+                    #     cutout.sub_psfflux = sub_stamps_psfflux[i]
+                    #     cutout.sub_psffluxerr = sub_stamps_psffluxerr[i]
+
                     cutout.ref_data = ref_stamps_data[i]
                     cutout.ref_weight = ref_stamps_weight[i]
                     cutout.ref_flags = ref_stamps_flags[i]
+
                     cutout.new_data = new_stamps_data[i]
                     cutout.new_weight = new_stamps_weight[i]
                     cutout.new_flags = new_stamps_flags[i]
+
                     cutout._upstream_bitflag = 0
                     cutout._upstream_bitflag |= detections.bitflag
+
                     cutout_list.append(cutout)
 
             # add the resulting list to the data store
diff --git a/pipeline/measuring.py b/pipeline/measuring.py
index 725631cd..9e264eec 100644
--- a/pipeline/measuring.py
+++ b/pipeline/measuring.py
@@ -10,7 +10,7 @@
 
 from models.cutouts import Cutouts
 from models.measurements import Measurements
-from models.enums_and_bitflags import BitFlagConverter
+from models.enums_and_bitflags import BitFlagConverter, BadnessConverter
 
 from pipeline.parameters import Parameters
 from pipeline.data_store import DataStore
@@ -52,7 +52,7 @@ def __init__(self, **kwargs):
 
         self.analytical_cuts = self.add_par(
             'analytical_cuts',
-            ['negatives', 'bad pixels', 'offsets', 'filter bank'],
+            ['negatives', 'bad pixels', 'offsets', 'filter bank', 'bad_flag'],
             [list],
             'Which kinds of analytic cuts are used to give scores to this measurement. '
         )
@@ -80,6 +80,14 @@ def __init__(self, **kwargs):
             'The same types are ignored when running photometry. '
         )
 
+        self.bad_flag_exclude = self.add_par(
+            'bad_flag_exclude',
+            [],
+            list,
+            'List of strings of the bad flag types (i.e., bitflag) to exclude from the bad flag cut. '
+            'This includes things like image saturation, too many sources, etc. '
+        )
+
         self.streak_filter_angle_step = self.add_par(
             'streak_filter_angle_step',
             5.0,
@@ -102,6 +110,7 @@ def __init__(self, **kwargs):
                 'bad pixels': 1,
                 'offsets': 5.0,
                 'filter bank': 1,
+                'bad_flag': 1,
             },
             dict,
             'Thresholds for the disqualifier scores. '
@@ -211,23 +220,20 @@ def run(self, *args, **kwargs):
 
                     annulus_radii_pixels = self.pars.annulus_radii
                     if self.pars.annulus_units == 'fwhm':
-                        annulus_radii_pixels = [rad * c.source.image.get_psf().fwhm_pixels for rad in annulus_radii_pixels]
+                        fwhm = c.source.image.get_psf().fwhm_pixels
+                        annulus_radii_pixels = [rad * fwhm for rad in annulus_radii_pixels]
 
                     # TODO: consider if there are any additional parameters that photometry needs
                     output = iterative_cutouts_photometry(
                         c.sub_data,
                         c.sub_weight,
                         flags,
-                        m.psf,
                         radii=m.aper_radii,
                         annulus=annulus_radii_pixels,
                     )
 
-                    m.flux_psf = output['psf_flux']
-                    m.flux_psf_err = output['psf_err']
-                    m.area_psf = output['psf_area']
                     m.flux_apertures = output['fluxes']
-                    m.flux_apertures_err = [np.sqrt(output['variance'] * a) for a in output['areas']]  # TODO: add source noise??
+                    m.flux_apertures_err = [np.sqrt(output['variance']) * norm for norm in output['normalizations']]
                     m.aper_radii = output['radii']
                     m.area_apertures = output['areas']
                     m.background = output['background']
@@ -238,6 +244,42 @@ def run(self, *args, **kwargs):
                     m.elongation = output['elongation']
                     m.position_angle = output['angle']
 
+                    # update the coordinates using the centroid offsets
+                    x = c.x + m.offset_x
+                    y = c.y + m.offset_y
+                    ra, dec = m.cutouts.sources.image.new_image.wcs.wcs.pixel_to_world_values(x, y)
+                    m.ra = float(ra)
+                    m.dec = float(dec)
+
+                    # PSF photometry:
+                    # Two options: use the PSF flux from ZOGY, or use the new image PSF to measure the flux.
+                    # TODO: this is currently commented out since I don't know how to normalize this flux
+                    # if c.sub_psfflux is not None and c.sub_psffluxerr is not None:
+                    #     ix = int(np.round(m.offset_x + c.sub_data.shape[1] // 2))
+                    #     iy = int(np.round(m.offset_y + c.sub_data.shape[0] // 2))
+                    #
+                    #     # when offsets are so big it really doesn't matter what we put here, it will fail the cuts
+                    #     if ix < 0 or ix >= c.sub_psfflux.shape[1] or iy < 0 or iy >= c.sub_psfflux.shape[0]:
+                    #         m.flux_psf = np.nan
+                    #         m.flux_psf_err = np.nan
+                    #         m.area_psf = np.nan
+                    #     else:
+                    #         m.flux_psf = c.sub_psfflux[iy, ix]
+                    #         m.flux_psf_err = c.sub_psffluxerr[iy, ix]
+                    #         psf = c.sources.image.get_psf()
+                    #         m.area_psf = np.nansum(psf.get_clip(c.x, c.y))
+                    # else:
+                    if np.isnan(ra) or np.isnan(dec):
+                        flux = np.nan
+                        fluxerr = np.nan
+                        area = np.nan
+                    else:
+                        flux, fluxerr, area = m.get_flux_at_point(ra, dec, aperture='psf')
+                    m.flux_psf = flux
+                    m.flux_psf_err = fluxerr
+                    m.area_psf = area
+
+                    # decide on the "best" aperture
                     if self.pars.chosen_aperture == 'auto':
                         raise NotImplementedError('Automatic aperture selection is not yet implemented.')
                     if self.pars.chosen_aperture == 'psf':
@@ -250,6 +292,7 @@ def run(self, *args, **kwargs):
                         )
                     m.best_aperture = ap_index
 
+                    # update the provenance
                     m.provenance = prov
                     m.provenance_id = prov.id
 
@@ -291,6 +334,18 @@ def run(self, *args, **kwargs):
 
                     # TODO: add additional disqualifiers
 
+                    m._upstream_bitflag = 0
+                    m._upstream_bitflag |= c.bitflag
+
+                    ignore_bits = 0
+                    for badness in self.pars.bad_flag_exclude:
+                        ignore_bits |= 2 ** BadnessConverter.convert(badness)
+
+                    m.disqualifier_scores['bad_flag'] = np.bitwise_and(
+                        np.array(m.bitflag).astype('uint64'),
+                        ~np.array(ignore_bits).astype('uint64'),
+                    )
+
                     # make sure disqualifier scores don't have any numpy types
                     for k, v in m.disqualifier_scores.items():
                         if isinstance(v, np.number):
diff --git a/pipeline/subtraction.py b/pipeline/subtraction.py
index 54edeff6..f3ba94b2 100644
--- a/pipeline/subtraction.py
+++ b/pipeline/subtraction.py
@@ -320,7 +320,7 @@ def run(self, *args, **kwargs):
                         sub_image.psffluxerr = outdict['alpha_err']
                     if 'psf' in outdict:
                         # TODO: clip the array to be a cutout around the PSF, right now it is same shape as image!
-                        sub_image.zogy_psf = outdict['psf']  # not saved but can be useful for testing / source detection
+                        sub_image.zogy_psf = outdict['psf']  # not saved, can be useful for testing / source detection
                         if 'alpha' in outdict and 'alpha_err' in outdict:
                             sub_image.psfflux = outdict['alpha']
                             sub_image.psffluxerr = outdict['alpha_err']
diff --git a/tests/conftest.py b/tests/conftest.py
index ec1e1fb0..9ff71332 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -21,7 +21,7 @@
 from models.provenance import CodeVersion, Provenance
 from models.catalog_excerpt import CatalogExcerpt
 from models.exposure import Exposure
-from models.objects import Object
+from models.object import Object
 
 from util.archive import Archive
 from util.util import remove_empty_folders
diff --git a/tests/fixtures/simulated.py b/tests/fixtures/simulated.py
index 2a061df7..b0016a0e 100644
--- a/tests/fixtures/simulated.py
+++ b/tests/fixtures/simulated.py
@@ -9,6 +9,7 @@
 
 from astropy.io import fits
 from astropy.time import Time
+from astropy.wcs import WCS
 
 from models.base import SmartSession
 from models.provenance import Provenance
@@ -16,6 +17,7 @@
 from models.image import Image
 from models.source_list import SourceList
 from models.psf import PSF
+from models.world_coordinates import WorldCoordinates
 from models.zero_point import ZeroPoint
 from models.reference import Reference
 from models.cutouts import Cutouts
@@ -25,6 +27,7 @@
 
 from tests.conftest import rnd_str
 
+
 def make_sim_exposure():
     e = Exposure(
         filepath=f"Demo_test_{rnd_str(5)}.fits",
@@ -350,6 +353,8 @@ def sim_image_list(
         fake_sources_data,
         ztf_filepaths_image_sources_psf
 ):
+    ra = np.random.uniform(30, 330)
+    dec = np.random.uniform(-30, 30)
     num = 5
     width = 1.0
     # use the ZTF files to generate a legitimate PSF (that has get_clip())
@@ -376,8 +381,8 @@ def sim_image_list(
             # add some additional products we may need down the line
             im.sources = SourceList(format='filter', data=fake_sources_data)
             # must randomize the sources data to get different MD5sum
-            im.sources.data['x'] += np.random.normal(0, 1, len(fake_sources_data))
-            im.sources.data['y'] += np.random.normal(0, 1, len(fake_sources_data))
+            im.sources.data['x'] += np.random.normal(0, .1, len(fake_sources_data))
+            im.sources.data['y'] += np.random.normal(0, .1, len(fake_sources_data))
 
             for j in range(len(im.sources.data)):
                 dx = im.sources.data['x'][j] - im.raw_data.shape[1] / 2
@@ -407,7 +412,18 @@ def sim_image_list(
             im.zp.aper_cor_radii = im.instrument_object.standard_apertures()
             im.zp.aper_cors = np.random.normal(0, 0.1, len(im.zp.aper_cor_radii))
             im.zp.provenance = provenance_extra
+            im.wcs = WorldCoordinates()
+            im.wcs.wcs = WCS()
+            # hack the pixel scale to reasonable values (0.3" per pixel)
+            im.wcs.wcs.wcs.pc = np.array([[0.0001, 0.0], [0.0, 0.0001]])
+            im.wcs.wcs.wcs.crval = np.array([ra, dec])
+            im.wcs.provenance = provenance_extra
+            im.wcs.provenance_id = im.wcs.provenance.id
+            im.wcs.sources = im.sources
+            im.wcs.sources_id = im.sources.id
+            im.wcs.save()
             im.sources.zp = im.zp
+            im.sources.wcs = im.wcs
             im = im.merge_all(session)
             images.append(im)
 
@@ -612,6 +628,7 @@ def sim_lightcurves(sim_sub_image_list, measurer):
     # for each image contains a list of measurements for the same source
     measurer.pars.thresholds['bad pixels'] = 100  # avoid losing measurements to random bad pixels
     measurer.pars.thresholds['offsets'] = 10  # avoid losing measurements to random offsets
+    measurer.pars.association_radius = 5.0  # make it harder for random offsets to dis-associate the measurements
     lightcurves = []
 
     with SmartSession() as session:
@@ -627,3 +644,4 @@ def sim_lightcurves(sim_sub_image_list, measurer):
     yield lightcurves
 
     # no cleanup for this one
+
diff --git a/tests/improc/test_photometry.py b/tests/improc/test_photometry.py
index 578656e0..296d5b5e 100644
--- a/tests/improc/test_photometry.py
+++ b/tests/improc/test_photometry.py
@@ -17,6 +17,7 @@
 # def test_circle_soft():
     # pass
 
+
 def test_circle_hard():
     circTst = get_circle(radius=3,imsize=7,soft=False).get_image(0,0)
     assert np.array_equal(circTst, np.array([[0., 0., 0., 1., 0., 0., 0.],
@@ -27,6 +28,7 @@ def test_circle_hard():
                                              [0., 1., 1., 1., 1., 1., 0.],
                                              [0., 0., 0., 1., 0., 0., 0.]]))
 
+
 def test_background_sigma_clip(ptf_datastore):
     imgClip = ptf_datastore.image.data[     clipCentX - clipHalfWidth : clipCentX + clipHalfWidth, 
                                             clipCentY - clipHalfWidth : clipCentY + clipHalfWidth]
@@ -34,9 +36,10 @@ def test_background_sigma_clip(ptf_datastore):
                                             clipCentY - clipHalfWidth : clipCentY + clipHalfWidth]
     flagsClip = ptf_datastore.image.flags[  clipCentX - clipHalfWidth : clipCentX + clipHalfWidth, 
                                             clipCentY - clipHalfWidth : clipCentY + clipHalfWidth]
-    result = iterative_cutouts_photometry(imgClip, weightClip, flagsClip, np.zeros_like(imgClip))
+    result = iterative_cutouts_photometry(imgClip, weightClip, flagsClip)
     assert result['background'] == pytest.approx(1199.1791, rel=1e-2)
-  
+
+
 @pytest.mark.skipif( os.getenv('INTERACTIVE') is None, reason='Set INTERACTIVE to run this test' )
 def test_plot_annulus(ptf_datastore):
     imgClip = ptf_datastore.image.data[clipCentX-clipHalfWidth:clipCentX+clipHalfWidth, 
diff --git a/tests/models/test_cutouts.py b/tests/models/test_cutouts.py
index 9c67a79a..2bb3b0e6 100644
--- a/tests/models/test_cutouts.py
+++ b/tests/models/test_cutouts.py
@@ -38,7 +38,7 @@ def test_make_save_load_cutouts(decam_detection_list, cutter):
         assert c.bitflag is not None
 
         # set the bitflag just to see if it is loaded or not
-        c.bitflag = 41  # should be Cosmic Ray
+        c.bitflag = 2 ** 41  # should be Cosmic Ray
 
         # save an individual cutout
         Cutouts.save_list([c])
diff --git a/tests/models/test_enums.py b/tests/models/test_enums.py
index 176df1f9..755388d1 100644
--- a/tests/models/test_enums.py
+++ b/tests/models/test_enums.py
@@ -13,7 +13,6 @@
 def test_enums_zero_values():
     assert 0 not in FormatConverter.dict
     assert 0 not in ImageTypeConverter.dict
-    assert data_badness_dict[0] == 'good'
 
 
 def test_converter_dict():
diff --git a/tests/models/test_image.py b/tests/models/test_image.py
index 2a5fa02b..aafae540 100644
--- a/tests/models/test_image.py
+++ b/tests/models/test_image.py
@@ -1345,6 +1345,7 @@ def test_image_multifile(sim_image_uncommitted, provenance_base, test_config):
         test_config.set_value('storage.images.single_file', single_fileness)
 
 
+@pytest.mark.skip(reason="This test is way too slow (see Issue #291")
 def test_image_products_are_deleted(ptf_datastore, data_dir, archive):
     ds = ptf_datastore  # shorthand
 
diff --git a/tests/models/test_measurements.py b/tests/models/test_measurements.py
index e77e93af..fced6bce 100644
--- a/tests/models/test_measurements.py
+++ b/tests/models/test_measurements.py
@@ -171,3 +171,25 @@ def test_measurements_cannot_be_saved_twice(ptf_datastore):
                 session.commit()
 
 
+def test_measurements_forced_photometry(ptf_datastore):
+    offset_max = 2.0
+    for m in ptf_datastore.measurements:
+        if abs(m.offset_x) < offset_max and abs(m.offset_y) < offset_max:
+            break
+    else:
+        raise RuntimeError(f'Cannot find any measurement with offsets less than {offset_max}')
+
+    flux_small_aperture = m.get_flux_at_point(m.ra, m.dec, aperture=1)
+    flux_large_aperture = m.get_flux_at_point(m.ra, m.dec, aperture=len(m.aper_radii) - 1)
+    flux_psf = m.get_flux_at_point(m.ra, m.dec, aperture=-1)
+    assert flux_small_aperture[0] == pytest.approx(m.flux_apertures[1], abs=0.01)
+    assert flux_large_aperture[0] == pytest.approx(m.flux_apertures[-1], abs=0.01)
+    assert flux_psf[0] == pytest.approx(m.flux_psf, abs=0.01)
+
+    # print(f'Flux regular, small: {m.flux_apertures[1]}+-{m.flux_apertures_err[1]} over area: {m.area_apertures[1]}')
+    # print(f'Flux regular, big: {m.flux_apertures[-1]}+-{m.flux_apertures_err[-1]} over area: {m.area_apertures[-1]}')
+    # print(f'Flux regular, PSF: {m.flux_psf}+-{m.flux_psf_err} over area: {m.area_psf}')
+    # print(f'Flux small aperture: {flux_small_aperture[0]}+-{flux_small_aperture[1]} over area: {flux_small_aperture[2]}')
+    # print(f'Flux big aperture: {flux_large_aperture[0]}+-{flux_large_aperture[1]} over area: {flux_large_aperture[2]}')
+    # print(f'Flux PSF forced: {flux_psf[0]}+-{flux_psf[1]} over area: {flux_psf[2]}')
+
diff --git a/tests/models/test_objects.py b/tests/models/test_objects.py
index 236e1ecc..cc38c1d4 100644
--- a/tests/models/test_objects.py
+++ b/tests/models/test_objects.py
@@ -9,7 +9,7 @@
 from models.base import SmartSession
 from models.provenance import Provenance
 from models.measurements import Measurements
-from models.objects import Object
+from models.object import Object
 
 
 def test_object_creation():
@@ -29,6 +29,7 @@ def test_object_creation():
         assert re.match(r'\w+\d{4}\w+', obj2.name)
 
 
+@pytest.mark.flaky(max_runs=3)
 def test_lightcurves_from_measurements(sim_lightcurves):
     for lc in sim_lightcurves:
         expected_flux = []
@@ -45,6 +46,7 @@ def test_lightcurves_from_measurements(sim_lightcurves):
             assert measured_flux[i] == pytest.approx(expected_flux[i], abs=expected_error[i] * 3)
 
 
+@pytest.mark.flaky(max_runs=3)
 def test_filtering_measurements_on_object(sim_lightcurves):
     assert len(sim_lightcurves) > 0
     assert len(sim_lightcurves[0]) > 3
@@ -85,7 +87,7 @@ def test_filtering_measurements_on_object(sim_lightcurves):
                     setattr(m2, key, value)
             m2.provenance = prov
             m2.provenance_id = prov.id
-            m2.ra += 0.1 * i / 3600.0  # move the RA by less than one arcsec
+            m2.ra += 0.05 * i / 3600.0  # move the RA by less than one arcsec
             m2.ra = m2.ra % 360.0  # make sure RA is in range
             m2.associate_object(session)
             m2 = session.merge(m2)
diff --git a/tests/pipeline/test_measuring.py b/tests/pipeline/test_measuring.py
index b88fd0d1..620ccb0b 100644
--- a/tests/pipeline/test_measuring.py
+++ b/tests/pipeline/test_measuring.py
@@ -5,21 +5,24 @@
 
 import numpy as np
 
+from models.base import SmartSession
 
 from improc.tools import make_gaussian
 
 
 @pytest.mark.flaky(max_runs=3)
-def test_measuring(measurer, decam_cutouts):
+def test_measuring(measurer, decam_cutouts, decam_default_calibrators):
     measurer.pars.test_parameter = uuid.uuid4().hex
-    measurer.pars.bad_pixel_exclude = ['saturated']
+    measurer.pars.bad_pixel_exclude = ['saturated']  # ignore saturated pixels
+    measurer.pars.bad_flag_exclude = ['satellite']  # ignore satellite cutouts
+
     sz = decam_cutouts[0].sub_data.shape
     fwhm = decam_cutouts[0].sources.image.get_psf().fwhm_pixels
 
     # clear any flags for the fake data we are using
-    for i in range(12):
+    for i in range(14):
         decam_cutouts[i].sub_flags = np.zeros_like(decam_cutouts[i].sub_flags)
-
+        # decam_cutouts[i].filepath = None  # make sure the cutouts don't re-load the original data
     # delta function
     decam_cutouts[0].sub_data = np.zeros_like(decam_cutouts[0].sub_data)
     decam_cutouts[0].sub_data[sz[0] // 2, sz[1] // 2] = 100.0
@@ -72,11 +75,17 @@ def test_measuring(measurer, decam_cutouts):
     decam_cutouts[10].sub_data += np.random.normal(0, 1, size=sz)
 
     # streak
-    decam_cutouts[11].sub_data = make_gaussian(
-        imsize=sz[0], sigma_x=fwhm / 2.355, sigma_y=20, rotation=25, norm=1
-    ) * 1000
+    decam_cutouts[11].sub_data = make_gaussian(imsize=sz[0], sigma_x=fwhm / 2.355, sigma_y=20, rotation=25, norm=1)
+    decam_cutouts[11].sub_data *= 1000
     decam_cutouts[11].sub_data += np.random.normal(0, 1, size=sz)
 
+    # a regular cutout but we'll put some bad flag on the cutout
+    decam_cutouts[12].badness = 'cosmic ray'
+
+    # a regular cutout with a bad flag that we are ignoring:
+    decam_cutouts[13].badness = 'satellite'
+
+    # run the measurer
     ds = measurer.run(decam_cutouts)
 
     assert len(ds.all_measurements) == len(ds.cutouts)
@@ -93,6 +102,7 @@ def test_measuring(measurer, decam_cutouts):
     assert m.get_filter_description() == f'PSF mismatch (FWHM= 0.25 x {fwhm:.2f})'
 
     assert np.allclose(m.flux_apertures, 100)  # aperture is irrelevant for delta function
+    assert m.flux_psf > 150  # flux is more focused than the PSF, so it will bias the flux to be higher than 100
     assert m.background == 0
     assert m.background_err == 0
     for i in range(3):  # check only the last apertures, that are smaller than cutout square
@@ -106,6 +116,7 @@ def test_measuring(measurer, decam_cutouts):
     assert m.get_filter_description() == f'PSF mismatch (FWHM= 0.25 x {fwhm:.2f})'
 
     assert np.allclose(m.flux_apertures, 200)
+    assert m.flux_psf > 300  # flux is more focused than the PSF, so it will bias the flux to be higher than 100
     assert m.background == 0
     assert m.background_err == 0
 
@@ -120,6 +131,7 @@ def test_measuring(measurer, decam_cutouts):
     assert m.flux_apertures[1] < 1000
     for i in range(2, len(m.flux_apertures)):
         assert m.flux_apertures[i] == pytest.approx(1000, rel=0.1)
+    assert m.flux_psf == pytest.approx(1000, rel=0.1)
     assert m.background == pytest.approx(0, abs=0.01)
     assert m.background_err == pytest.approx(0, abs=0.01)
 
@@ -135,6 +147,7 @@ def test_measuring(measurer, decam_cutouts):
     assert m.flux_apertures[1] < 500
     for i in range(2, len(m.flux_apertures)):
         assert m.flux_apertures[i] == pytest.approx(500, rel=0.1)
+    assert m.flux_psf == pytest.approx(500, rel=0.1)
     assert m.background == pytest.approx(0, abs=0.01)
     assert m.background_err == pytest.approx(0, abs=0.01)
 
@@ -145,11 +158,11 @@ def test_measuring(measurer, decam_cutouts):
     assert m.disqualifier_scores['filter bank'] > 0
 
     # the dipole's large offsets will short-circuit the iterative repositioning of the aperture (should be flagged!)
-    assert np.allclose(m.flux_apertures, 0)
-    assert np.allclose(m.area_apertures, sz[0] * sz[1])
-    assert m.background == pytest.approx(0, abs=0.01)
-    assert m.background_err > 1.0
-    assert m.background_err < 10.0
+    assert all(np.isnan(m.flux_apertures))
+    assert all(np.isnan(m.area_apertures))
+    assert m.background == 0
+    assert m.background_err == 0
+    assert m.background_err == 0
 
     m = ds.all_measurements[5]  # shifted gaussian with noise
     assert m.disqualifier_scores['negatives'] < 1.0
@@ -200,21 +213,45 @@ def test_measuring(measurer, decam_cutouts):
     assert m.flux_apertures[1] < 600
     for i in range(2, len(m.flux_apertures)):
         assert m.flux_apertures[i] == pytest.approx(1000, rel=1)
+    assert m.flux_psf < 500  # flux is more spread out than the PSF, so it will bias the flux to be lower
 
     assert m.background == pytest.approx(0, abs=0.2)
     assert m.background_err == pytest.approx(1.0, abs=0.2)
 
     m = ds.all_measurements[11]  # streak
-    # TODO: this fails because background is too high, need to fix this by using a better background estimation
-    #  one way this could work is by doing a hard-edge annulus and taking sigma_clipping (or median) of the pixel
-    #  values, instead of the weighted mean we are using now.
-    # assert m.disqualifier_scores['negatives'] < 1.0
+    assert m.disqualifier_scores['negatives'] < 0.5
     assert m.disqualifier_scores['bad pixels'] == 0
     assert m.disqualifier_scores['offsets'] < 0.7
     assert m.disqualifier_scores['filter bank'] == 28
     assert m.get_filter_description() == 'Streaked (angle= 25.0 deg)'
-    assert m.background < 1.0  # see TODO above
-    assert m.background_err < 3.0  # TODO: above
+    assert m.background < 0.5
+    assert m.background_err < 3.0
+
+    m = ds.all_measurements[12]  # regular cutout with a bad flag
+    assert m.disqualifier_scores['bad_flag'] == 2 ** 41  # this is the bit for 'cosmic ray'
+
+    m = ds.all_measurements[13]  # regular cutout with a bad flag that we are ignoring
+    assert m.disqualifier_scores['bad_flag'] == 0  # we've included the satellite flag in the ignore list
+
+    # check that coordinates have been modified:
+    for i in range(14):
+        m = ds.all_measurements[i]
+        if m.offset_x != 0 and m.offset_y != 0:
+            assert m.ra != m.cutouts.ra
+            assert m.dec != m.cutouts.dec
+
+
+def test_propagate_badness(decam_datastore):
+    ds = decam_datastore
+    with SmartSession() as session:
+        ds.measurements[0].badness = 'cosmic ray'
+        # find the index of the cutout that corresponds to the measurement
+        idx = [i for i, c in enumerate(ds.cutouts) if c.id == ds.measurements[0].cutouts_id][0]
+        ds.cutouts[idx].badness = 'cosmic ray'
+        ds.cutouts[idx].update_downstream_badness(session)
+        m = session.merge(ds.measurements[0])
+
+        assert m.badness == 'cosmic ray'  # note that this does not change disqualifier_scores!
 
 
 def test_warnings_and_exceptions(decam_datastore, measurer):
@@ -232,3 +269,4 @@ def test_warnings_and_exceptions(decam_datastore, measurer):
         ds.reraise()
     assert "Exception injected by pipeline parameters in process 'measuring'." in str(excinfo.value)
     ds.read_exception()
+

From 6a611c8b91a94617bf97d7d2325afec57c2edd73 Mon Sep 17 00:00:00 2001
From: whohensee <106775295+whohensee@users.noreply.github.com>
Date: Mon, 3 Jun 2024 12:40:34 -0700
Subject: [PATCH 06/11] measurements and cutouts (#295)

Add threshold dictionaries for marking/deleting bad Measurements
---
 ...27dde_add_is_bad_column_to_objects_and_.py | 34 +++++++++
 default_config.yaml                           |  7 +-
 models/cutouts.py                             |  7 +-
 models/measurements.py                        | 29 ++++----
 models/object.py                              |  7 ++
 pipeline/measuring.py                         | 46 +++++++++++-
 tests/fixtures/pipeline_objects.py            |  9 ++-
 tests/fixtures/simulated.py                   |  3 +-
 tests/models/test_measurements.py             | 70 ++++++++++++++++++-
 tests/models/test_objects.py                  | 47 ++++++++++++-
 10 files changed, 232 insertions(+), 27 deletions(-)
 create mode 100644 alembic/versions/2024_05_31_1352-a7dde2327dde_add_is_bad_column_to_objects_and_.py

diff --git a/alembic/versions/2024_05_31_1352-a7dde2327dde_add_is_bad_column_to_objects_and_.py b/alembic/versions/2024_05_31_1352-a7dde2327dde_add_is_bad_column_to_objects_and_.py
new file mode 100644
index 00000000..023027f8
--- /dev/null
+++ b/alembic/versions/2024_05_31_1352-a7dde2327dde_add_is_bad_column_to_objects_and_.py
@@ -0,0 +1,34 @@
+"""Add is_bad column to Objects and Measurements
+
+Revision ID: a7dde2327dde
+Revises: f36d17393be7
+Create Date: 2024-05-31 13:52:26.008896
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a7dde2327dde'
+down_revision = 'f36d17393be7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('measurements', sa.Column('is_bad', sa.Boolean(), nullable=False))
+    op.create_index(op.f('ix_measurements_is_bad'), 'measurements', ['is_bad'], unique=False)
+    op.add_column('objects', sa.Column('is_bad', sa.Boolean(), nullable=False))
+    op.create_index(op.f('ix_objects_is_bad'), 'objects', ['is_bad'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_objects_is_bad'), table_name='objects')
+    op.drop_column('objects', 'is_bad')
+    op.drop_index(op.f('ix_measurements_is_bad'), table_name='measurements')
+    op.drop_column('measurements', 'is_bad')
+    # ### end Alembic commands ###
diff --git a/default_config.yaml b/default_config.yaml
index bac2205b..034e5e44 100644
--- a/default_config.yaml
+++ b/default_config.yaml
@@ -129,7 +129,12 @@ measuring:
     negatives: 0.3
     bad pixels: 1
     offsets: 5.0
-    filter bank: 1.0
+    filter bank: 1
+  deletion_thresholds:
+    negatives: 0.3
+    bad pixels: 1
+    offsets: 5.0
+    filter bank: 1
 
 
 # Specific configuration for specific instruments.
diff --git a/models/cutouts.py b/models/cutouts.py
index 59e323ab..fa98d78b 100644
--- a/models/cutouts.py
+++ b/models/cutouts.py
@@ -40,7 +40,7 @@ class Cutouts(Base, AutoIDMixin, FileOnDiskMixin, SpatiallyIndexed, HasBitFlagBa
         nullable=False,
         default=CutoutsFormatConverter.convert('hdf5'),
         doc="Format of the file on disk. Should be fits, hdf5, csv or npy. "
-            "Saved as integer but is converter to string when loaded. "
+            "Saved as integer but is converted to string when loaded. "
     )
 
     @hybrid_property
@@ -114,11 +114,6 @@ def format(self, value):
         )
     )
 
-    @property
-    def new_image(self):
-        """Get the aligned new image using the sub_image. """
-        return self.sub_image.new_aligned_image
-
     @property
     def new_image(self):
         """Get the aligned new image using the sub_image. """
diff --git a/models/measurements.py b/models/measurements.py
index f800cb59..59df8e3e 100644
--- a/models/measurements.py
+++ b/models/measurements.py
@@ -272,6 +272,13 @@ def instrument_object(self):
             "Given by the angle of the major axis of the distribution of counts in the aperture. "
     )
 
+    is_bad = sa.Column(
+        sa.Boolean,
+        nullable=False,
+        index=True,
+        doc='Boolean flag to indicate if the measurement failed one or more threshold value comparisons. '
+    )
+
     disqualifier_scores = sa.Column(
         JSONB,
         nullable=False,
@@ -359,16 +366,6 @@ def find_cutouts_in_list(self, cutouts_list):
 
         raise ValueError('Cutouts not found in the list. ')
 
-    def passes(self):
-        """check if there are disqualifiers above the threshold
-
-        Note that if a threshold is missing or None, that disqualifier is not checked
-        """
-        for key, value in self.provenance.parameters['thresholds'].items():
-            if value is not None and self.disqualifier_scores[key] >= value:
-                return False
-        return True
-
     def associate_object(self, session=None):
         """Find or create a new object and associate it with this measurement.
 
@@ -378,8 +375,12 @@ def associate_object(self, session=None):
         If no Object is found, a new one is created, and its coordinates will be identical
         to those of this Measurements object.
 
-        This should only be done for measurements that have passed all preliminary cuts,
-        which mostly rules out obvious artefacts.
+        This should only be done for measurements that have passed deletion_threshold 
+        preliminary cuts, which mostly rules out obvious artefacts. However, measurements
+        which passed the deletion_threshold cuts but failed the threshold cuts should still
+        be allowed to use this method - in this case, they will create an object with
+        attribute is_bad set to True so they are available to review in the db.
+        
         """
         from models.object import Object  # avoid circular import
 
@@ -392,12 +393,14 @@ def associate_object(self, session=None):
                     radunit='arcsec',
                 ),
                 Object.is_test.is_(self.provenance.is_testing),  # keep testing sources separate
+                Object.is_bad.is_(self.is_bad),    # keep good objects with good measurements
             )).first()
 
             if obj is None:  # no object exists, make one based on these measurements
                 obj = Object(
                     ra=self.ra,
                     dec=self.dec,
+                    is_bad=self.is_bad
                 )
                 obj.is_test = self.provenance.is_testing
 
@@ -488,7 +491,7 @@ def get_upstreams(self, session=None):
         """Get the image that was used to make this source list. """
         with SmartSession(session) as session:
             return session.scalars(sa.select(Cutouts).where(Cutouts.id == self.cutouts_id)).all()
-        
+
     def get_downstreams(self, session=None):
         """Get the downstreams of this Measurements"""
         return []
diff --git a/models/object.py b/models/object.py
index c164a6b7..dcb38a7c 100644
--- a/models/object.py
+++ b/models/object.py
@@ -41,6 +41,13 @@ class Object(Base, AutoIDMixin, SpatiallyIndexed):
         doc='Boolean flag to indicate if the object is a fake object that has been artificially injected. '
     )
 
+    is_bad = sa.Column(
+        sa.Boolean,
+        nullable=False,
+        index=True,
+        doc='Boolean flag to indicate if the object is associated with measurements marked "bad". '
+    )
+
     measurements = orm.relationship(
         Measurements,
         back_populates='object',
diff --git a/pipeline/measuring.py b/pipeline/measuring.py
index 9e264eec..b8aa36eb 100644
--- a/pipeline/measuring.py
+++ b/pipeline/measuring.py
@@ -113,8 +113,17 @@ def __init__(self, **kwargs):
                 'bad_flag': 1,
             },
             dict,
-            'Thresholds for the disqualifier scores. '
-            'If the score is higher than (or equal to) the threshold, the measurement is disqualified. '
+            'Failure thresholds for the disqualifier scores. '
+            'If the score is higher than (or equal to) the threshold, the measurement is marked as bad. '
+        )
+
+        self.deletion_thresholds = self.add_par(
+            'deletion_thresholds',
+            None,
+            (dict, None),
+            'Deletion thresholds for the disqualifier scores. '
+            'If the score is higher than (or equal to) the threshold, the measurement is not saved. ',
+            critical=False
         )
 
         self.association_radius = self.add_par(
@@ -355,7 +364,9 @@ def run(self, *args, **kwargs):
 
                 saved_measurements = []
                 for m in measurements_list:
-                    if m.passes():  # all disqualifiers are below threshold
+                    threshold_comparison = self.compare_measurement_to_thresholds(m)
+                    if threshold_comparison != "delete":  # all disqualifiers are below threshold
+                        m.is_bad = threshold_comparison == "bad"
                         saved_measurements.append(m)
 
                 # add the resulting measurements to the data store
@@ -414,3 +425,32 @@ def make_filter_bank(self, imsize, psf_fwhm):
         self._filter_bank = templates
         self._filter_psf_fwhm = psf_fwhm
 
+    def compare_measurement_to_thresholds(self, m):
+        """Compare measurement disqualifiers of a Measurements object to the thresholds set for 
+        this measurer object.
+
+        Inputs:
+          - m : a Measurements object to be compared
+
+        returns one of three strings to indicate the result
+          - "ok"     : All disqualifiers below both thresholds
+          - "bad"    : Some disqualifiers above mark_thresh but all 
+                       below deletion_thresh
+          - "delete" : Some disqualifiers above deletion_thresh
+        
+        """
+        passing_status = "ok"
+
+        mark_thresh = m.provenance.parameters["thresholds"] # thresholds above which measurement is marked 'bad'
+        deletion_thresh = ( mark_thresh if self.pars.deletion_thresholds is None
+                           else self.pars.deletion_thresholds )
+
+        combined_keys = np.unique(list(mark_thresh.keys()) + list(deletion_thresh.keys())) # unique keys from both
+        for key in combined_keys:
+            if deletion_thresh.get(key) is not None and m.disqualifier_scores[key] >= deletion_thresh[key]:
+                passing_status =  "delete"
+                break
+            if mark_thresh.get(key) is not None and m.disqualifier_scores[key] >= mark_thresh[key]:
+                passing_status = "bad" # no break because another key could trigger "delete"
+
+        return passing_status
diff --git a/tests/fixtures/pipeline_objects.py b/tests/fixtures/pipeline_objects.py
index 04a8ab86..99c20733 100644
--- a/tests/fixtures/pipeline_objects.py
+++ b/tests/fixtures/pipeline_objects.py
@@ -747,7 +747,14 @@ def make_datastore(
                 ds.all_measurements = Measurements.copy_list_from_cache(cache_dir, cache_name)
                 [setattr(m, 'provenance', prov) for m in ds.all_measurements]
                 [setattr(m, 'cutouts', c) for m, c in zip(ds.all_measurements, ds.cutouts)]
-                ds.measurements = [m for m in ds.all_measurements if m.passes()]
+
+                ds.measurements = []
+                for m in ds.all_measurements:
+                    threshold_comparison = p.measurer.compare_measurement_to_thresholds(m)
+                    if threshold_comparison != "delete":  # all disqualifiers are below threshold
+                        m.is_bad = threshold_comparison == "bad"
+                        ds.measurements.append(m)
+
                 [m.associate_object(session) for m in ds.measurements]  # create or find an object for each measurement
                 # no need to save list because Measurements is not a FileOnDiskMixin!
             else:  # cannot find measurements on cache
diff --git a/tests/fixtures/simulated.py b/tests/fixtures/simulated.py
index b0016a0e..97e7c61b 100644
--- a/tests/fixtures/simulated.py
+++ b/tests/fixtures/simulated.py
@@ -627,7 +627,9 @@ def sim_lightcurves(sim_sub_image_list, measurer):
     # a nested list of measurements, each one for a different part of the images,
     # for each image contains a list of measurements for the same source
     measurer.pars.thresholds['bad pixels'] = 100  # avoid losing measurements to random bad pixels
+    measurer.pars.deletion_thresholds['bad pixels'] = 100
     measurer.pars.thresholds['offsets'] = 10  # avoid losing measurements to random offsets
+    measurer.pars.deletion_thresholds['offsets'] = 10
     measurer.pars.association_radius = 5.0  # make it harder for random offsets to dis-associate the measurements
     lightcurves = []
 
@@ -644,4 +646,3 @@ def sim_lightcurves(sim_sub_image_list, measurer):
     yield lightcurves
 
     # no cleanup for this one
-
diff --git a/tests/models/test_measurements.py b/tests/models/test_measurements.py
index fced6bce..ea2912d2 100644
--- a/tests/models/test_measurements.py
+++ b/tests/models/test_measurements.py
@@ -170,6 +170,75 @@ def test_measurements_cannot_be_saved_twice(ptf_datastore):
                 session.delete(m2)
                 session.commit()
 
+def test_threshold_flagging(ptf_datastore, measurer):
+
+    measurements = ptf_datastore.measurements
+    m = measurements[0]  # grab the first one as an example
+
+    m.provenance.parameters['thresholds']['negatives'] = 0.3
+    measurer.pars.deletion_thresholds['negatives'] = 0.5
+
+    m.disqualifier_scores['negatives'] = 0.1 # set a value that will pass both
+    assert measurer.compare_measurement_to_thresholds(m) == "ok"
+
+    m.disqualifier_scores['negatives'] = 0.4 # set a value that will fail one
+    assert measurer.compare_measurement_to_thresholds(m) == "bad"
+
+    m.disqualifier_scores['negatives'] = 0.6 # set a value that will fail both
+    assert measurer.compare_measurement_to_thresholds(m) == "delete"
+
+    # test what happens if we set deletion_thresholds to unspecified
+    #   This should not test at all for deletion
+    measurer.pars.deletion_thresholds = {}
+
+    m.disqualifier_scores['negatives'] = 0.1 # set a value that will pass
+    assert measurer.compare_measurement_to_thresholds(m) == "ok"
+
+    m.disqualifier_scores['negatives'] = 0.8 # set a value that will fail
+    assert measurer.compare_measurement_to_thresholds(m) == "bad"
+
+    # test what happens if we set deletion_thresholds to None
+    #   This should set the deletion threshold same as threshold
+    measurer.pars.deletion_thresholds = None
+    m.disqualifier_scores['negatives'] = 0.1 # set a value that will pass
+    assert measurer.compare_measurement_to_thresholds(m) == "ok"
+
+    m.disqualifier_scores['negatives'] = 0.4 # a value that would fail mark
+    assert measurer.compare_measurement_to_thresholds(m) == "delete"
+
+    m.disqualifier_scores['negatives'] = 0.9 # a value that would fail both (earlier)
+    assert measurer.compare_measurement_to_thresholds(m) == "delete"
+
+def test_deletion_thresh_is_non_critical(ptf_datastore, measurer):
+
+    # hard code in the thresholds to ensure no problems arise
+    # if the defaults for testing change
+    measurer.pars.threshold = {
+                'negatives': 0.3,
+                'bad pixels': 1,
+                'offsets': 5.0,
+                'filter bank': 1,
+                'bad_flag': 1,
+            }
+
+    measurer.pars.deletion_threshold = {
+                'negatives': 0.3,
+                'bad pixels': 1,
+                'offsets': 5.0,
+                'filter bank': 1,
+                'bad_flag': 1,
+            }
+
+    ds1 = measurer.run(ptf_datastore.cutouts)
+
+    # This run should behave identical to the above
+    measurer.pars.deletion_threshold = None
+    ds2 = measurer.run(ptf_datastore.cutouts)
+
+    m1 = ds1.measurements[0]
+    m2 = ds2.measurements[0]
+
+    assert m1.provenance.id == m2.provenance.id
 
 def test_measurements_forced_photometry(ptf_datastore):
     offset_max = 2.0
@@ -192,4 +261,3 @@ def test_measurements_forced_photometry(ptf_datastore):
     # print(f'Flux small aperture: {flux_small_aperture[0]}+-{flux_small_aperture[1]} over area: {flux_small_aperture[2]}')
     # print(f'Flux big aperture: {flux_large_aperture[0]}+-{flux_large_aperture[1]} over area: {flux_large_aperture[2]}')
     # print(f'Flux PSF forced: {flux_psf[0]}+-{flux_psf[1]} over area: {flux_psf[2]}')
-
diff --git a/tests/models/test_objects.py b/tests/models/test_objects.py
index cc38c1d4..2a8056fa 100644
--- a/tests/models/test_objects.py
+++ b/tests/models/test_objects.py
@@ -13,7 +13,7 @@
 
 
 def test_object_creation():
-    obj = Object(ra=1.0, dec=2.0, is_test=True)
+    obj = Object(ra=1.0, dec=2.0, is_test=True, is_bad=False)
     with SmartSession() as session:
         session.add(obj)
         session.commit()
@@ -213,3 +213,48 @@ def test_filtering_measurements_on_object(sim_lightcurves):
         # get the new and only if not found go to the old
         found = obj.get_measurements_list(prov_hash_list=[prov.id, measurements[0].provenance.id])
         assert set([m.id for m in found]) == set(new_id_list)
+
+def test_separate_good_and_bad_objects(measurer, ptf_datastore):
+    measurements = ptf_datastore.measurements
+    m = measurements[0]  # grab the first one as an example
+
+    with SmartSession() as session:
+        m = session.merge(m)
+
+        prov=Provenance(
+            process=m.provenance.process,
+            upstreams=m.provenance.upstreams,
+            code_version=m.provenance.code_version,
+            parameters=m.provenance.parameters.copy(),
+            is_testing=True,
+        )
+        prov.parameters['test_parameter'] = uuid.uuid4().hex
+        prov.update_id()
+        obj1 = session.merge(m.object)
+
+        m2 = Measurements()
+        for key, value in m.__dict__.items():
+            if key not in [
+                '_sa_instance_state',
+                'id',
+                'created_at',
+                'modified',
+                'from_db',
+                'provenance',
+                'provenance_id',
+                'object',
+                'object_id',
+            ]:
+                setattr(m2, key, value)
+        m2.provenance = prov
+        m2.provenance_id = prov.id
+        m2.is_bad = not m.is_bad # flip the is_bad tag
+        m2.associate_object(session)
+        m2 = session.merge(m2)
+        obj2 = session.merge(m2.object)
+
+        # check we got a new obj, proper badness on each, one of each badness
+        assert obj1 is not obj2
+        assert obj1.is_bad == m.is_bad
+        assert obj2.is_bad == m2.is_bad
+        assert not obj1.is_bad == obj2.is_bad

From 5c5218d05f4788f7bada67115c31cc95b1f62e7a Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Tue, 4 Jun 2024 14:52:47 -0700
Subject: [PATCH 07/11] Add the webap used during the stress test (#298)

---
 .gitignore                           |   1 +
 .gitmodules                          |   3 +
 devshell/docker-compose.yaml         |  45 +-
 extern/nersc-desi-gaia-dr3-server    |   2 +-
 models/base.py                       | 239 -----------
 pipeline/astro_cal.py                |   1 -
 pipeline/data_store.py               |   6 +-
 requirements.txt                     |   1 +
 spin/rknop-dev/webap-cert.yaml       |   9 +
 spin/rknop-dev/webap-secrets.yaml    |  15 +
 spin/rknop-dev/webap.yaml            | 145 +++++++
 tests/docker-compose.yaml            |  32 ++
 tests/fixtures/decam.py              |  80 ++--
 tests/fixtures/pipeline_objects.py   | 119 ++++--
 tests/fixtures/ptf.py                | 150 ++++---
 tests/fixtures/ztf.py                |  15 +-
 tests/models/test_decam.py           |  11 +-
 tests/pipeline/test_astro_cal.py     |   7 +-
 tests/pipeline/test_pipeline.py      |  10 +-
 util/cache.py                        | 267 ++++++++++++
 webap/Dockerfile                     |  82 ++++
 webap/Makefile                       |  32 ++
 webap/rkwebutil                      |   1 +
 webap/seechange_webap.py             | 478 ++++++++++++++++++++++
 webap/static/favicon.ico             | Bin 0 -> 2393 bytes
 webap/static/seechange.css           |  87 ++++
 webap/static/seechange.js            | 587 +++++++++++++++++++++++++++
 webap/static/seechange_start.js      |  32 ++
 webap/templates/base.html            |  13 +
 webap/templates/seechange_webap.html |  16 +
 30 files changed, 2085 insertions(+), 401 deletions(-)
 create mode 100644 spin/rknop-dev/webap-cert.yaml
 create mode 100644 spin/rknop-dev/webap-secrets.yaml
 create mode 100644 spin/rknop-dev/webap.yaml
 create mode 100644 util/cache.py
 create mode 100644 webap/Dockerfile
 create mode 100644 webap/Makefile
 create mode 160000 webap/rkwebutil
 create mode 100644 webap/seechange_webap.py
 create mode 100644 webap/static/favicon.ico
 create mode 100644 webap/static/seechange.css
 create mode 100644 webap/static/seechange.js
 create mode 100644 webap/static/seechange_start.js
 create mode 100644 webap/templates/base.html
 create mode 100644 webap/templates/seechange_webap.html

diff --git a/.gitignore b/.gitignore
index c794ff55..729e2594 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@ data/DECam_default_calibrators
 .pytest.ini
 tests/plots
 tests/temp_data
+webap/static/rkwebutil.js
 coadd.weight.fits
 tests/coadd.weight.fits
 
diff --git a/.gitmodules b/.gitmodules
index 46acf546..93b6d37f 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,6 +1,9 @@
 [submodule "extern/nersc-upload-connector"]
 	path = extern/nersc-upload-connector
 	url = https://github.com/c3-time-domain/nersc-upload-connector
+[submodule "webap/rkwebutil"]
+	path = webap/rkwebutil
+	url = https://github.com/rknop/rkwebutil.git
 [submodule "extern/nersc-desi-gaia-dr3-server"]
 	path = extern/nersc-desi-gaia-dr3-server
 	url = https://github.com/c3-time-domain/nersc-desi-gaia-dr3-server.git
diff --git a/devshell/docker-compose.yaml b/devshell/docker-compose.yaml
index 68556c8a..5a8447f8 100644
--- a/devshell/docker-compose.yaml
+++ b/devshell/docker-compose.yaml
@@ -2,7 +2,7 @@ version: "3.3"
 
 services:
   devshell_make-archive-directories:
-    image: rknop/upload-connector:tests
+    image: rknop/upload-connector:${IMGTAG:-devshell}
     build:
       context: ../extern/nersc-upload-connector
       args:
@@ -18,7 +18,7 @@ services:
     depends_on:
       devshell_make-archive-directories:
         condition: service_completed_successfully
-    image: rknop/upload-connector:tests
+    image: rknop/upload-connector:${IMGTAG:-devshell}
     build:
       context: ../extern/nersc-upload-connector
       args:
@@ -45,13 +45,14 @@ services:
     user: ${USERID:?err}:${GROUPID:?err}
 
   devshell_seechange_postgres:
-    image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange-postgres
+    image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange-postgres:${IMGTAG:-devshell}
     build:
       context: ../docker/postgres
     volumes:
       - seechange-devshell-postgres-dbdata:/var/lib/postgresql/data
     environment:
       POSTGRES_DATA_DIR: /var/lib/postgresql/data
+      POSTGRES_PASSWORD: fragile
     healthcheck:
       test: netcat -w 1 localhost 5432 || exit 1
       interval: 5s
@@ -59,7 +60,7 @@ services:
       retries: 5
 
   devshell_setuptables:
-    image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange
+    image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-devshell}
     build:
       context: ../
       dockerfile: ./docker/application/Dockerfile
@@ -76,8 +77,34 @@ services:
     user: ${USERID:?err}:${GROUPID:?err}
     entrypoint: [ "alembic", "upgrade", "head" ]
 
+  devshell_webap:
+    depends_on:
+      devshell_setuptables:
+        condition: service_completed_successfully
+      devshell_make-archive-directories:
+        condition: service_completed_successfully
+    image: gchr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange-webap:${IMGTAG:-devshell}
+    build:
+      context: ../webap
+    user: ${USERID:-0}:${GROUPID:-0}
+    ports:
+      - "8081:8081"
+    healthcheck:
+      test: netcat -w 1 localhost 8081
+      interval: 5s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - type: volume
+        source: devshell-archive-storage
+        target: /archive-storage
+      - type: bind
+        source: ../tests/webap_secrets
+        target: /secrets
+    entrypoint: [ "gunicorn", "-w", "4", "-b", "0.0.0.0:8081", "--timeout", "0", "seechange_webap:app" ]
+
   devshell_make_data_dir:
-    image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange
+    image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-devshell}
     build:
       context: ../
       dockerfile: ./docker/application/Dockerfile
@@ -90,7 +117,7 @@ services:
     entrypoint: bash -c "mkdir -p /seechange/devshell/temp_data"
 
   seechange:
-    image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange
+    image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-devshell}
     build:
       context: ../
       dockerfile: ./docker/application/Dockerfile
@@ -103,6 +130,8 @@ services:
         condition: service_healthy
       devshell_make_data_dir:
         condition: service_completed_successfully
+      devshell_webap:
+        condition: service_healthy
     volumes:
       - type: bind
         source: ..
@@ -115,7 +144,7 @@ services:
     entrypoint: [ "tail", "-f", "/etc/issue" ]
 
   seechange_nodb:
-    image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange
+    image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-devshell}
     build:
       context: ../
       dockerfile: ./docker/application/Dockerfile
@@ -133,7 +162,7 @@ services:
     entrypoint: [ "tail", "-f", "/etc/issue" ]
 
   seechange_nomigrate:
-    image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange
+    image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-devshell}
     build:
       context: ../
       dockerfile: ./docker/application/Dockerfile
diff --git a/extern/nersc-desi-gaia-dr3-server b/extern/nersc-desi-gaia-dr3-server
index c71420c6..21ef7c1f 160000
--- a/extern/nersc-desi-gaia-dr3-server
+++ b/extern/nersc-desi-gaia-dr3-server
@@ -1 +1 @@
-Subproject commit c71420c65b64c32d5cf2e40564c140470494b21a
+Subproject commit 21ef7c1f0a71e795165346f180ad7e29d82e4756
diff --git a/models/base.py b/models/base.py
index a9fd7a62..cb42e354 100644
--- a/models/base.py
+++ b/models/base.py
@@ -499,245 +499,6 @@ def to_json(self, filename):
             except:
                 raise
 
-    def copy_to_cache(self, cache_dir, filepath=None):
-        """Save a copy of the object (and associated files) into a cache directory.
-
-        If the object is a FileOnDiskMixin, then the file(s) pointed by get_fullpath()
-        will be copied to the cache directory with their original names,
-        unless filepath is specified, in which case the cached files will
-        have a different name than the files in the data folder (and the database filepath).
-        The filepath (with optional first extension) will be used to create a JSON file
-        which holds the object's column attributes (i.e., only those that are
-        database persistent).
-
-        If caching a non-FileOnDiskMixin object, the filepath argument must be given,
-        because it is used to name the JSON file.
-
-        Parameters
-        ----------
-        cache_dir: str or path
-            The path to the cache directory.
-        filepath: str or path (optional)
-            Must be given if the object is not a FileOnDiskMixin.
-            If it is a FileOnDiskMixin, it will be used to name
-            the data files and the JSON file in the cache folder.
-
-        Returns
-        -------
-        str
-            The full path to the output json file.
-        """
-        if filepath is not None and filepath.endswith('.json'):  # remove .json if it exists
-            filepath = filepath[:-5]
-
-        json_filepath = filepath
-        if not isinstance(self, FileOnDiskMixin):
-            if filepath is None:
-                raise ValueError("filepath must be given when caching a non FileOnDiskMixin object")
-
-        else:  # it is a FileOnDiskMixin
-            if filepath is None:  # use the FileOnDiskMixin filepath as default
-                filepath = self.filepath  # use this filepath for the data files
-                json_filepath = self.filepath  # use the same filepath for the json file too
-            if (
-                    self.filepath_extensions is not None and
-                    len(self.filepath_extensions) > 0 and
-                    not json_filepath.endswith(self.filepath_extensions[0])
-            ):
-                    json_filepath += self.filepath_extensions[0]  # only append this extension to the json filename
-
-            for i, source_f in enumerate(self.get_fullpath(as_list=True)):
-                if source_f is None:
-                    continue
-                target_f = os.path.join(cache_dir, filepath)
-                if self.filepath_extensions is not None and i < len(self.filepath_extensions):
-                    target_f += self.filepath_extensions[i]
-                SCLogger.debug(f"Copying {source_f} to {target_f}")
-                os.makedirs(os.path.dirname(target_f), exist_ok=True)
-                shutil.copy2(source_f, target_f)
-
-        # attach the cache_dir and the .json extension if needed
-        json_filepath = os.path.join(cache_dir, json_filepath)
-        if not json_filepath.endswith('.json'):
-            json_filepath += '.json'
-        self.to_json(json_filepath)
-
-        return json_filepath
-
-    @classmethod
-    def copy_list_to_cache(cls, obj_list, cache_dir, filepath=None):
-        """Copy a list of objects to the cache directory.
-
-        The first object on the list will be used to copy any associated files
-        (if it is a FileOnDiskMixin). The filepath argument must be given
-        if the objects are not FileOnDiskMixin.
-        The type and filepath of all objects on the list must be the same!
-
-        The object's column data is saved into the JSON file as a list of dictionaries.
-
-        Parameters
-        ----------
-        obj_list: list
-            A list of objects to save to the cache directory.
-        cache_dir: str or path
-            The path to the cache directory.
-        filepath: str or path (optional)
-            Must be given if the objects are not FileOnDiskMixin.
-            If it is a FileOnDiskMixin, it will be used to name
-            the data files and the JSON file in the cache folder.
-
-        Returns
-        -------
-        str
-            The full path to the output JSON file.
-        """
-        if len(obj_list) == 0:
-            if filepath is None:
-                return  # can't do anything without a filepath
-            json_filepath = os.path.join(cache_dir, filepath)
-            if not json_filepath.endswith('.json'):
-                json_filepath += '.json'
-        else:
-            types = set([type(obj) for obj in obj_list])
-            if len(types) != 1:
-                raise ValueError("All objects must be of the same type!")
-
-            filepaths = set([getattr(obj, 'filepath', None) for obj in obj_list])
-            if len(filepaths) != 1:
-                raise ValueError("All objects must have the same filepath!")
-
-            # save the JSON file and copy associated files
-            json_filepath = obj_list[0].copy_to_cache(cache_dir, filepath=filepath)
-
-        # overwrite the JSON file with the list of dictionaries
-        with open(json_filepath, 'w') as fp:
-            json.dump([obj.to_dict() for obj in obj_list], fp, indent=2)
-
-        return json_filepath
-
-    @classmethod
-    def copy_from_cache(cls, cache_dir, filepath):
-        """Copy and reconstruct an object from the cache directory.
-
-        Will need the JSON file that contains all the column attributes of the file.
-        Once those are successfully loaded, and if the object is a FileOnDiskMixin,
-        it will be able to figure out where all the associated files are saved
-        based on the filepath and extensions in the JSON file.
-        Those files will be copied into the current data directory
-        (i.e., that pointed to by FileOnDiskMixin.local_path).
-        The reconstructed object should be correctly associated
-        with its files but will not necessarily have the correct
-        relationships to other objects.
-
-        Parameters
-        ----------
-        cache_dir: str or path
-            The path to the cache directory.
-        filepath: str or path
-            The name of the JSON file that holds the column attributes.
-
-        Returns
-        -------
-        output: SeeChangeBase
-            The reconstructed object, of the same type as the class.
-        """
-        # allow user to give an absolute path, so long as it is in the cache dir
-        if filepath.startswith(cache_dir):
-            filepath = filepath[len(cache_dir) + 1:]
-
-        # allow the user to give the filepath with or without the .json extension
-        if filepath.endswith('.json'):
-            filepath = filepath[:-5]
-
-        full_path = os.path.join(cache_dir, filepath)
-        with open(full_path + '.json', 'r') as fp:
-            json_dict = json.load(fp)
-
-        output = cls.from_dict(json_dict)
-
-        # copy any associated files
-        if isinstance(output, FileOnDiskMixin):
-            # if fullpath ends in filepath_extensions[0]
-            if (
-                    output.filepath_extensions is not None and
-                    output.filepath_extensions[0] is not None and
-                    full_path.endswith(output.filepath_extensions[0])
-            ):
-                full_path = full_path[:-len(output.filepath_extensions[0])]
-
-            for i, target_f in enumerate(output.get_fullpath(as_list=True)):
-                if target_f is None:
-                    continue
-                source_f = os.path.join(cache_dir, full_path)
-                if output.filepath_extensions is not None and i < len(output.filepath_extensions):
-                    source_f += output.filepath_extensions[i]
-                SCLogger.debug(f"Copying {source_f} to {target_f}")
-                os.makedirs(os.path.dirname(target_f), exist_ok=True)
-                shutil.copyfile(source_f, target_f)
-
-        return output
-
-    @classmethod
-    def copy_list_from_cache(cls, cache_dir, filepath):
-        """Copy and reconstruct a list of objects from the cache directory.
-
-        Will need the JSON file that contains all the column attributes of the file.
-        Once those are successfully loaded, and if the object is a FileOnDiskMixin,
-        it will be able to figure out where all the associated files are saved
-        based on the filepath and extensions in the JSON file.
-
-        Parameters
-        ----------
-        cache_dir: str or path
-            The path to the cache directory.
-        filepath: str or path
-            The name of the JSON file that holds the column attributes.
-
-        Returns
-        -------
-        output: list
-            The list of reconstructed objects, of the same type as the class.
-        """
-        # allow user to give an absolute path, so long as it is in the cache dir
-        if filepath.startswith(cache_dir):
-            filepath = filepath[len(cache_dir) + 1:]
-
-        # allow the user to give the filepath with or without the .json extension
-        if filepath.endswith('.json'):
-            filepath = filepath[:-5]
-
-        full_path = os.path.join(cache_dir, filepath)
-        with open(full_path + '.json', 'r') as fp:
-            json_list = json.load(fp)
-
-        output = []
-        for obj_dict in json_list:
-            output.append(cls.from_dict(obj_dict))
-
-        if len(output) == 0:
-            return []
-
-        if isinstance(output[0], FileOnDiskMixin):
-            # if fullpath ends in filepath_extensions[0]
-            if (
-                    output[0].filepath_extensions is not None and
-                    output[0].filepath_extensions[0] is not None and
-                    full_path.endswith(output[0].filepath_extensions[0])
-            ):
-                full_path = full_path[:-len(output[0].filepath_extensions[0])]
-
-            for i, target_f in enumerate(output[0].get_fullpath(as_list=True)):
-                if target_f is None:
-                    continue
-                source_f = os.path.join(cache_dir, full_path)
-                if output[0].filepath_extensions is not None and i < len(output[0].filepath_extensions):
-                    source_f += output[0].filepath_extensions[i]
-                SCLogger.debug(f"Copying {source_f} to {target_f}")
-                os.makedirs(os.path.dirname(target_f), exist_ok=True)
-                shutil.copyfile(source_f, target_f)
-
-        return output
-
 
 Base = declarative_base(cls=SeeChangeBase)
 
diff --git a/pipeline/astro_cal.py b/pipeline/astro_cal.py
index 7d12e761..3106eeda 100644
--- a/pipeline/astro_cal.py
+++ b/pipeline/astro_cal.py
@@ -5,7 +5,6 @@
 import improc.scamp
 
 from util.exceptions import CatalogNotFoundError, SubprocessFailure, BadMatchException
-
 from util.logger import SCLogger
 from util.util import parse_bool
 
diff --git a/pipeline/data_store.py b/pipeline/data_store.py
index 14f589a3..5c5d227b 100644
--- a/pipeline/data_store.py
+++ b/pipeline/data_store.py
@@ -3,6 +3,7 @@
 import sqlalchemy as sa
 
 from util.util import get_latest_provenance, parse_session
+from util.logger import SCLogger
 
 from models.base import SmartSession, FileOnDiskMixin
 from models.provenance import CodeVersion, Provenance
@@ -16,8 +17,6 @@
 from models.cutouts import Cutouts
 from models.measurements import Measurements
 
-from util.logger import SCLogger
-
 # for each process step, list the steps that go into its upstream
 UPSTREAM_STEPS = {
     'exposure': [],  # no upstreams
@@ -1070,6 +1069,9 @@ def get_reference(self, minovfrac=0.85, must_match_instrument=True, must_match_f
         ref: Image object
             The reference image for this image, or None if no reference is found.
 
+        It will only return references whose validity date range
+        includes DataStore.image.observation_time.
+
         If minovfrac is given, it will return the reference that has the
         highest ovfrac.  (If, by unlikely chance, more than one have
         identical overlap fractions, an undeterministically chosen
diff --git a/requirements.txt b/requirements.txt
index 7125174d..cf8a1cb3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,6 +17,7 @@ psutil==5.9.8
 psycopg2==2.9.9
 pylandau==2.2.1
 pytest==7.4.3
+pytest-timestamper==0.0.10
 python-dateutil==2.8.2
 pytz==2022.7.1
 PyYAML==6.0.1
diff --git a/spin/rknop-dev/webap-cert.yaml b/spin/rknop-dev/webap-cert.yaml
new file mode 100644
index 00000000..379890d9
--- /dev/null
+++ b/spin/rknop-dev/webap-cert.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+data:
+  tls.crt: PUT THE RIGHT THING HERE
+  tls.key: PUT THE RIGHT THING HERE
+kind: Secret
+metadata:
+  name: ls4-rknop-dev-webap-cert
+  namespace: ls4-rknop-dev
+type: kubernetes.io/tls
diff --git a/spin/rknop-dev/webap-secrets.yaml b/spin/rknop-dev/webap-secrets.yaml
new file mode 100644
index 00000000..fc46b3a5
--- /dev/null
+++ b/spin/rknop-dev/webap-secrets.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+stringData:
+  seechange_webap_config.py: |
+    import pathlib
+    PG_HOST = 'decatdb.lbl.gov'
+    PG_PORT = 5432
+    PG_USER = 'ls4_rknop_dev'
+    PG_PASS = PUT THE RIGHT THING HERE
+    PG_NAME = 'seechange_rknop_dev'
+    ARCHIVE_DIR = pathlib.Path( '/archive/base' )
+kind: Secret
+metadata:
+  name: webap-secrets
+  namespace: ls4-rknop-dev
+type: Opaque
diff --git a/spin/rknop-dev/webap.yaml b/spin/rknop-dev/webap.yaml
new file mode 100644
index 00000000..42e07307
--- /dev/null
+++ b/spin/rknop-dev/webap.yaml
@@ -0,0 +1,145 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    workload.user.cattle.io/workloadselector: deployment-ls4-rknop-dev-webap
+  name: webap
+  namespace: ls4-rknop-dev
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      workload.user.cattle.io/workloadselector: deployment-ls4-rknop-dev-webap
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        workload.user.cattle.io/workloadselector: deployment-ls4-rknop-dev-webap
+    spec:
+      containers:
+      - image: registry.nersc.gov/m4616/seechange-webap:rknop-dev
+        imagePullPolicy: Always
+        name: seechange-webap
+        resources: {}
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - ALL
+          privileged: false
+          readOnlyRootFilesystem: false
+          runAsNonRoot: true
+          runAsUser: 95089
+        stdin: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        tty: true
+        volumeMounts:
+        - mountPath: /secrets
+          name: webap-secrets
+        - mountPath: /archive
+          name: seechange-archive-dir
+        # Comment the next two lines out to use the code baked into the Dockerfile
+        # - mountPath: /code
+        #   name: seechange-webap-code
+      dnsConfig: {}
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext:
+        fsGroup: 103988
+      terminationGracePeriodSeconds: 30
+      imagePullSecrets:
+        - name: registry-nersc
+      volumes:
+      - name: webap-secrets
+        secret:
+          defaultMode: 256
+          optional: false
+          secretName: webap-secrets
+      - hostPath:
+          path: /global/cfs/cdirs/m4616/users/raknop/SeeChange-rknop-dev-webap-install
+          type: Directory
+        name: seechange-webap-code
+      - hostPath:
+          path: /global/cfs/cdirs/m4616/archive-rknop-dev
+          type: Directory
+        name: seechange-archive-dir
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: webap
+  namespace: ls4-rknop-dev
+spec:
+  clusterIP: None
+  clusterIPs:
+  - None
+  ports:
+  - name: default
+    port: 42
+    protocol: TCP
+    targetPort: 42
+  selector:
+    workload.user.cattle.io/workloadselector: deployment-ls4-rknop-dev-webap
+  sessionAffinity: None
+  type: ClusterIP
+status:
+  loadBalancer: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: webap-ingress
+  namespace: ls4-rknop-dev
+spec:
+  ports:
+  - port: 8080
+    protocol: TCP
+    targetPort: 8080
+  selector:
+    workload.user.cattle.io/workloadselector: deployment-ls4-rknop-dev-webap
+  sessionAffinity: None
+  type: ClusterIP
+status:
+  loadBalancer: {}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: webap
+  namespace: ls4-rknop-dev
+spec:
+  rules:
+  - host: webap.ls4-rknop-dev.production.svc.spin.nersc.org
+    http:
+      paths:
+      - backend:
+          service:
+            name: webap-ingress
+            port:
+              number: 8080
+        pathType: ImplementationSpecific
+  - host: ls4-rknop-dev-webap.lbl.gov
+    http:
+      paths:
+      - backend:
+          service:
+            name: webap-ingress
+            port:
+              number: 8080
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - ls4-rknop-dev-webap.lbl.gov
+    secretName: ls4-rknop-dev-webap-cert
+---
diff --git a/tests/docker-compose.yaml b/tests/docker-compose.yaml
index f2e6a938..1c456bd5 100644
--- a/tests/docker-compose.yaml
+++ b/tests/docker-compose.yaml
@@ -75,6 +75,32 @@ services:
     user: ${USERID:-0}:${GROUPID:-0}
     entrypoint: [ "alembic", "upgrade", "head" ]
 
+  webap:
+    depends_on:
+      setuptables:
+        condition: service_completed_successfully
+      make-archive-directories:
+        condition: service_completed_successfully
+    image: gchr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange-webap:${IMGTAG:-tests}
+    build:
+      context: ../webap
+    user: ${USERID:-0}:${GROUPID:-0}
+    ports:
+      - "8081:8081"
+    healthcheck:
+      test: netcat -w 1 localhost 8081
+      interval: 5s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - type: volume
+        source: archive-storage
+        target: /archive-storage
+      - type: bind
+        source: webap_secrets
+        target: /secrets
+    entrypoint: [ "gunicorn", "-w", "4", "-b", "0.0.0.0:8081", "--timeout", "0", "seechange_webap:app" ]
+
   runtests:
     image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-c3-time-domain}/seechange:${IMGTAG:-tests}
     build:
@@ -89,6 +115,8 @@ services:
         condition: service_completed_successfully
       archive:
         condition: service_healthy
+      # webap:
+      #   condition: service_healthy
     volumes:
       - type: bind
         source: ..
@@ -114,6 +142,8 @@ services:
         condition: service_completed_successfully
       archive:
         condition: service_healthy
+      # webap:
+      #   condition: service_healthy
     volumes:
       - type: bind
         source: ..
@@ -138,6 +168,8 @@ services:
         condition: service_completed_successfully
       archive:
         condition: service_healthy
+      webap:
+        condition: service_healthy
     volumes:
       - type: bind
         source: ..
diff --git a/tests/fixtures/decam.py b/tests/fixtures/decam.py
index 75b070d8..e2c41b8b 100644
--- a/tests/fixtures/decam.py
+++ b/tests/fixtures/decam.py
@@ -25,7 +25,7 @@
 
 from util.retrydownload import retry_download
 from util.logger import SCLogger
-
+from util.cache import copy_to_cache, copy_list_to_cache, copy_from_cache, copy_list_from_cache
 
 @pytest.fixture(scope='session')
 def decam_cache_dir(cache_dir):
@@ -54,12 +54,13 @@ def decam_cache_dir(cache_dir):
 def decam_default_calibrators(cache_dir, data_dir):
     try:
         # try to get the calibrators from the cache folder
-        if os.path.isdir(os.path.join(cache_dir, 'DECam_default_calibrators')):
-            shutil.copytree(
-                os.path.join(cache_dir, 'DECam_default_calibrators'),
-                os.path.join(data_dir, 'DECam_default_calibrators'),
-                dirs_exist_ok=True,
-            )
+        if not os.getenv( "LIMIT_CACHE_USAGE" ):
+            if os.path.isdir(os.path.join(cache_dir, 'DECam_default_calibrators')):
+                shutil.copytree(
+                    os.path.join(cache_dir, 'DECam_default_calibrators'),
+                    os.path.join(data_dir, 'DECam_default_calibrators'),
+                    dirs_exist_ok=True,
+                )
 
         decam = get_instrument_instance( 'DECam' )
         sections = [ 'N1', 'S1' ]
@@ -71,16 +72,17 @@ def decam_default_calibrators(cache_dir, data_dir):
         decam._get_default_calibrator( 60000, sec, calibtype='linearity' )
 
         # store the calibration files in the cache folder
-        if not os.path.isdir(os.path.join(cache_dir, 'DECam_default_calibrators')):
-            os.makedirs(os.path.join(cache_dir, 'DECam_default_calibrators'), exist_ok=True)
-        for folder in os.listdir(os.path.join(data_dir, 'DECam_default_calibrators')):
-            if not os.path.isdir(os.path.join(cache_dir, 'DECam_default_calibrators', folder)):
-                os.makedirs(os.path.join(cache_dir, 'DECam_default_calibrators', folder), exist_ok=True)
-            for file in os.listdir(os.path.join(data_dir, 'DECam_default_calibrators', folder)):
-                shutil.copy2(
-                    os.path.join(data_dir, 'DECam_default_calibrators', folder, file),
-                    os.path.join(cache_dir, 'DECam_default_calibrators', folder, file)
-                )
+        if not os.getenv( "LIMIT_CACHE_USAGE" ):
+            if not os.path.isdir(os.path.join(cache_dir, 'DECam_default_calibrators')):
+                os.makedirs(os.path.join(cache_dir, 'DECam_default_calibrators'), exist_ok=True)
+            for folder in os.listdir(os.path.join(data_dir, 'DECam_default_calibrators')):
+                if not os.path.isdir(os.path.join(cache_dir, 'DECam_default_calibrators', folder)):
+                    os.makedirs(os.path.join(cache_dir, 'DECam_default_calibrators', folder), exist_ok=True)
+                for file in os.listdir(os.path.join(data_dir, 'DECam_default_calibrators', folder)):
+                    shutil.copy2(
+                        os.path.join(data_dir, 'DECam_default_calibrators', folder, file),
+                        os.path.join(cache_dir, 'DECam_default_calibrators', folder, file)
+                    )
 
         yield sections, filters
 
@@ -181,17 +183,21 @@ def decam_filename(download_url, data_dir, decam_cache_dir):
     """
     base_name = 'c4d_221104_074232_ori.fits.fz'
     filename = os.path.join(data_dir, base_name)
+    os.makedirs(os.path.dirname(filename), exist_ok=True)
+    url = os.path.join(download_url, 'DECAM', base_name)
+
     if not os.path.isfile(filename):
-        cachedfilename = os.path.join(decam_cache_dir, base_name)
-        os.makedirs(os.path.dirname(cachedfilename), exist_ok=True)
+        if os.getenv( "LIMIT_CACHE_USAGE" ):
+            wget.download( url=url, out=filename )
+        else:
+            cachedfilename = os.path.join(decam_cache_dir, base_name)
+            os.makedirs(os.path.dirname(cachedfilename), exist_ok=True)
 
-        if not os.path.isfile(cachedfilename):
-            url = os.path.join(download_url, 'DECAM', base_name)
-            response = wget.download(url=url, out=cachedfilename)
-            assert response == cachedfilename
+            if not os.path.isfile(cachedfilename):
+                response = wget.download(url=url, out=cachedfilename)
+                assert response == cachedfilename
 
-        os.makedirs(os.path.dirname(filename), exist_ok=True)
-        shutil.copy2(cachedfilename, filename)
+            shutil.copy2(cachedfilename, filename)
 
     yield filename
 
@@ -262,7 +268,8 @@ def decam_datastore(
         decam_exposure,
         'N1',
         cache_dir=decam_cache_dir,
-        cache_base_name='115/c4d_20221104_074232_N1_g_Sci_FVOSOC'
+        cache_base_name='115/c4d_20221104_074232_N1_g_Sci_FVOSOC',
+        save_original_image=True
     )
     # This save is redundant, as the datastore_factory calls save_and_commit
     # However, I leave this here because it is a good test that calling it twice
@@ -287,6 +294,9 @@ def decam_datastore(
         if obj is not None and hasattr(obj, 'delete_from_disk_and_database'):
             obj.delete_from_disk_and_database(archive=True)
 
+    # Because save_original_image was True in the call to datastore_factory above
+    os.unlink( ds.path_to_original_image )
+
     ImageAligner.cleanup_temp_images()
 
 
@@ -311,6 +321,11 @@ def decam_fits_image_filename(download_url, decam_cache_dir):
 
     yield filename
 
+    if os.getenv( "LIMIT_CACHE_USAGE" ):
+        try:
+            os.unlink( filepath )
+        except FileNotFoundError:
+            pass
 
 @pytest.fixture
 def decam_fits_image_filename2(download_url, decam_cache_dir):
@@ -324,6 +339,11 @@ def decam_fits_image_filename2(download_url, decam_cache_dir):
 
     yield filename
 
+    if os.getenv( "LIMIT_CACHE_USAGE" ):
+        try:
+            os.unlink( filepath )
+        except FileNotFoundError:
+            pass
 
 @pytest.fixture
 def decam_ref_datastore( code_version, download_url, decam_cache_dir, data_dir, datastore_factory ):
@@ -351,7 +371,10 @@ def decam_ref_datastore( code_version, download_url, decam_cache_dir, data_dir,
         if not ext.endswith('.yaml'):
             destination = os.path.join(data_dir, f'115/{filebase}{ext}')
             os.makedirs(os.path.dirname(destination), exist_ok=True)
-            shutil.copy2( cache_path, destination )
+            if os.getenv( "LIMIT_CACHE_USAGE" ):
+                shutil.move( cache_path, destination )
+            else:
+                shutil.copy2( cache_path, destination )
 
     yaml_path = os.path.join(decam_cache_dir, f'115/{filebase}.image.yaml')
 
@@ -388,7 +411,8 @@ def decam_ref_datastore( code_version, download_url, decam_cache_dir, data_dir,
         image.is_coadd = True
         image.save(verify_md5=False)  # make sure to upload to archive as well
 
-        image.copy_to_cache( decam_cache_dir )
+        if not os.getenv( "LIMIT_CACHE_USAGE" ):
+            copy_to_cache( image, decam_cache_dir )
 
         ds = datastore_factory(image, cache_dir=decam_cache_dir, cache_base_name=f'115/{filebase}')
 
diff --git a/tests/fixtures/pipeline_objects.py b/tests/fixtures/pipeline_objects.py
index 99c20733..94c4aea1 100644
--- a/tests/fixtures/pipeline_objects.py
+++ b/tests/fixtures/pipeline_objects.py
@@ -9,7 +9,7 @@
 
 import sep
 
-from models.base import SmartSession
+from models.base import SmartSession, FileOnDiskMixin
 from models.provenance import Provenance
 from models.enums_and_bitflags import BitFlagConverter
 from models.image import Image
@@ -32,10 +32,10 @@
 from pipeline.top_level import Pipeline
 
 from util.logger import SCLogger
+from util.cache import copy_to_cache, copy_list_to_cache, copy_from_cache, copy_list_from_cache
 
 from improc.bitmask_tools import make_saturated_flag
 
-
 @pytest.fixture(scope='session')
 def preprocessor_factory(test_config):
 
@@ -266,11 +266,18 @@ def datastore_factory(data_dir, pipeline_factory):
     simply change the pipeline object's "test_parameter" value to a unique
     new value, so the provenance will not match and the data will be regenerated.
 
+    If "save_original_image" is True, then a copy of the image before
+    going through source extraction, WCS, etc. will be saved along side
+    the image, with ".image.fits.original" appended to the filename;
+    this path will be in ds.path_to_original_image.  In this case, the
+    thing that calls this factory must delete that file when done.
+
     EXAMPLE
     -------
     extractor.pars.test_parameter = uuid.uuid().hex
     extractor.run(datastore)
     assert extractor.has_recalculated is True
+
     """
     def make_datastore(
             *args,
@@ -280,11 +287,12 @@ def make_datastore(
             overrides={},
             augments={},
             bad_pixel_map=None,
+            save_original_image=False
     ):
         code_version = args[0].provenance.code_version
         ds = DataStore(*args)  # make a new datastore
 
-        if cache_dir is not None and cache_base_name is not None:
+        if ( cache_dir is not None ) and ( cache_base_name is not None ) and ( not os.getenv( "LIMIT_CACHE_USE" ) ):
             ds.cache_base_name = os.path.join(cache_dir, cache_base_name)  # save this for testing purposes
 
         p = pipeline_factory()
@@ -299,19 +307,27 @@ def make_datastore(
                 ds.image = ds.image.merge_all(session)
 
             ############ preprocessing to create image ############
-            if ds.image is None and cache_dir is not None and cache_base_name is not None:
+            if (   ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
+                   ( ds.image is None ) and ( cache_dir is not None ) and ( cache_base_name is not None )
+                ):
                 # check if preprocessed image is in cache
                 cache_name = cache_base_name + '.image.fits.json'
                 cache_path = os.path.join(cache_dir, cache_name)
                 if os.path.isfile(cache_path):
                     SCLogger.debug('loading image from cache. ')
-                    ds.image = Image.copy_from_cache(cache_dir, cache_name)
+                    ds.image = copy_from_cache(Image, cache_dir, cache_name)
                     # assign the correct exposure to the object loaded from cache
                     if ds.exposure_id is not None:
                         ds.image.exposure_id = ds.exposure_id
                     if ds.exposure is not None:
                         ds.image.exposure = ds.exposure
 
+                    # Copy the original image from the cache if requested
+                    if save_original_image:
+                        ds.path_to_original_image = ds.image.get_fullpath()[0] + '.image.fits.original'
+                        cache_path = os.path.join(cache_dir, ds.image.filepath + '.image.fits.original')
+                        shutil.copy2( cache_path, ds.path_to_original_image )
+
                     # add the preprocessing steps from instrument (TODO: remove this as part of Issue #142)
                     preprocessing_steps = ds.image.instrument_object.preprocessing_steps
                     prep_pars = p.preprocessor.pars.get_critical_pars()
@@ -358,18 +374,24 @@ def make_datastore(
                 ds.image.flags |= (mask * 2 ** BitFlagConverter.convert('saturated')).astype(np.uint16)
 
                 ds.image.save()
-                output_path = ds.image.copy_to_cache(cache_dir)
-                # also save the original image to the cache as a separate file
-                shutil.copy2(
-                    ds.image.get_fullpath()[0],
-                    os.path.join(cache_dir, ds.image.filepath + '.image.fits.original')
-                )
+                if not os.getenv( "LIMIT_CACHE_USAGE" ):
+                    output_path = copy_to_cache(ds.image, cache_dir)
 
-                if cache_dir is not None and cache_base_name is not None and output_path != cache_path:
-                    warnings.warn(f'cache path {cache_path} does not match output path {output_path}')
-                elif cache_dir is not None and cache_base_name is None:
-                    ds.cache_base_name = output_path
-                    SCLogger.debug(f'Saving image to cache at: {output_path}')
+                    if cache_dir is not None and cache_base_name is not None and output_path != cache_path:
+                        warnings.warn(f'cache path {cache_path} does not match output path {output_path}')
+                    elif cache_dir is not None and cache_base_name is None:
+                        ds.cache_base_name = output_path
+                        SCLogger.debug(f'Saving image to cache at: {output_path}')
+
+                # In test_astro_cal, there's a routine that needs the original
+                # image before being processed through the rest of what this
+                # factory function does, so save it if requested
+                if save_original_image:
+                    ds.path_to_original_image = ds.image.get_fullpath()[0] + '.image.fits.original'
+                    shutil.copy2( ds.image.get_fullpath()[0], ds.path_to_original_image )
+                    if not os.getenv( "LIMIT_CACHE_USAGE" ):
+                        shutil.copy2( ds.image.get_fullpath()[0],
+                                      os.path.join(cache_dir, ds.image.filepath + '.image.fits.original') )
 
             # check if background was calculated
             if ds.image.bkg_mean_estimate is None or ds.image.bkg_rms_estimate is None:
@@ -389,7 +411,9 @@ def make_datastore(
                 ds.image.bkg_rms_estimate = backgrounder.globalrms
 
             ############# extraction to create sources / PSF #############
-            if cache_dir is not None and cache_base_name is not None:
+            if (   ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
+                   ( cache_dir is not None ) and ( cache_base_name is not None )
+                ):
                 # try to get the SourceList from cache
                 prov = Provenance(
                     code_version=code_version,
@@ -403,7 +427,7 @@ def make_datastore(
                 cache_path = os.path.join(cache_dir, cache_name)
                 if os.path.isfile(cache_path):
                     SCLogger.debug('loading source list from cache. ')
-                    ds.sources = SourceList.copy_from_cache(cache_dir, cache_name)
+                    ds.sources = copy_from_cache(SourceList, cache_dir, cache_name)
 
                     # if SourceList already exists on the database, use that instead of this one
                     existing = session.scalars(
@@ -431,7 +455,7 @@ def make_datastore(
                 cache_path = os.path.join(cache_dir, cache_name)
                 if os.path.isfile(cache_path):
                     SCLogger.debug('loading PSF from cache. ')
-                    ds.psf = PSF.copy_from_cache(cache_dir, cache_name)
+                    ds.psf = copy_from_cache(PSF, cache_dir, cache_name)
 
                     # if PSF already exists on the database, use that instead of this one
                     existing = session.scalars(
@@ -458,11 +482,12 @@ def make_datastore(
                 SCLogger.debug('extracting sources. ')
                 ds = p.extractor.run(ds)
                 ds.sources.save()
-                ds.sources.copy_to_cache(cache_dir)
                 ds.psf.save(overwrite=True)
-                output_path = ds.psf.copy_to_cache(cache_dir)
-                if cache_dir is not None and cache_base_name is not None and output_path != cache_path:
-                    warnings.warn(f'cache path {cache_path} does not match output path {output_path}')
+                if not os.getenv( "LIMIT_CACHE_USAGE" ):
+                    copy_to_cache(ds.sources, cache_dir)
+                    output_path = copy_to_cache(ds.psf, cache_dir)
+                    if cache_dir is not None and cache_base_name is not None and output_path != cache_path:
+                        warnings.warn(f'cache path {cache_path} does not match output path {output_path}')
 
             ############## astro_cal to create wcs ################
             if cache_dir is not None and cache_base_name is not None:
@@ -477,7 +502,7 @@ def make_datastore(
                 cache_path = os.path.join(cache_dir, cache_name)
                 if os.path.isfile(cache_path):
                     SCLogger.debug('loading WCS from cache. ')
-                    ds.wcs = WorldCoordinates.copy_from_cache(cache_dir, cache_name)
+                    ds.wcs = copy_from_cache(WorldCoordinates, cache_dir, cache_name)
                     prov = session.merge(prov)
 
                     # check if WCS already exists on the database
@@ -507,18 +532,21 @@ def make_datastore(
                 SCLogger.debug('Running astrometric calibration')
                 ds = p.astro_cal.run(ds)
                 ds.wcs.save()
-                if cache_dir is not None and cache_base_name is not None:
-                    output_path = ds.wcs.copy_to_cache(cache_dir)
+                if ( ( cache_dir is not None ) and ( cache_base_name is not None ) and
+                     ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) ):
+                    output_path = copy_to_cache(ds.wcs, cache_dir)
                     if output_path != cache_path:
                         warnings.warn(f'cache path {cache_path} does not match output path {output_path}')
 
             ########### photo_cal to create zero point ############
-            if cache_dir is not None and cache_base_name is not None:
+            if (   ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
+                   ( cache_dir is not None ) and ( cache_base_name is not None )
+                ):
                 cache_name = cache_base_name + '.zp.json'
                 cache_path = os.path.join(cache_dir, cache_name)
                 if os.path.isfile(cache_path):
                     SCLogger.debug('loading zero point from cache. ')
-                    ds.zp = ZeroPoint.copy_from_cache(cache_dir, cache_name)
+                    ds.zp = copy_from_cache(ZeroPoint, cache_dir, cache_name)
                     prov = Provenance(
                         code_version=code_version,
                         process='photo_cal',
@@ -552,8 +580,9 @@ def make_datastore(
             if ds.zp is None:  # make the zero point
                 SCLogger.debug('Running photometric calibration')
                 ds = p.photo_cal.run(ds)
-                if cache_dir is not None and cache_base_name is not None:
-                    output_path = ds.zp.copy_to_cache(cache_dir, cache_name)
+                if ( ( cache_dir is not None ) and ( cache_base_name is not None ) and
+                     ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) ):
+                    output_path = copy_to_cache(ds.zp, cache_dir, cache_name)
                     if output_path != cache_path:
                         warnings.warn(f'cache path {cache_path} does not match output path {output_path}')
 
@@ -590,7 +619,7 @@ def make_datastore(
                 cache_name = cache_sub_name + '.image.fits.json'
                 if os.path.isfile(os.path.join(cache_dir, cache_name)):
                     SCLogger.debug('loading subtraction image from cache. ')
-                    ds.sub_image = Image.copy_from_cache(cache_dir, cache_name)
+                    ds.sub_image = copy_from_cache(Image, cache_dir, cache_name)
 
                     ds.sub_image.provenance = prov
                     ds.sub_image.upstream_images.append(ref.image)
@@ -654,7 +683,7 @@ def make_datastore(
                             os.path.isfile(os.path.join(cache_dir, cache_name_new))
                     ):
                         SCLogger.debug('loading aligned reference image from cache. ')
-                        image_aligned_ref = Image.copy_from_cache(cache_dir, cache_name)
+                        image_aligned_ref = copy_from_cache(Image, cache_dir, cache_name)
                         image_aligned_ref.provenance = prov_aligned_ref
                         image_aligned_ref.info['original_image_id'] = ds.ref_image_id
                         image_aligned_ref.info['original_image_filepath'] = ds.ref_image.filepath
@@ -662,7 +691,7 @@ def make_datastore(
                         # TODO: should we also load the aligned image's sources, PSF, and ZP?
 
                         SCLogger.debug('loading aligned new image from cache. ')
-                        image_aligned_new = Image.copy_from_cache(cache_dir, cache_name)
+                        image_aligned_new = copy_from_cache(Image, cache_dir, cache_name)
                         image_aligned_new.provenance = prov_aligned_new
                         image_aligned_new.info['original_image_id'] = ds.image_id
                         image_aligned_new.info['original_image_filepath'] = ds.image.filepath
@@ -677,17 +706,19 @@ def make_datastore(
             if ds.sub_image is None:  # no hit in the cache
                 ds = p.subtractor.run(ds)
                 ds.sub_image.save(verify_md5=False)  # make sure it is also saved to archive
-                ds.sub_image.copy_to_cache(cache_dir)
+                if not os.getenv( "LIMIT_CACHE_USAGE" ):
+                    copy_to_cache(ds.sub_image, cache_dir)
 
             # make sure that the aligned images get into the cache, too
             if (
+                    ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
                     'cache_name_ref' in locals() and
                     os.path.isfile(os.path.join(cache_dir, cache_name_ref)) and
                     'cache_name_new' in locals() and
                     os.path.isfile(os.path.join(cache_dir, cache_name_new))
             ):
                 for im in ds.sub_image.aligned_images:
-                    im.copy_to_cache(cache_dir)
+                    copy_to_cache(im, cache_dir)
 
             ############ detecting to create a source list ############
             prov = Provenance(
@@ -698,9 +729,9 @@ def make_datastore(
                 is_testing=True,
             )
             cache_name = os.path.join(cache_dir, cache_sub_name + f'.sources_{prov.id[:6]}.npy.json')
-            if os.path.isfile(cache_name):
+            if ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and ( os.path.isfile(cache_name) ):
                 SCLogger.debug('loading detections from cache. ')
-                ds.detections = SourceList.copy_from_cache(cache_dir, cache_name)
+                ds.detections = copy_from_cache(SourceList, cache_dir, cache_name)
                 ds.detections.provenance = prov
                 ds.detections.image = ds.sub_image
                 ds.sub_image.sources = ds.detections
@@ -708,7 +739,8 @@ def make_datastore(
             else:  # cannot find detections on cache
                 ds = p.detector.run(ds)
                 ds.detections.save(verify_md5=False)
-                ds.detections.copy_to_cache(cache_dir, cache_name)
+                if not os.getenv( "LIMIT_CACHE_USAGE" ):
+                    copy_to_cache(ds.detections, cache_dir, cache_name)
 
             ############ cutting to create cutouts ############
             prov = Provenance(
@@ -719,9 +751,9 @@ def make_datastore(
                 is_testing=True,
             )
             cache_name = os.path.join(cache_dir, cache_sub_name + f'.cutouts_{prov.id[:6]}.h5')
-            if os.path.isfile(cache_name):
+            if ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and ( os.path.isfile(cache_name) ):
                 SCLogger.debug('loading cutouts from cache. ')
-                ds.cutouts = Cutouts.copy_list_from_cache(cache_dir, cache_name)
+                ds.cutouts = copy_list_from_cache(Cutouts, cache_dir, cache_name)
                 ds.cutouts = Cutouts.load_list(os.path.join(ds.cutouts[0].local_path, ds.cutouts[0].filepath))
                 [setattr(c, 'provenance', prov) for c in ds.cutouts]
                 [setattr(c, 'sources', ds.detections) for c in ds.cutouts]
@@ -729,7 +761,8 @@ def make_datastore(
             else:  # cannot find cutouts on cache
                 ds = p.cutter.run(ds)
                 Cutouts.save_list(ds.cutouts)
-                Cutouts.copy_list_to_cache(ds.cutouts, cache_dir)
+                if not os.getenv( "LIMIT_CACHE_USAGE" ):
+                    copy_list_to_cache(ds.cutouts, cache_dir)
 
             ############ measuring to create measurements ############
             prov = Provenance(
@@ -744,7 +777,7 @@ def make_datastore(
 
             if os.path.isfile(cache_name):  # note that the cache contains ALL the measurements, not only the good ones
                 SCLogger.debug('loading measurements from cache. ')
-                ds.all_measurements = Measurements.copy_list_from_cache(cache_dir, cache_name)
+                ds.all_measurements = copy_list_from_cache(Measurements, cache_dir, cache_name)
                 [setattr(m, 'provenance', prov) for m in ds.all_measurements]
                 [setattr(m, 'cutouts', c) for m, c in zip(ds.all_measurements, ds.cutouts)]
 
@@ -759,7 +792,7 @@ def make_datastore(
                 # no need to save list because Measurements is not a FileOnDiskMixin!
             else:  # cannot find measurements on cache
                 ds = p.measurer.run(ds)
-                Measurements.copy_list_to_cache(ds.all_measurements, cache_dir, cache_name)  # must provide filepath!
+                copy_list_to_cache(ds.all_measurements, cache_dir, cache_name)  # must provide filepath!
 
             ds.save_and_commit(session=session)
 
diff --git a/tests/fixtures/ptf.py b/tests/fixtures/ptf.py
index 0e883c2f..a08ded5f 100644
--- a/tests/fixtures/ptf.py
+++ b/tests/fixtures/ptf.py
@@ -27,6 +27,7 @@
 
 from util.retrydownload import retry_download
 from util.logger import SCLogger
+from util.cache import copy_to_cache, copy_list_to_cache, copy_from_cache, copy_list_from_cache
 
 
 @pytest.fixture(scope='session')
@@ -43,21 +44,28 @@ def ptf_bad_pixel_map(download_url, data_dir, ptf_cache_dir):
     filename = 'C11/masktot.fits'  # TODO: add more CCDs if needed
     # url = 'https://portal.nersc.gov/project/m2218/pipeline/test_images/2012021x/'
     url = os.path.join(download_url, 'PTF/10cwm/2012021x/')
-
-    # is this file already on the cache? if not, download it
-    cache_path = os.path.join(ptf_cache_dir, filename)
-    if not os.path.isfile(cache_path):
-        os.makedirs(os.path.dirname(cache_path), exist_ok=True)
-        retry_download(url + filename, cache_path)
-
-    if not os.path.isfile(cache_path):
-        raise FileNotFoundError(f"Can't read {cache_path}. It should have been downloaded!")
-
     data_dir = os.path.join(data_dir, 'PTF_calibrators')
     data_path = os.path.join(data_dir, filename)
-    if not os.path.isfile(data_path):
-        os.makedirs(os.path.dirname(data_path), exist_ok=True)
-        shutil.copy2(cache_path, data_path)
+
+    if os.getenv( "LIMIT_CACHE_USAGE" ):
+        if not os.path.isfile( data_path ):
+            os.makedirs( os.path.dirname( data_path ), exist_ok=True )
+            retry_download( url + filename, data_path )
+        if not os.path.isfile( data_path ):
+            raise FileNotFoundError( f"Can't read {data_path}.  It should have been downloaded!" )
+    else:
+        # is this file already on the cache? if not, download it
+        cache_path = os.path.join(ptf_cache_dir, filename)
+        if not os.path.isfile(cache_path):
+            os.makedirs(os.path.dirname(cache_path), exist_ok=True)
+            retry_download(url + filename, cache_path)
+
+        if not os.path.isfile(cache_path):
+            raise FileNotFoundError(f"Can't read {cache_path}. It should have been downloaded!")
+
+        if not os.path.isfile(data_path):
+            os.makedirs(os.path.dirname(data_path), exist_ok=True)
+            shutil.copy2(cache_path, data_path)
 
     with fits.open(data_path) as hdul:
         data = (hdul[0].data == 0).astype('uint16')  # invert the mask (good is False, bad is True)
@@ -81,29 +89,42 @@ def ptf_bad_pixel_map(download_url, data_dir, ptf_cache_dir):
 
 @pytest.fixture(scope='session')
 def ptf_downloader(provenance_preprocessing, download_url, data_dir, ptf_cache_dir):
+    """Downloads an image for ptf.
+
+    At the end, only count on the file being in data_dir.  It *might*
+    have also put the file in ptf_cache_dir, depending on an environment
+    variable setting; don't count on the file being in cache_dir outside
+    of this function.
+
+    """
 
     def download_ptf_function(filename='PTF201104291667_2_o_45737_11.w.fits'):
 
         os.makedirs(ptf_cache_dir, exist_ok=True)
         cachedpath = os.path.join(ptf_cache_dir, filename)
+        destination = os.path.join(data_dir, filename)
+        # url = f'https://portal.nersc.gov/project/m2218/pipeline/test_images/{filename}'
+        url = os.path.join(download_url, 'PTF/10cwm', filename)
 
-        # first make sure file exists in the cache
-        if os.path.isfile(cachedpath):
-            SCLogger.info(f"{cachedpath} exists, not redownloading.")
+        if os.getenv( "LIMIT_CACHE_USAGE" ):
+            retry_download( url, destination )
+            if not os.path.isfile( destination ):
+                raise FileNotFoundError( f"Can't read {destination}.  It should have been downloaded!" )
         else:
-            # url = f'https://portal.nersc.gov/project/m2218/pipeline/test_images/{filename}'
-            url = os.path.join(download_url, 'PTF/10cwm', filename)
-            retry_download(url, cachedpath)  # make the cached copy
+            # first make sure file exists in the cache
+            if os.path.isfile(cachedpath):
+                SCLogger.info(f"{cachedpath} exists, not redownloading.")
+            else:
+                retry_download(url, cachedpath)  # make the cached copy
 
-        if not os.path.isfile(cachedpath):
-            raise FileNotFoundError(f"Can't read {cachedpath}. It should have been downloaded!")
+            if not os.path.isfile(cachedpath):
+                raise FileNotFoundError(f"Can't read {cachedpath}. It should have been downloaded!")
 
-        # copy the PTF exposure from cache to local storage:
-        destination = os.path.join(data_dir, filename)
+            # copy the PTF exposure from cache to local storage:
 
-        if not os.path.isfile(destination):
-            os.makedirs(os.path.dirname(destination), exist_ok=True)
-            shutil.copy(cachedpath, destination)
+            if not os.path.isfile(destination):
+                os.makedirs(os.path.dirname(destination), exist_ok=True)
+                shutil.copy(cachedpath, destination)
 
         exposure = Exposure(filepath=filename)
 
@@ -186,7 +207,9 @@ def ptf_images_factory(ptf_urls, ptf_downloader, datastore_factory, ptf_cache_di
     def factory(start_date='2009-04-04', end_date='2013-03-03', max_images=None):
         # see if any of the cache names were saved to a manifest file
         cache_names = {}
-        if os.path.isfile(os.path.join(ptf_cache_dir, 'manifest.txt')):
+        if (   ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
+               ( os.path.isfile(os.path.join(ptf_cache_dir, 'manifest.txt')) )
+            ):
             with open(os.path.join(ptf_cache_dir, 'manifest.txt')) as f:
                 text = f.read().splitlines()
             for line in text:
@@ -222,7 +245,9 @@ def factory(start_date='2009-04-04', end_date='2013-03-03', max_images=None):
                     bad_pixel_map=ptf_bad_pixel_map,
                 )
 
-                if hasattr(ds, 'cache_base_name') and ds.cache_base_name is not None:
+                if (    ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
+                        ( hasattr(ds, 'cache_base_name') ) and ( ds.cache_base_name is not None )
+                    ):
                     cache_name = ds.cache_base_name
                     if cache_name.startswith(ptf_cache_dir):
                         cache_name = cache_name[len(ptf_cache_dir) + 1:]
@@ -293,15 +318,17 @@ def ptf_aligned_images(request, ptf_cache_dir, data_dir, code_version):
     cache_dir = os.path.join(ptf_cache_dir, 'aligned_images')
 
     # try to load from cache
-    if os.path.isfile(os.path.join(cache_dir, 'manifest.txt')):
+    if (    ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
+            ( os.path.isfile(os.path.join(cache_dir, 'manifest.txt')) )
+        ):
         with open(os.path.join(cache_dir, 'manifest.txt')) as f:
             filenames = f.read().splitlines()
         output_images = []
         for filename in filenames:
             imfile, psffile = filename.split()
-            output_images.append(Image.copy_from_cache(cache_dir, imfile + '.image.fits'))
-            output_images[-1].psf = PSF.copy_from_cache(cache_dir, psffile + '.fits')
-            output_images[-1].zp = ZeroPoint.copy_from_cache(cache_dir, imfile + '.zp')
+            output_images.append(copy_from_cache(Image, cache_dir, imfile + '.image.fits'))
+            output_images[-1].psf = copy_from_cache(PSF, cache_dir, psffile + '.fits')
+            output_images[-1].zp = copy_from_cache(ZeroPoint, cache_dir, imfile + '.zp')
     else:  # no cache available
         ptf_reference_images = request.getfixturevalue('ptf_reference_images')
 
@@ -322,18 +349,21 @@ def ptf_aligned_images(request, ptf_cache_dir, data_dir, code_version):
         psf_paths = []
         for image in coadd_image.aligned_images:
             image.save()
-            filepath = image.copy_to_cache(cache_dir)
+            filepath = copy_to_cache(image, cache_dir)
             if image.psf.filepath is None:  # save only PSF objects that haven't been saved yet
                 image.psf.save()
-            image.psf.copy_to_cache(cache_dir)
-            image.zp.copy_to_cache(cache_dir, filepath=filepath[:-len('.image.fits.json')]+'.zp.json')
+            if not os.getenv( "LIMIT_CACHE_USAGE" ):
+               copy_to_cache(image.psf, cache_dir)
+               copy_to_cache(image.zp, cache_dir, filepath=filepath[:-len('.image.fits.json')]+'.zp.json')
             filenames.append(image.filepath)
             psf_paths.append(image.psf.filepath)
 
-        os.makedirs(cache_dir, exist_ok=True)
-        with open(os.path.join(cache_dir, 'manifest.txt'), 'w') as f:
-            for filename, psf_path in zip(filenames, psf_paths):
-                f.write(f'{filename} {psf_path}\n')
+        if not os.getenv( "LIMIT_CACHE_USAGE" ):
+            os.makedirs(cache_dir, exist_ok=True)
+            with open(os.path.join(cache_dir, 'manifest.txt'), 'w') as f:
+                for filename, psf_path in zip(filenames, psf_paths):
+                    f.write(f'{filename} {psf_path}\n')
+
         output_images = coadd_image.aligned_images
 
     yield output_images
@@ -410,10 +440,15 @@ def ptf_ref(ptf_reference_images, ptf_aligned_images, coadder, ptf_cache_dir, da
         )
 
     extensions = ['image.fits', f'psf_{psf_prov.id[:6]}.fits', f'sources_{sources_prov.id[:6]}.fits', 'wcs', 'zp']
-    filenames = [os.path.join(ptf_cache_dir, cache_base_name) + f'.{ext}.json' for ext in extensions]
-    if all([os.path.isfile(filename) for filename in filenames]):  # can load from cache
+    if not os.getenv( "LIMIT_CACHE_USAGE" ):
+        filenames = [os.path.join(ptf_cache_dir, cache_base_name) + f'.{ext}.json' for ext in extensions]
+    else:
+        filenames = []
+    if (     ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
+             ( all([os.path.isfile(filename) for filename in filenames]) )
+        ):  # can load from cache
         # get the image:
-        coadd_image = Image.copy_from_cache(ptf_cache_dir, cache_base_name + '.image.fits')
+        coadd_image = copy_from_cache(Image, ptf_cache_dir, cache_base_name + '.image.fits')
         # we must load these images in order to save the reference image with upstreams
         coadd_image.upstream_images = ptf_reference_images
         coadd_image.provenance = im_prov
@@ -421,25 +456,25 @@ def ptf_ref(ptf_reference_images, ptf_aligned_images, coadder, ptf_cache_dir, da
         assert coadd_image.provenance_id == coadd_image.provenance.id
 
         # get the PSF:
-        coadd_image.psf = PSF.copy_from_cache(ptf_cache_dir, cache_base_name + f'.psf_{psf_prov.id[:6]}.fits')
+        coadd_image.psf = copy_from_cache(PSF, ptf_cache_dir, cache_base_name + f'.psf_{psf_prov.id[:6]}.fits')
         coadd_image.psf.provenance = psf_prov
         assert coadd_image.psf.provenance_id == coadd_image.psf.provenance.id
 
         # get the source list:
-        coadd_image.sources = SourceList.copy_from_cache(
-            ptf_cache_dir, cache_base_name + f'.sources_{sources_prov.id[:6]}.fits'
+        coadd_image.sources =copy_from_cache(
+            SourceList, ptf_cache_dir, cache_base_name + f'.sources_{sources_prov.id[:6]}.fits'
         )
         coadd_image.sources.provenance = sources_prov
         assert coadd_image.sources.provenance_id == coadd_image.sources.provenance.id
 
         # get the WCS:
-        coadd_image.wcs = WorldCoordinates.copy_from_cache(ptf_cache_dir, cache_base_name + '.wcs')
+        coadd_image.wcs = copy_from_cache(WorldCoordinates, ptf_cache_dir, cache_base_name + '.wcs')
         coadd_image.wcs.provenance = wcs_prov
         coadd_image.sources.wcs = coadd_image.wcs
         assert coadd_image.wcs.provenance_id == coadd_image.wcs.provenance.id
 
         # get the zero point:
-        coadd_image.zp = ZeroPoint.copy_from_cache(ptf_cache_dir, cache_base_name + '.zp')
+        coadd_image.zp = copy_from_cache(ZeroPoint, ptf_cache_dir, cache_base_name + '.zp')
         coadd_image.zp.provenance = zp_prov
         coadd_image.sources.zp = coadd_image.zp
         assert coadd_image.zp.provenance_id == coadd_image.zp.provenance.id
@@ -452,12 +487,13 @@ def ptf_ref(ptf_reference_images, ptf_aligned_images, coadder, ptf_cache_dir, da
         pipe.datastore.save_and_commit()
         coadd_image = pipe.datastore.image
 
-        # save all products into cache:
-        pipe.datastore.image.copy_to_cache(ptf_cache_dir)
-        pipe.datastore.sources.copy_to_cache(ptf_cache_dir)
-        pipe.datastore.psf.copy_to_cache(ptf_cache_dir)
-        pipe.datastore.wcs.copy_to_cache(ptf_cache_dir, cache_base_name + '.wcs.json')
-        pipe.datastore.zp.copy_to_cache(ptf_cache_dir, cache_base_name + '.zp.json')
+        if not os.getenv( "LIMIT_CACHE_USAGE" ):
+            # save all products into cache:
+            copy_to_cache(pipe.datastore.image, ptf_cache_dir)
+            copy_to_cache(pipe.datastore.sources, ptf_cache_dir)
+            copy_to_cache(pipe.datastore.psf, ptf_cache_dir)
+            copy_to_cache(pipe.datastore.wcs, ptf_cache_dir, cache_base_name + '.wcs.json')
+            copy_to_cache(pipe.datastore.zp, ptf_cache_dir, cache_base_name + '.zp.json')
 
     with SmartSession() as session:
         coadd_image = coadd_image.merge_all(session)
@@ -486,8 +522,8 @@ def ptf_subtraction1(ptf_ref, ptf_supernova_images, subtractor, ptf_cache_dir):
 
     cache_path = os.path.join(ptf_cache_dir, '187/PTF_20100216_075004_11_R_Diff_VXUBFA_u-7ogkop.image.fits.json')
 
-    if os.path.isfile(cache_path):  # try to load this from cache
-        im = Image.copy_from_cache(ptf_cache_dir, cache_path)
+    if ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and ( os.path.isfile(cache_path) ):  # try to load this from cache
+        im = copy_from_cache(Image, ptf_cache_dir, cache_path)
         im.upstream_images = [ptf_ref.image, ptf_supernova_images[0]]
         im.ref_image_id = ptf_ref.image.id
         prov = Provenance(
@@ -503,7 +539,8 @@ def ptf_subtraction1(ptf_ref, ptf_supernova_images, subtractor, ptf_cache_dir):
         ds = subtractor.run(ptf_supernova_images[0])
         ds.sub_image.save()
 
-        ds.sub_image.copy_to_cache(ptf_cache_dir)
+        if not os.getenv( "LIMIT_CACHE_USAGE" ) :
+            copy_to_cache(ds.sub_image, ptf_cache_dir)
         im = ds.sub_image
 
     # save the subtraction image to DB and the upstreams (if they are not already there)
@@ -514,4 +551,3 @@ def ptf_subtraction1(ptf_ref, ptf_supernova_images, subtractor, ptf_cache_dir):
     yield im
 
     im.delete_from_disk_and_database(remove_downstreams=True)
-
diff --git a/tests/fixtures/ztf.py b/tests/fixtures/ztf.py
index 7f171659..8f3736e3 100644
--- a/tests/fixtures/ztf.py
+++ b/tests/fixtures/ztf.py
@@ -40,12 +40,15 @@ def ztf_filepaths_image_sources_psf(data_dir, ztf_cache_dir, download_url):
     output = image, weight, flags, sources, psf, psfxml
 
     for filepath in output:
-        if not os.path.isfile(os.path.join(ztf_cache_dir, filepath)):
-            retry_download(download_url + filepath, os.path.join(ztf_cache_dir, filepath))
-        if not os.path.isfile(os.path.join(ztf_cache_dir, filepath)):
-            raise FileNotFoundError(f"Can't read {filepath}. It should have been downloaded! ")
-        if not os.path.isfile(os.path.join(data_dir, filepath)):
-            shutil.copy2(os.path.join(ztf_cache_dir, filepath), os.path.join(data_dir, filepath))
+        if os.getenv( "LIMIT_CACHE_USAGE" ):
+            retry_download( download_url + filepath, os.path.join( data_dir, filepath ) )
+        else:
+            if not os.path.isfile(os.path.join(ztf_cache_dir, filepath)):
+                retry_download(download_url + filepath, os.path.join(ztf_cache_dir, filepath))
+            if not os.path.isfile(os.path.join(ztf_cache_dir, filepath)):
+                raise FileNotFoundError(f"Can't read {filepath}. It should have been downloaded! ")
+            if not os.path.isfile(os.path.join(data_dir, filepath)):
+                shutil.copy2(os.path.join(ztf_cache_dir, filepath), os.path.join(data_dir, filepath))
 
     output = tuple(
         pathlib.Path(os.path.join(data_dir, filepath)) for filepath in output
diff --git a/tests/models/test_decam.py b/tests/models/test_decam.py
index e35ab6b2..890fac29 100644
--- a/tests/models/test_decam.py
+++ b/tests/models/test_decam.py
@@ -120,6 +120,7 @@ def test_image_from_decam_exposure(decam_filename, provenance_base, data_dir):
 # guidance for how to do things, do *not* write code that mucks about
 # with the _frame member of one of those objects; that's internal state
 # not intended for external consumption.
+@pytest.mark.skipif( os.getenv('SKIP_NOIRLAB_DOWNLOADS'), reason="SKIP_NOIRLAB_DOWNLOADS is set" )
 def test_decam_search_noirlab( decam_reduced_origin_exposures ):
     origloglevel = SCLogger.get().getEffectiveLevel()
     try:
@@ -162,6 +163,7 @@ def test_decam_search_noirlab( decam_reduced_origin_exposures ):
         SCLogger.setLevel( origloglevel )
 
 
+@pytest.mark.skipif( os.getenv('SKIP_NOIRLAB_DOWNLOADS'), reason="SKIP_NOIRLAB_DOWNLOADS is set" )
 def test_decam_download_origin_exposure( decam_reduced_origin_exposures, cache_dir ):
     assert all( [ row.proc_type == 'instcal' for i, row in decam_reduced_origin_exposures._frame.iterrows() ] )
     try:
@@ -206,6 +208,7 @@ def test_decam_download_origin_exposure( decam_reduced_origin_exposures, cache_d
         pass
 
 
+@pytest.mark.skipif( os.getenv('SKIP_NOIRLAB_DOWNLOADS'), reason="SKIP_NOIRLAB_DOWNLOADS is set" )
 def test_decam_download_and_commit_exposure(
         code_version, decam_raw_origin_exposures, cache_dir, data_dir, test_config, archive
 ):
@@ -234,12 +237,6 @@ def test_decam_download_and_commit_exposure(
                 eids.append( exposure.id )
                 fname = pathlib.Path( decam_raw_origin_exposures._frame.iloc[i].archive_filename ).name
 
-                # cache the files
-                os.makedirs( os.path.join( cache_dir, 'DECam' ), exist_ok=True )
-                if not os.path.isfile( os.path.join( cache_dir, 'DECam', fname ) ):
-                    shutil.copy2( os.path.join( data_dir, fname ),
-                                os.path.join( cache_dir, 'DECam', fname ) )
-
                 match = re.search( r'^c4d_(?P<yymmdd>\d{6})_(?P<hhmmss>\d{6})_ori.fits', fname )
                 assert match is not None
                 # Todo : add the subdirectory to dbfname once that is implemented
@@ -271,14 +268,12 @@ def test_decam_download_and_commit_exposure(
             for exposure in exposures:
                 exposure.delete_from_disk_and_database( session=session, commit=False )
             session.commit()
-            # remove downloaded files from data_dir (a cached version should remain)
             if 'downloaded' in locals():
                 for d in downloaded:
                     path = os.path.join(data_dir, d['exposure'].name)
                     if os.path.isfile(path):
                         os.unlink(path)
 
-            # remove downloaded files from data_dir (a cached version should remain)
             if 'downloaded' in locals():
                 for d in downloaded:
                     path = os.path.join(data_dir, d['exposure'].name)
diff --git a/tests/pipeline/test_astro_cal.py b/tests/pipeline/test_astro_cal.py
index 37bad569..7d527791 100644
--- a/tests/pipeline/test_astro_cal.py
+++ b/tests/pipeline/test_astro_cal.py
@@ -82,15 +82,16 @@ def test_solve_wcs_scamp( ztf_gaia_dr3_excerpt, ztf_datastore_uncommitted, astro
 
 def test_run_scamp( decam_datastore, astrometor ):
     ds = decam_datastore
-    original_filename = ds.cache_base_name + '.image.fits.original'
-    with open(original_filename, "rb") as ifp:
+
+    # Get the md5sum and WCS from the image before we do things to it
+    with open(ds.path_to_original_image, "rb") as ifp:
         md5 = hashlib.md5()
         md5.update(ifp.read())
         origmd5 = uuid.UUID(md5.hexdigest())
 
     xvals = [0, 0, 2047, 2047]
     yvals = [0, 4095, 0, 4095]
-    with fits.open(original_filename) as hdu:
+    with fits.open(ds.path_to_original_image) as hdu:
         origwcs = WCS(hdu[ds.section_id].header)
 
     astrometor.pars.cross_match_catalog = 'gaia_dr3'
diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py
index 091df2d6..0dd25b9b 100644
--- a/tests/pipeline/test_pipeline.py
+++ b/tests/pipeline/test_pipeline.py
@@ -302,7 +302,7 @@ def test_bitflag_propagation(decam_exposure, decam_reference, decam_default_cali
         desired_bitflag = 2**1 + 2**17 # bitflag for 'banding' and 'many sources'
         ds = p.run(ds)
 
-        assert ds.sources.bitflag == desired_bitflag 
+        assert ds.sources.bitflag == desired_bitflag
         assert ds.wcs._upstream_bitflag == desired_bitflag
         assert ds.zp._upstream_bitflag == desired_bitflag
         assert ds.sub_image._upstream_bitflag == desired_bitflag
@@ -320,7 +320,7 @@ def test_bitflag_propagation(decam_exposure, decam_reference, decam_default_cali
             ds.image = session.merge(ds.image)
 
             # add a bitflag and check that it appears in downstreams
-            ds.image._bitflag = 16  # 16=2**4 is the bitflag for 'bad subtraction'  
+            ds.image._bitflag = 16  # 16=2**4 is the bitflag for 'bad subtraction'
             session.add(ds.image)
             session.commit()
             ds.image.exposure.update_downstream_badness(session)
@@ -339,7 +339,7 @@ def test_bitflag_propagation(decam_exposure, decam_reference, decam_default_cali
                 assert cutout.bitflag == desired_bitflag
 
             # remove the bitflag and check that it disappears in downstreams
-            ds.image._bitflag = 0  # remove 'bad subtraction'  
+            ds.image._bitflag = 0  # remove 'bad subtraction'
             session.commit()
             ds.image.exposure.update_downstream_badness(session)
             session.commit()
@@ -409,7 +409,7 @@ def test_get_upstreams_and_downstreams(decam_exposure, decam_reference, decam_de
             cutout_ids = np.unique([cutout.id for cutout in ds.cutouts])
             for measurement in ds.measurements:
                 m_upstream_ids =  np.array([upstream.id for upstream in measurement.get_upstreams(session)])
-                assert np.all(np.isin(m_upstream_ids, cutout_ids)) 
+                assert np.all(np.isin(m_upstream_ids, cutout_ids))
 
             # test get_downstreams
             assert [downstream.id for downstream in ds.exposure.get_downstreams(session)] == [ds.image.id]
@@ -431,7 +431,7 @@ def test_get_upstreams_and_downstreams(decam_exposure, decam_reference, decam_de
                 assert np.all(np.isin(c_downstream_ids, measurement_ids))
             for measurement in ds.measurements:
                 assert [downstream.id for downstream in measurement.get_downstreams(session)] == []
-            
+
 
     finally:
         if 'ds' in locals():
diff --git a/util/cache.py b/util/cache.py
new file mode 100644
index 00000000..4f4611f4
--- /dev/null
+++ b/util/cache.py
@@ -0,0 +1,267 @@
+import os
+import shutil
+import json
+
+from models.base import FileOnDiskMixin
+from util.logger import SCLogger
+
+# ======================================================================
+# Functions for copying FileOnDisk objects to/from cache
+
+def copy_to_cache(FoD, cache_dir, filepath=None):
+    """Save a copy of the object (and, potentially, associated files) into a cache directory.
+
+    If the object is a FileOnDiskMixin, then the file(s) pointed by get_fullpath()
+    will be copied to the cache directory with their original names,
+    unless filepath is specified, in which case the cached files will
+    have a different name than the files in the data folder (and the database filepath).
+    The filepath (with optional first extension) will be used to create a JSON file
+    which holds the object's column attributes (i.e., only those that are
+    database persistent).
+
+    If caching a non-FileOnDiskMixin object, the filepath argument must
+    be given, because it is used to name the JSON file.  The object must
+    implement a method to_json(path) that serializes itself to the file
+    specified by path.
+
+    Parameters
+    ----------
+    FoD: FileOnDiskMixin or another object that implements to_json()
+        The object to cache.
+    cache_dir: str or path
+        The path to the cache directory.
+    filepath: str or path (optional)
+        Must be given if the FoD is None.
+        If it is a FileOnDiskMixin, it will be used to name
+        the data files and the JSON file in the cache folder.
+
+    Returns
+    -------
+    str
+        The full path to the output json file.
+
+    """
+    if filepath is not None and filepath.endswith('.json'):  # remove .json if it exists
+        filepath = filepath[:-5]
+
+    json_filepath = filepath
+    if not isinstance(FoD, FileOnDiskMixin):
+        if filepath is None:
+            raise ValueError("filepath must be given when caching a non FileOnDiskMixin object")
+
+    else:  # it is a FileOnDiskMixin
+        if filepath is None:  # use the FileOnDiskMixin filepath as default
+            filepath = FoD.filepath  # use this filepath for the data files
+            json_filepath = FoD.filepath  # use the same filepath for the json file too
+        if (
+                FoD.filepath_extensions is not None and
+                len(FoD.filepath_extensions) > 0 and
+                not json_filepath.endswith(FoD.filepath_extensions[0])
+        ):
+                json_filepath += FoD.filepath_extensions[0]  # only append this extension to the json filename
+
+        for i, source_f in enumerate(FoD.get_fullpath(as_list=True)):
+            if source_f is None:
+                continue
+            target_f = os.path.join(cache_dir, filepath)
+            if FoD.filepath_extensions is not None and i < len(FoD.filepath_extensions):
+                target_f += FoD.filepath_extensions[i]
+            SCLogger.debug(f"Copying {source_f} to {target_f}")
+            os.makedirs(os.path.dirname(target_f), exist_ok=True)
+            shutil.copy2(source_f, target_f)
+
+    # attach the cache_dir and the .json extension if needed
+    json_filepath = os.path.join(cache_dir, json_filepath)
+    os.makedirs( os.path.dirname( json_filepath ), exist_ok=True )
+    if not json_filepath.endswith('.json'):
+        json_filepath += '.json'
+    FoD.to_json(json_filepath)
+
+    return json_filepath
+
+def copy_list_to_cache(obj_list, cache_dir, filepath=None):
+    """Copy a correlated list of objects to the cache directory.
+
+    All objects must be of the same type.  If they are of type
+    FileOnDiskMixin, the files associated with the *first* object in the
+    list (only) will be copied to the cache.  (Use case: something like
+    Cutouts where a whole bunch of objects all have the same file.)
+
+    The objects implement the to_dict() method that serializes
+    themselves to a dictionary (as FileOnDiskMixin does).  All of the
+    objects' dictionary data will be written to filepath as a JSON list
+    of dictionaries.
+
+    In either case, if the objects have a "filepath" attribute, the
+    value of that attribute must be the same for every object in the
+    list.
+
+    Parameters
+    ----------
+    obj_list: list
+        A list of objects to save to the cache directory.
+    cache_dir: str or path
+        The path to the cache directory.
+    filepath: str or path (optional)
+        Must be given if the objects are not FileOnDiskMixin.
+        If it is a FileOnDiskMixin, it will be used to name
+        the data files and the JSON file in the cache folder.
+
+    Returns
+    -------
+    str
+        The full path to the output JSON file.
+
+    """
+    if len(obj_list) == 0:
+        if filepath is None:
+            return  # can't do anything without a filepath
+        json_filepath = os.path.join(cache_dir, filepath)
+        if not json_filepath.endswith('.json'):
+            json_filepath += '.json'
+    else:
+        types = set([type(obj) for obj in obj_list])
+        if len(types) != 1:
+            raise ValueError("All objects must be of the same type!")
+
+        filepaths = set([getattr(obj, 'filepath', None) for obj in obj_list])
+        if len(filepaths) != 1:
+            raise ValueError("All objects must have the same filepath!")
+
+        # save the JSON file and copy associated files
+        json_filepath = copy_to_cache(obj_list[0], cache_dir, filepath=filepath)
+
+    # overwrite the JSON file with the list of dictionaries
+    with open(json_filepath, 'w') as fp:
+        json.dump([obj.to_dict() for obj in obj_list], fp, indent=2)
+
+    return json_filepath
+
+def copy_from_cache(cls, cache_dir, filepath):
+    """Copy and reconstruct an object from the cache directory.
+
+    Will need the JSON file that contains all the column attributes of the file.
+    Once those are successfully loaded, and if the object is a FileOnDiskMixin,
+    it will be able to figure out where all the associated files are saved
+    based on the filepath and extensions in the JSON file.
+    Those files will be copied into the current data directory
+    (i.e., that pointed to by FileOnDiskMixin.local_path).
+    The reconstructed object should be correctly associated
+    with its files but will not necessarily have the correct
+    relationships to other objects.
+
+    Parameters
+    ----------
+    cls : Class that derives from FileOnDiskMixin, or that implements from_dict(dict)
+        The class of the object that's being copied
+    cache_dir: str or path
+        The path to the cache directory.
+    filepath: str or path
+        The name of the JSON file that holds the column attributes.
+
+    Returns
+    -------
+    output: SeeChangeBase
+        The reconstructed object, of the same type as the class.
+    """
+    # allow user to give an absolute path, so long as it is in the cache dir
+    if filepath.startswith(cache_dir):
+        filepath = filepath[len(cache_dir) + 1:]
+
+    # allow the user to give the filepath with or without the .json extension
+    if filepath.endswith('.json'):
+        filepath = filepath[:-5]
+
+    full_path = os.path.join(cache_dir, filepath)
+    with open(full_path + '.json', 'r') as fp:
+        json_dict = json.load(fp)
+
+    output = cls.from_dict(json_dict)
+
+    # copy any associated files
+    if isinstance(output, FileOnDiskMixin):
+        # if fullpath ends in filepath_extensions[0]
+        if (
+                output.filepath_extensions is not None and
+                output.filepath_extensions[0] is not None and
+                full_path.endswith(output.filepath_extensions[0])
+        ):
+            full_path = full_path[:-len(output.filepath_extensions[0])]
+
+        for i, target_f in enumerate(output.get_fullpath(as_list=True)):
+            if target_f is None:
+                continue
+            source_f = os.path.join(cache_dir, full_path)
+            if output.filepath_extensions is not None and i < len(output.filepath_extensions):
+                source_f += output.filepath_extensions[i]
+            SCLogger.debug(f"Copying {source_f} to {target_f}")
+            os.makedirs(os.path.dirname(target_f), exist_ok=True)
+            shutil.copyfile(source_f, target_f)
+
+    return output
+
+def copy_list_from_cache(cls, cache_dir, filepath):
+    """Copy and reconstruct a list of objects from the cache directory.
+
+    Will need the JSON file that contains all the column attributes of the file.
+    Once those are successfully loaded, and if the object is a FileOnDiskMixin,
+    it will be able to figure out where all the associated files are saved
+    based on the filepath and extensions in the JSON file.
+
+    Parameters
+    ----------
+    cls: Class that derives from FileOnDiskMixin, or that implements from_dict(dict)
+        The class of the objects that are being copied
+    cache_dir: str or path
+        The path to the cache directory.
+    filepath: str or path
+        The name of the JSON file that holds the column attributes.
+
+    Returns
+    -------
+    output: list
+        The list of reconstructed objects, of the same type as the class.
+    """
+    # allow user to give an absolute path, so long as it is in the cache dir
+    if filepath.startswith(cache_dir):
+        filepath = filepath[len(cache_dir) + 1:]
+
+    # allow the user to give the filepath with or without the .json extension
+    if filepath.endswith('.json'):
+        filepath = filepath[:-5]
+
+    full_path = os.path.join(cache_dir, filepath)
+    with open(full_path + '.json', 'r') as fp:
+        json_list = json.load(fp)
+
+    output = []
+    for obj_dict in json_list:
+        output.append(cls.from_dict(obj_dict))
+
+    if len(output) == 0:
+        return []
+
+    if isinstance(output[0], FileOnDiskMixin):
+        # if fullpath ends in filepath_extensions[0]
+        if (
+                output[0].filepath_extensions is not None and
+                output[0].filepath_extensions[0] is not None and
+                full_path.endswith(output[0].filepath_extensions[0])
+        ):
+            full_path = full_path[:-len(output[0].filepath_extensions[0])]
+
+        for i, target_f in enumerate(output[0].get_fullpath(as_list=True)):
+            if target_f is None:
+                continue
+            source_f = os.path.join(cache_dir, full_path)
+            if output[0].filepath_extensions is not None and i < len(output[0].filepath_extensions):
+                source_f += output[0].filepath_extensions[i]
+            SCLogger.debug(f"Copying {source_f} to {target_f}")
+            os.makedirs(os.path.dirname(target_f), exist_ok=True)
+            shutil.copyfile(source_f, target_f)
+
+    return output
+
+
+# ======================================================================
+
diff --git a/webap/Dockerfile b/webap/Dockerfile
new file mode 100644
index 00000000..e9a4564f
--- /dev/null
+++ b/webap/Dockerfile
@@ -0,0 +1,82 @@
+# To build:
+#
+# Look at the file seechange_webap_config.py ; make a copy of this
+#   to a secrets location, and put in the appropriate database
+#   information.  This secrets location must then be bind-mounted
+#   to where the webap runs.
+#
+# docker build -t seechange-webap .
+#
+# ...you might want to use a name other than "seechange-webap", to
+# reflect the specifics of the database you built for.  If you
+# omit the variables, it will build with the database suitable
+# for the tests environment (in ../tests/docker_compose.yaml).
+#
+# Run a container built from this image with:
+#
+#   docker run -d \
+#     --mount type=bind,source=<secretsdir>,target=/secrets \
+#     --name seechange-webap -p 8080:8080 seechange-webap
+#
+# ...putting whatever you want after "--name", replacing the
+# "seechange-webap" at the end of the line with what you gave the -t
+# argument of docker build, and replacing <secretsdir> with the place
+# wher eyou wrote the seechange_webap_config.py file.
+#
+# Thereafer, you can find the webap at localhost:8080
+#
+# To bind-mount the default install directory for testing purposes, after -d:
+#    --mount type=bind,source=$PWD/install,target=/code
+#
+# You can of course bind mount different directories to which you
+# installed the code, if you know what you're doing.
+#
+# This dockerfile is also used in tests, with appropriate options for the test database and
+# opening up port 8080 on the machine you're running the tests from.
+
+FROM rknop/devuan-daedalus-rknop
+
+MAINTAINER Rob Knop <raknop@lbl.gov>
+
+SHELL [ "/bin/bash", "-c" ]
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND="noninteractive" apt-get -y upgrade \
+    && DEBIAN_FRONTEND="noninteractive" TZ="US/Pacific" apt-get -y install -y \
+         python3 \
+         python3-pip \
+         postgresql-client \
+         python3-psycopg2 \
+         pkg-config \
+         libhdf5-dev \
+         build-essential \
+         netcat-openbsd \
+    && apt-get -y autoremove \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN rm /usr/lib/python3.11/EXTERNALLY-MANAGED
+
+RUN pip install gunicorn flask pyyaml numpy pandas matplotlib astropy h5py pillow \
+    && rm -rf /.cache/pip
+
+# Todo : multi-stage docker file so that the stuff in /build
+#  isn't in any of the saved image layers
+
+# I'm using knowledge of what's in the Makefile
+# to construct this...
+
+RUN mkdir /code
+RUN mkdir -p /build/seechange_webap/rkwebutil
+COPY Makefile seechange_webap.py /build/seechange_webap/
+COPY static/ /build/seechange_webap/static/
+COPY templates/ /build/seechange_webap/templates/
+COPY rkwebutil/rkwebutil.js /build/seechange_webap/rkwebutil/
+WORKDIR /build/seechange_webap
+RUN make INSTALLDIR=/code install
+RUN rm -rf /build/seechange_webap
+
+WORKDIR /code
+
+# TODO : think about -w 4 below
+CMD [ "gunicorn", "-w", "4", "-b", "0.0.0.0:8080", "--timeout", "0", "seechange_webap:app" ]
diff --git a/webap/Makefile b/webap/Makefile
new file mode 100644
index 00000000..d0aaf8b4
--- /dev/null
+++ b/webap/Makefile
@@ -0,0 +1,32 @@
+INSTALLDIR = install
+
+#PG_HOST = placeholder
+#PG_PORT = placeholder
+#PG_USER = placeholder
+#PG_PASS = placeholder
+#PG_NAME = placeholder
+#ARCHIVE_DIR = placeholder
+
+toinstall = seechange_webap.py \
+	static/seechange.css static/seechange.js static/seechange_start.js static/rkwebutil.js static/favicon.ico \
+	templates/base.html templates/seechange_webap.html
+#	seechange_webap_config.py
+
+.PHONY: install # seechange_webap_config.py
+
+install: $(patsubst %, $(INSTALLDIR)/%, $(toinstall))
+
+$(INSTALLDIR)/%: %
+	install -Dp $< $@
+
+#seechange_webap_config.py:
+#	echo import pathlib > seechange_webap_config.py
+#	echo PG_HOST = \"$(PG_HOST)\" >> seechange_webap_config.py
+#	echo PG_PORT = $(PG_PORT) >> seechange_webap_config.py
+#	echo PG_USER = \"$(PG_USER)\" >> seechange_webap_config.py
+#	echo PG_PASS = \"$(PG_PASS)\" >> seechange_webap_config.py
+#	echo PG_NAME = \"$(PG_NAME)\" >> seechange_webap_config.py
+#	echo ARCHIVE_DIR = pathlib.Path\( \"$(ARCHIVE_DIR)\" \) >> seechange_webap_config.py
+
+static/rkwebutil.js: rkwebutil/rkwebutil.js
+	ln -s ../rkwebutil/rkwebutil.js static/rkwebutil.js
diff --git a/webap/rkwebutil b/webap/rkwebutil
new file mode 160000
index 00000000..7fc131b0
--- /dev/null
+++ b/webap/rkwebutil
@@ -0,0 +1 @@
+Subproject commit 7fc131b06113218094a974aa5ac1af8c7eb8f60b
diff --git a/webap/seechange_webap.py b/webap/seechange_webap.py
new file mode 100644
index 00000000..0d10e4b8
--- /dev/null
+++ b/webap/seechange_webap.py
@@ -0,0 +1,478 @@
+# MASSIVE TODO : this whole webap doesn't handle provenances at all
+# We need a way to choose provenances.  I sugest some sort of tag
+# table that allows us to associate tags with provenances so that
+# we can choose a set of provenances based a simple tag name.
+
+# Put this first so we can be sure that there are no calls that subvert
+#  this in other includes.
+import matplotlib
+matplotlib.use( "Agg" )
+# matplotlib.rc('font', **{'family': 'serif', 'serif': ['Computer Modern']})
+# matplotlib.rc('text', usetex=True)  #  Need LaTeX in Dockerfile
+from matplotlib import pyplot
+
+# TODO : COUNT(DISTINCT()) can be slow, deal with this if necessary
+#   I'm hoping that since they all show up inside a group and the
+#   total number of things I expect to have to distinct on within each group is
+#   not likely to more than ~10^2, it won't matter.
+
+import sys
+import traceback
+import math
+import io
+import re
+import json
+import pathlib
+import logging
+import base64
+
+import psycopg2
+import psycopg2.extras
+import numpy
+import h5py
+import PIL
+import astropy.time
+import astropy.visualization
+
+import flask
+
+# Read the database config
+
+sys.path.append( '/secrets' )
+from seechange_webap_config import PG_HOST, PG_PORT, PG_USER, PG_PASS, PG_NAME, ARCHIVE_DIR
+
+# Figure out where we are
+
+workdir = pathlib.Path(__name__).resolve().parent
+
+# Create the flask app, which is what gunicorn is going to look for
+
+app = flask.Flask( __name__, instance_relative_config=True )
+
+# app.logger.setLevel( logging.INFO )
+app.logger.setLevel( logging.DEBUG )
+
+# **********************************************************************
+
+def dbconn():
+    conn = psycopg2.connect( host=PG_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_NAME )
+    yield conn
+    conn.rollback()
+    conn.close()
+
+# **********************************************************************
+
+@app.route( "/", strict_slashes=False )
+def mainpage():
+    return flask.render_template( "seechange_webap.html" )
+
+# **********************************************************************
+
+@app.route( "/exposures", methods=['POST'], strict_slashes=False )
+def exposures():
+    try:
+        data = { 'startdate': None,
+                 'enddate': None
+                }
+        if flask.request.is_json:
+            data.update( flask.request.json )
+
+        app.logger.debug( f"After parsing, data = {data}" )
+        t0 = None if data['startdate'] is None else astropy.time.Time( data['startdate'], format='isot' ).mjd
+        t1 = None if data['enddate'] is None else astropy.time.Time( data['enddate'], format='isot' ).mjd
+        app.logger.debug( f"t0 = {t0}, t1 = {t1}" )
+
+        conn = next( dbconn() )
+        cursor = conn.cursor()
+        # TODO : deal with provenance!
+        # (We need some kind of provenance tagging table, so that the user can specify
+        # a user-readable name (e.g. "default", "latest", "dr1", whatever) that specifies
+        # a set of provenances to search.  One of these names must be all the provenances
+        # we're using "right now" in the active pipeline; that will be the one that
+        # (by default) the webap uses.
+        q = ( 'SELECT m.id, m.filepath, m.mjd, m.target, m.filter, m.filter_array, m.exp_time, '
+              '       m.n_images, m.n_cutouts, m.n_measurements, '
+              '       SUM( CASE WHEN r.success THEN 1 ELSE 0 END ) AS n_successim, '
+              '       SUM( CASE WHEN r.error_message IS NOT NULL THEN 1 ELSE 0 END ) as n_errors '
+              'FROM ( '
+              '  SELECT e.id, e.filepath, e.mjd, e.target, e.filter, e.filter_array, e.exp_time, '
+              '         COUNT(DISTINCT(i.id)) AS n_images, COUNT(c.id) AS n_cutouts, COUNT(m.id) AS n_measurements '
+              '  FROM exposures e '
+              '  LEFT JOIN images i ON i.exposure_id=e.id '
+              '  LEFT JOIN image_upstreams_association ias ON ias.upstream_id=i.id '
+              '  LEFT JOIN images s ON s.id = ias.downstream_id AND s.is_sub '
+              '  LEFT JOIN source_lists sl ON sl.image_id=s.id '
+              '  LEFT JOIN cutouts c ON c.sources_id=sl.id '
+              '  LEFT JOIN measurements m ON m.cutouts_id=c.id '
+              '  LEFT JOIN reports r ON r.exposure_id=e.id ' )
+        subdict = {}
+        if ( t0 is not None ) or ( t1 is not None ):
+            q += "  WHERE "
+            if t0 is not None:
+                q += 'e.mjd >= %(t0)s'
+                subdict['t0'] = t0
+            if t1 is not None:
+                if t0 is not None: q += ' AND '
+                q += 'e.mjd <= %(t1)s'
+                subdict['t1'] = t1
+        q += ( '   GROUP BY e.id ' # ,e.filepath,e.mjd,e.target,e.filter,e.filter_array,e.exp_time '
+               '   ORDER BY e.mjd, e.filter, e.filter_array ' )
+
+        q += ( ') m '
+               'LEFT JOIN reports r ON m.id=r.exposure_id '
+               'GROUP BY m.id, m.filepath, m.mjd, m.target, m.filter, m.filter_array, m.exp_time, '
+               '         m.n_images, m.n_cutouts, m.n_measurements ' )
+
+        cursor.execute( q, subdict )
+        columns = { cursor.description[i][0]: i for i in range(len(cursor.description)) }
+
+        ids = []
+        name = []
+        mjd = []
+        target = []
+        filtername = []
+        exp_time = []
+        n_images = []
+        n_cutouts = []
+        n_sources = []
+        n_successim = []
+        n_errors = []
+
+        slashre = re.compile( '^.*/([^/]+)$' )
+        for row in cursor.fetchall():
+            ids.append( row[columns['id']] )
+            match = slashre.search( row[columns['filepath']] )
+            if match is None:
+                name.append( row[columns['filepath']] )
+            else:
+                name.append( match.group(1) )
+            mjd.append( row[columns['mjd']] )
+            target.append( row[columns['target']] )
+            app.logger.debug( f"filter={row[columns['filter']]} type {row[columns['filter']]}; "
+                              f"filter_array={row[columns['filter_array']]} type {row[columns['filter_array']]}" )
+            filtername.append( row[columns['filter']] )
+            exp_time.append( row[columns['exp_time']] )
+            n_images.append( row[columns['n_images']] )
+            n_cutouts.append( row[columns['n_cutouts']] )
+            n_sources.append( row[columns['n_measurements']] )
+            n_successim.append( row[columns['n_successim']] )
+            n_errors.append( row[columns['n_errors']] )
+
+        return { 'status': 'ok',
+                 'startdate': t0,
+                 'enddate': t1,
+                 'exposures': {
+                     'id': ids,
+                     'name': name,
+                     'mjd': mjd,
+                     'target': target,
+                     'filter': filtername,
+                     'exp_time': exp_time,
+                     'n_images': n_images,
+                     'n_cutouts': n_cutouts,
+                     'n_sources': n_sources,
+                     'n_successim': n_successim,
+                     'n_errors': n_errors,
+                 }
+                }
+    except Exception as ex:
+        # sio = io.StringIO()
+        # traceback.print_exc( file=sio )
+        # app.logger.debug( sio.getvalue() )
+        app.logger.exception( ex )
+        return { 'status': 'error',
+                 'error': f'Exception: {ex}'
+                }
+
+# **********************************************************************
+
+@app.route( "/exposure_images/<expid>", methods=['GET', 'POST'], strict_slashes=False )
+def exposure_images( expid ):
+    try:
+        conn = next( dbconn() )
+        cursor = conn.cursor()
+        # TODO : deal with provenance!
+        q = ( 'SELECT i.id, i.filepath, i.ra, i.dec, i.gallat, i.section_id, i.fwhm_estimate, '
+              '       i.zero_point_estimate, i.lim_mag_estimate, i.bkg_mean_estimate, i.bkg_rms_estimate, '
+              '       s.id AS subid, COUNT(c.id) AS numcutouts, COUNT(m.id) AS nummeasurements, '
+              '       r.error_step, r.error_type, r.error_message, r.warnings, '
+              '       r.process_memory, r.process_runtime, r.progress_steps_bitflag, r.products_exist_bitflag '
+              'FROM images i '
+              'LEFT JOIN image_upstreams_association ias ON ias.upstream_id=i.id '
+              'LEFT JOIN images s ON s.id = ias.downstream_id AND s.is_sub '
+              'LEFT JOIN source_lists sl ON sl.image_id=s.id '
+              'LEFT JOIN cutouts c ON  c.sources_id=sl.id '
+              'LEFT JOIN measurements m ON c.id=m.cutouts_id '
+              'LEFT JOIN reports r ON r.exposure_id=i.exposure_id AND r.section_id=i.section_id '
+              'WHERE i.is_sub=false AND i.exposure_id=%(expid)s '
+              'GROUP BY i.id,s.id,r.id '
+              'ORDER BY i.section_id,s.id ' )
+        app.logger.debug( f"Getting images for exposure {expid}; query = {cursor.mogrify(q, {'expid': int(expid)})}" )
+        cursor.execute( q, { 'expid': int(expid) } )
+        columns = { cursor.description[i][0]: i for i in range(len(cursor.description)) }
+        app.logger.debug( f"Got {len(columns)} columns, {cursor.rowcount} rows" )
+
+        fields = ( 'id', 'ra', 'dec', 'gallat', 'section_id', 'fwhm_estimate', 'zero_point_estimate',
+                   'lim_mag_estimate', 'bkg_mean_estimate', 'bkg_rms_estimate',
+                   'numcutouts', 'nummeasurements', 'subid',
+                   'error_step', 'error_type', 'error_message', 'warnings',
+                   'process_memory', 'process_runtime', 'progress_steps_bitflag', 'products_exist_bitflag' )
+
+        retval = { 'status': 'ok', 'name': [] }
+        for field in fields :
+            retval[ field ] = []
+
+        lastimg = -1
+        slashre = re.compile( '^.*/([^/]+)$' )
+        for row in cursor.fetchall():
+            if row[columns['id']] == lastimg:
+                app.logger.warning( f'Multiple subtractions for image {lastimg}, need to deal with provenance!' )
+                continue
+            lastimg = row[columns['id']]
+
+            match = slashre.search( row[columns['filepath']] )
+            retval['name'].append( row[columns['filepath']] if match is None else match.group(1) )
+            for field in fields:
+                retval[field].append( row[columns[field]] )
+
+        return retval
+
+    except Exception as ex:
+        app.logger.exception( ex )
+        return { 'status': 'error',
+                 'error': f'Exception: {ex}' }
+
+# **********************************************************************
+
+@app.route( "/png_cutouts_for_sub_image/<int:exporsubid>/<int:issubid>/<int:nomeas>",
+            methods=['GET', 'POST'], strict_slashes=False )
+@app.route( "/png_cutouts_for_sub_image/<int:exporsubid>/<int:issubid>/<int:nomeas>/<int:limit>",
+            methods=['GET', 'POST'], strict_slashes=False )
+@app.route( "/png_cutouts_for_sub_image/<int:exporsubid>/<int:issubid>/<int:nomeas>/<int:limit>/<int:offset>",
+            methods=['GET', 'POST'], strict_slashes=False )
+def png_cutouts_for_sub_image( exporsubid, issubid, nomeas, limit=None, offset=0 ):
+    try:
+        data = { 'sortby': 'fluxdesc_chip_index' }
+        if flask.request.is_json:
+            data.update( flask.request.json )
+
+        app.logger.debug( f"Processing {flask.request.url}" )
+        if nomeas:
+            app.logger.debug( f"Looking for cutouts from subid {exporsubid} ({'with' if nomeas else 'without'} "
+                              f"missing-measurements)" )
+        else:
+            app.logger.debug( f"Looking for cutouts from exposure {exporsubid} ({'with' if nomeas else 'without'} "
+                              f"missing-measurements)" )
+
+        conn = next( dbconn() )
+        cursor = conn.cursor()
+        # TODO : deal with provenance!
+        # TODO : r/b and sorting
+
+        # Figure out the subids, zeropoints, backgrounds, and apertures we need
+
+        subids = []
+        zps = {}
+        dzps = {}
+        imageids = {}
+        newbkgs = {}
+        aperradses = {}
+        apercorses = {}
+
+        q = ( 'SELECT s.id AS subid, z.zp, z.dzp, z.aper_cor_radii, z.aper_cors, '
+              '  i.id AS imageid, i.bkg_mean_estimate '
+              'FROM images s '
+              'INNER JOIN image_upstreams_association ias ON ias.downstream_id=s.id '
+              '   AND s.ref_image_id != ias.upstream_id '
+              'INNER JOIN images i ON ias.upstream_id=i.id '
+              'INNER JOIN source_lists sl ON sl.image_id=i.id '
+              'INNER JOIN zero_points z ON sl.id=z.sources_id ' )
+        if issubid:
+            q += 'WHERE s.id=%(subid)s '
+            cursor.execute( q, { 'subid': exporsubid } )
+            cols = { cursor.description[i][0]: i for i in range(len(cursor.description)) }
+            rows = cursor.fetchall()
+            if len(rows) > 1:
+                app.logger.warning( f"Multiple zeropoints for subid {exporsubid}, deal with provenance" )
+            if len(rows) == 0:
+                app.logger.error( f"Couldn't find a zeropoint for subid {exporsubid}" )
+                zp = -99
+                dzp = -99
+                imageid = -99
+            subids.append( exporsubid )
+            zps[exporsubid] = rows[0][cols['zp']]
+            dzps[exporsubid] = rows[0][cols['dzp']]
+            imageids[exporsubid] = rows[0][cols['imageid']]
+            newbkgs[exporsubid] = rows[0][cols['bkg_mean_estimate']]
+            aperradses[exporsubid] = rows[0][cols['aper_cor_radii']]
+            apercorses[exporsubid] = rows[0][cols['aper_cors']]
+
+        else:
+            q += ( 'INNER JOIN exposures e ON i.exposure_id=e.id '
+                   'WHERE e.id=%(expid)s ORDER BY i.section_id  ' )
+            cursor.execute( q, { 'expid': exporsubid } )
+            cols = { cursor.description[i][0]: i for i in range(len(cursor.description)) }
+            rows = cursor.fetchall()
+            for row in rows:
+                subid = row[cols['subid']]
+                if ( subid in subids ):
+                    app.logger.warning( f"subid {subid} showed up more than once in zp qury, deal with provenance" )
+                subids.append( subid )
+                zps[subid] = row[cols['zp']]
+                dzps[subid] = row[cols['dzp']]
+                imageids[subid] = row[cols['imageid']]
+                newbkgs[subid] = row[cols['bkg_mean_estimate']]
+                aperradses[subid] = row[cols['aper_cor_radii']]
+                apercorses[subid] = row[cols['aper_cors']]
+
+        app.logger.debug( f"Getting cutouts for sub images {subids}" )
+        q = ( 'SELECT c.id AS id, c.filepath, c.ra, c.dec, c.x, c.y, c.index_in_sources, m.best_aperture, '
+              '       m.flux, m.dflux, m.name, m.is_test, m.is_fake, '
+              '       m.ra AS measra, m.dec AS measdec, s.id AS subid, s.section_id '
+              'FROM cutouts c '
+              'INNER JOIN source_lists sl ON c.sources_id=sl.id '
+              'INNER JOIN images s ON sl.image_id=s.id '
+              'LEFT JOIN '
+              '  ( SELECT meas.cutouts_id AS meascutid, meas.ra, meas.dec, meas.best_aperture, '
+              '           meas.flux_apertures[meas.best_aperture+1] AS flux, '
+              '           meas.flux_apertures_err[meas.best_aperture+1] AS dflux, obj.name, obj.is_test, obj.is_fake '
+              '    FROM measurements meas '
+              '    INNER JOIN objects obj ON meas.object_id=obj.id '
+             '   ) AS m ON m.meascutid=c.id '
+              'WHERE s.id IN %(subids)s ' )
+        if not nomeas:
+            q += "AND m.best_aperture IS NOT NULL "
+        if data['sortby'] == 'fluxdesc_chip_index':
+            q += 'ORDER BY flux DESC NULLS LAST,s.section_id,c.index_in_sources '
+        else:
+            raise RuntimeError( f"Unknown sort criterion {data['sortby']}" )
+        if limit is not None:
+            q += 'LIMIT %(limit)s OFFSET %(offset)s'
+        subdict = { 'subids': tuple(subids), 'limit': limit, 'offset': offset }
+        cursor.execute( q, subdict );
+        cols = { cursor.description[i][0]: i for i in range(len(cursor.description)) }
+        rows = cursor.fetchall()
+        app.logger.debug( f"Got {len(cols)} columns, {len(rows)} rows" )
+
+        hdf5files = {}
+        retval = { 'status': 'ok',
+                   'cutouts': {
+                       'sub_id': [],
+                       'image_id': [],
+                       'section_id': [],
+                       'id': [],
+                       'ra': [],
+                       'dec': [],
+                       'measra': [],
+                       'measdec': [],
+                       'flux': [],
+                       'dflux': [],
+                       'aperrad': [],
+                       'mag': [],
+                       'dmag': [],
+                       'objname': [],
+                       'is_test': [],
+                       'is_fake': [],
+                       'x': [],
+                       'y': [],
+                       'w': [],
+                       'h': [],
+                       'new_png': [],
+                       'ref_png': [],
+                       'sub_png': []
+                   }
+                  }
+
+        scaler = astropy.visualization.ZScaleInterval()
+
+        for row in rows:
+            subid = row[cols['subid']]
+            if row[cols['filepath']] not in hdf5files:
+                hdf5files[row[cols['filepath']]] = h5py.File( ARCHIVE_DIR / row[cols['filepath']], 'r' )
+            grp = hdf5files[row[cols['filepath']]][f'source_{row[cols["index_in_sources"]]}']
+            vmin, vmax = scaler.get_limits( grp['new_data'] )
+            scalednew = ( grp['new_data'] - vmin ) * 255. / ( vmax - vmin )
+            # TODO : there's an assumption here that the ref is background
+            #   subtracted. They probably usually will be -- that tends to be
+            #   part of a coadd process.
+            vmin -= newbkgs[subid]
+            vmax -= newbkgs[subid]
+            scaledref = ( grp['ref_data'] - vmin ) * 255. / ( vmax - vmin )
+            vmin, vmax = scaler.get_limits( grp['sub_data'] )
+            scaledsub = ( grp['sub_data'] - vmin ) * 255. / ( vmax - vmin )
+
+            scalednew[ scalednew < 0 ] = 0
+            scalednew[ scalednew > 255 ] = 255
+            scaledref[ scaledref < 0 ] = 0
+            scaledref[ scaledref > 255 ] = 255
+            scaledsub[ scaledsub < 0 ] = 0
+            scaledsub[ scaledsub > 255 ] = 255
+
+            scalednew = numpy.array( scalednew, dtype=numpy.uint8 )
+            scaledref = numpy.array( scaledref, dtype=numpy.uint8 )
+            scaledsub = numpy.array( scaledsub, dtype=numpy.uint8 )
+
+            # TODO : transpose, flip for principle of least surprise
+            # Figure out what PIL.Image does.
+            #  (this will affect w and h below)
+
+            newim = io.BytesIO()
+            refim = io.BytesIO()
+            subim = io.BytesIO()
+            PIL.Image.fromarray( scalednew ).save( newim, format='png' )
+            PIL.Image.fromarray( scaledref ).save( refim, format='png' )
+            PIL.Image.fromarray( scaledsub ).save( subim, format='png' )
+
+            retval['cutouts']['sub_id'].append( subid )
+            retval['cutouts']['image_id'].append( imageids[subid] )
+            retval['cutouts']['section_id'].append( row[cols['section_id']] )
+            retval['cutouts']['new_png'].append( base64.b64encode( newim.getvalue() ).decode('ascii') )
+            retval['cutouts']['ref_png'].append( base64.b64encode( refim.getvalue() ).decode('ascii') )
+            retval['cutouts']['sub_png'].append( base64.b64encode( subim.getvalue() ).decode('ascii') )
+            retval['cutouts']['id'].append( row[cols['id']] )
+            retval['cutouts']['ra'].append( row[cols['ra']] )
+            retval['cutouts']['dec'].append( row[cols['dec']] )
+            retval['cutouts']['x'].append( row[cols['x']] )
+            retval['cutouts']['y'].append( row[cols['y']] )
+            retval['cutouts']['w'].append( scalednew.shape[0] )
+            retval['cutouts']['h'].append( scalednew.shape[1] )
+            retval['cutouts']['objname'].append( row[cols['name']] )
+            retval['cutouts']['is_test'].append( row[cols['is_test']] )
+            retval['cutouts']['is_fake'].append( row[cols['is_fake']] )
+
+            # Measurements columns
+
+            # WARNING : assumption here that the aper cor radii list in the
+            #   zero point is the same as was used in the measurements.
+            # (I think that's a good assumption, but still.)
+
+            flux = row[cols['flux']]
+            dflux = row[cols['dflux']]
+            if flux is None:
+                for field in [ 'flux', 'dflux', 'aperrad', 'mag', 'dmag', 'measra', 'measdec' ]:
+                    retval['cutouts'][field].append( None )
+            else:
+                mag = -99
+                dmag = -99
+                if ( zps[subid] > 0 ) and ( flux > 0 ):
+                    mag = -2.5 * math.log10( flux ) + zps[subid] + apercorses[subid][ row[cols['best_aperture']] ]
+                    # Ignore zp and apercor uncertainties
+                    dmag = 1.0857 * dflux / flux
+                retval['cutouts']['measra'].append( row[cols['measra']] )
+                retval['cutouts']['measdec'].append( row[cols['measdec']] )
+                retval['cutouts']['flux'].append( flux )
+                retval['cutouts']['dflux'].append( dflux )
+                retval['cutouts']['aperrad'].append( aperradses[subid][ row[cols['best_aperture']] ] )
+                retval['cutouts']['mag'].append( mag )
+                retval['cutouts']['dmag'].append( dmag )
+
+        for f in hdf5files.values():
+            f.close()
+
+        app.logger.debug( f"Returning {len(retval['cutouts']['sub_id'])} cutouts" )
+        return retval
+
+    except Exception as ex:
+        app.logger.exception( ex )
+        return { 'status': 'error',
+                 'error': f'Exception: {ex}' }
diff --git a/webap/static/favicon.ico b/webap/static/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..199fa6033a79b7973f54b2f0d7a9d11a22cbd180
GIT binary patch
literal 2393
zcmV-f38wamP)<h;3K|Lk000e1NJLTq001BW001Be0ssI2{21+{0004mX+uL$Nkc;*
zaB^>EX>4Tx04R}tkv&MmKpe$iTZ<ws3U(0b5TrWUMMWHI6^c+H)C#RSm|Xe=O&XFE
z7e~Rh;NZt%)xpJCR|i)?5c~jfc5qU3krMxx6k5c1aNLh~_a1le0DrT}RI?`msG4PD
zQb{3~Uln>^5kwe+7)4ZKramW%X?TvWd-(Wz7w1{t=l&dFrC>6^ClDu?Zdk+{#50?g
z&Uv3W!pf3Dd`>)J&;^Mfxh}i>#<}FMpJzslY-XM~LM)WJSngt0HdNwi;+Udpl<&{E
ztZ?4qtXAu+eNX<vP(fQ+<~q$NQdq<iL<o>kM+H?_h|{W(Vj@HPNe};s;}^*#ldA$o
zjs?`9LUR1zfAG6ovp6;BCWVqf@Wr-2#(>^kpw+PL?_=9;odAJn;7aTGYfWJGlk`SM
ziyZ+2+rY(jN0aw}%N=0&NtX=CkpeXR#Uk*2M&FbNhHioWHMh6+K29HiEOnK>0S*p<
z@e*aPd%U}^ySIPOwEO!3jfHZtR#bvc00009a7bBm000b}000b}0h5oGng9R*2XskI
zMF;2w4G<+0d?Hq>0000PbVXQnLvL+uWo~o;Lvm$dbY)~9cWHEJAV*0}P*;Ht7XSbU
zLrFwIR7l6=R#|LaR~6m&oO928?_R&_H`#uE&#@ER8JxrkZW<C2($GQ$EeN3ssZ=V0
z3Mmy${|J-%M}-hVk@(=RA52n(m`a<9q#_E65=AMgNn*!|V<+(t+j)M2-*DfZ&OHMk
zw^FMz2qgM}*dIqn+B$o!z0cO#TMOJae3*vdL$?2Y0P?Th8_jmL-fVS(O0~*3NI*(*
z4G{t?o6V(dvoDwK$)yoO{{ev0@$M|GUYc6?-NhSahd<&r`vNx=`bp-CD8L#ZL_%ub
zpos^Dxzd%nZRpP*-8Hde(?@n}Nq2XBAe#36E}CCledFxqm)~A{s#%$+E@v06X|;vA
zMp##GFgT#PQa}g-P1g`%VQRf%tu0%!!-=`E?PE6!!@tRuUOcer^N;Nx8{PCl0LS%y
z{@VEyf1ZC~V|CxHi_z-LY_7PndB7dCm7T|lB#{DBY9@i|l!ej{h0^tiMS7>Tb-mSp
zx7s^5sd|Q{cRhNtfApn;LoYo2DLZAp3-F+_ar{TWnyLEVnz_(@^W0mdft8PrSh@Z}
zI-g5qb&Lo?N(w><=Ume?!!V*K@`DiTR3W`By;Wyj->B}IT--T*tu=D+2NRzj${9a>
z>Dk^~T5C0GU;W-MPnG?jTzS)2z541SyX-C72D*y9nJlF`V~jC|5F&)c7hTsoosLq<
zwr$S2Qpzw4Y9z|_#*+Uay?E>JZ!d&94jp^wbH}!2Pk!TR{Dm)m>BO0-=Qq|Ke&@A6
zd}h3`^U(Ib;UuM+2Ap%oSTdOe0M0o=NGTPDA*Ix^EXJ4+B6dZL=?MCg=_pq$+s@ug
ze-VRwe%1({nb?BvE>{54ZnxoKY;1mLC|4AMBZN5TQcBY_Ip;wTD5VfWnx;`o4a0C<
zH%<s+3;=P8VHo6Tk{2gJUALnUFm+rXfe1iy$I4hj3~DB0j8Yo&H4LNOZU;eNnx?Mn
zLWnq=<2V3d+cp3QAqXL17`m=YDa9CXbpj&f^;8a`0Jx8+j)6z7<Tu^lab(-%B-ON-
z8US=U9m6ne+ZI9yA>y=z5Q#*>vaBEogb<phan1=LmStg#(b`(^?Adcizw)+`gk~8O
zN5$SecrE5OPVRlY)wV`YoibW24Wan=lF6j+`(YRY0LC~j5h28J9L888k%*g}Qpy?Y
zo}M0l_4nuYKK0|CAy}K&{`zKDmrXX6`kpMN{#fVp%s;;8vFEBw`(M9kk7hQ;##)(7
zyWNhf5r!e>T-Wt54C5pyr9luVfMgh2*RPvbXIA?sUirjx*Oh{$X}EXg#Idh!9vajF
zw|(a9`S1MV{47p^hyY>G&=W!DK%-n(zbi`#+Slt55_NYgLgJcGLa^`SdR+?wyi{u$
zh56jTue;LCD1_<~)K?&I4i2P$`2Cj;?4KZ<^W~+*eTnj3-+#mC_lW@jr*-2rO+f#s
zCdJ`^4TfHJ)wMLuQ6dFeq-k~Cs3k`(U}$eQpj`tEpnMlX7eX(iqHf~mGlP>?ukG1A
zMpjnW=H_lE62_MDtzC9?&KP)y<Y#pgXc9nEN?zm<ByEr!5QZoKM1TbV3ZwuLLFj_u
z5C%x_k)*#<IBUF?85yxGYkqEa`Th!FVW{gmLTGk>uFzSXD7(AuPKBnLEwn^ZtB|;j
zup%ZX2~q+gAUV(kNDf@}2$5Bum3Fg9X1f{!Yw*quIW}fyvlwH;FdU~%qA22=*X#9c
zwhvN6_wt_U&W+rOP;Tq~+PqgNc$rL#OQBJsQx-*1N|4-j8W4qAvsoTyIhoHVih_$U
z4Bf6S!!!XPiXz5XtJNY3l&<SNJw5kR_Xo6r5*<hfL4EgPl%RcHtADo10sM4Y5)w!$
zq~t;fr6lLP(P%N2+x9r6)bqT!UsI8H6Y4CijOWK=r9zo>MhM~iewJsa?5SJT%l%?8
z@AcValQ%v(c?8vm!D=)!)oM?*nyb~a^?IsaCs9OfTgR9X0sxw(#Q>*|=+$Ur)E<qZ
zXm)mXY{zy|DiueDhnJR@A%gMc@npi7!qUk0^mpzNtR;*?>pNk7d!2@g2n7-9B6&g$
zr4(IXw=P|R_t6(ZjI#sD{v-g*&(FVj{OhGsNekR|quD(3)}QX)TZp2l)oNjkQ>l~^
z&FS7dV`y&<a#T^Jlu}Ac8J|*0w6apKU3su&d#lqLghAi;rIeiWIQn<L{msul`NVq@
zO-glRX6E9h%bw>+DILf0JkJ+?TW_aPI-krdr3fK0KrBr`r*kJ-G+oosj8rNG0AUzb
zDwT)#?fdex&uktV`Uf+avFP5y!qoJQ`bHzxAI4ZrZGC+`l}aImq?E4fVvJ4Gj9<ml
zjWPb@qenk}_)wux_@_hvy~V<D%IlSCwN|Oss?}PouTn}OL@cPWB4x9gLSJ90*k39Y
zOT`i)`0qn$|ITj8qR97sA%v1D9xs_p`oFVyy{pE$PIdkNbvOMR#qcHbsCn@+00000
LNkvXXu0mjfPTGAR

literal 0
HcmV?d00001

diff --git a/webap/static/seechange.css b/webap/static/seechange.css
new file mode 100644
index 00000000..218c65d8
--- /dev/null
+++ b/webap/static/seechange.css
@@ -0,0 +1,87 @@
+:root {
+    --link-color: #000066;
+    --link-decoration: underline;
+    --main-color: black;
+    --background-color: white;
+    --half-faded-color: #888888;
+    --some-faded-color: #666666;
+    --most-faded-color: #eeeeee;
+    --highlight-color: #993333;
+    --full-color-border: black;
+    --mid-color-border: #999999;
+    --mostlyfull-color-border: #666666;
+    --embiggen-background: #bbbbbb;
+    --tooltip-background: #ffff66;
+    --tooltip-border: #0000ff;
+    --tooltip-color: black;
+    --good-color: #006000;
+    --bad-color: #c00000;
+    --warning-color: #c06040;
+}
+
+.good { color: var(--good-color) }
+.bad { color: var(--bad-color) }
+.warning { color: var(--warning-color) }
+
+.italic { font-style: italic }
+.bold { font-weight: bold }
+
+.bgwhite { background: var(--background-color) }
+.bgfade { background: var(--most-faded-color) }
+.link { background: none; border: none; padding: 0;
+        color: var(--link-color); text-decoration: var(--link-decoration); cursor: pointer; }
+
+.vcenter { vertical-align: center }
+
+.tooltipsource { color: var(--link-color);
+                 text-decoration: var(--link-decoration);
+                 cursor: pointer; }
+.tooltipsource .tooltiptext { visibility: hidden;
+                              background-color: var(--tooltip-background);
+                              border: 1px solid var(--tooltip-border);
+                              color: var(--tooltip-color);
+                              z-index: 10;
+                              position: absolute;
+                            }
+.tooltipsource:hover .tooltiptext { visibility: visible; }
+
+
+.tabunsel { background: var(--most-faded-color);
+            border: 2px outset var(--full-color-border);
+            margin-left: 0.5ex;
+            margin-right: 0.5ex; }
+.tabsel { background: var(--half-faded-color);
+          border: 2px inset var(--full-color-border);
+          margin-left: 0.5ex;
+          margin-right: 0.5ex; }
+div.tabdiv {}
+div.buttonboxdiv {}
+div.tabcontentdiv { border: 2px solid var(--mostlyfull-color-border);
+                    padding: 0.5ex;
+                  }
+
+.padhalfex { padding: 0.5ex }
+
+.bold { font-weight: bold }
+
+img { image-rendering: pixelated }
+a { color: var(--link-color); text-decorastion: var(--link-decoration); cursor: pointer; }
+
+table { border: 2px solid var(--full-color-border); border-spacing: 2px }
+table th { border-bottom: 2px solid var(--full-color-border); font-weight: bold }
+table td { border-bottom: 2px solid var(--half-faded-color);
+           border-spacing: 2px;
+           padding: 2px; }
+table th.borderleft { border-left: 2px solid var(--half-faded-color); }
+
+table.exposurelist td { border: 2px solid var(--half-faded-color); }
+
+div.hbox { display: flex;
+           flex-direction: row;
+           min-width: 0;
+           min-height: 0; }
+
+div.vbox { display: flex;
+           flex-direction: column;
+           min-width: 0;
+           min-height: 0; }
diff --git a/webap/static/seechange.js b/webap/static/seechange.js
new file mode 100644
index 00000000..1959d360
--- /dev/null
+++ b/webap/static/seechange.js
@@ -0,0 +1,587 @@
+import { rkWebUtil } from "./rkwebutil.js";
+
+// Namespace, which is the only thing exported
+
+var seechange = {};
+
+seechange.nullorfixed = function( val, num ) { return val == null ? null : val.toFixed(num); }
+
+// **********************************************************************
+// **********************************************************************
+// **********************************************************************
+// The global context
+
+seechange.Context = function()
+{
+    this.parentdiv = document.getElementById( "pagebody" );
+    let h1 = rkWebUtil.elemaker( "h1", this.parentdiv, { "text": "SeeChange Webap" } );
+    this.maindiv = rkWebUtil.elemaker( "div", this.parentdiv );
+    this.frontpagediv = null;
+
+    // TODO : make this configurable?  Or at least remember how to
+    //   detect in javascript what the URL you're running from is.  (In
+    //   case the webap is not running as the root ap of the webserver.)
+    this.connector = new rkWebUtil.Connector( "/" );
+}
+
+seechange.Context.prototype.render_page = function()
+{
+    var self = this;
+
+    if ( this.frontpagediv == null ) {
+
+        // TODO : users, login
+
+        this.frontpagediv = rkWebUtil.elemaker( "div", this.maindiv );
+        let p = rkWebUtil.elemaker( "p", this.frontpagediv );
+        let button = rkWebUtil.button( p, "Show Exposures", function() { self.show_exposures(); } );
+        p.appendChild( document.createTextNode( " from " ) );
+        this.startdatewid = rkWebUtil.elemaker( "input", p,
+                                                { "attributes": { "type": "text",
+                                                                  "size": 20 } } );
+        this.startdatewid.addEventListener( "blur", function(e) {
+            rkWebUtil.validateWidgetDate( self.startdatewid );
+        } );
+        p.appendChild( document.createTextNode( " to " ) );
+        this.enddatewid = rkWebUtil.elemaker( "input", p,
+                                              { "attributes": { "type": "text",
+                                                                "size": 20 } } );
+        this.enddatewid.addEventListener( "blur", function(e) {
+            rkWebUtil.validateWidgetDate( self.enddatewid );
+        } );
+        p.appendChild( document.createTextNode( " (YYYY-MM-DD [HH:MM] — leave blank for no limit)" ) );
+    }
+    else {
+        rkWebUtil.wipeDiv( this.maindiv );
+        this.maindiv.appendChild( this.frontpagediv );
+    }
+}
+
+seechange.Context.prototype.show_exposures = function()
+{
+    var self = this;
+    var startdate, enddate;
+    try {
+        startdate = this.startdatewid.value.trim();
+        if ( startdate.length > 0 )
+            startdate = rkWebUtil.parseStandardDateString( startdate ).toISOString();
+        else startdate = null;
+        enddate = this.enddatewid.value.trim();
+        if ( enddate.length > 0 )
+            enddate = rkWebUtil.parseStandardDateString( enddate ).toISOString();
+        else enddate = null;
+    }
+    catch (ex) {
+        window.alert( "Error parsing at least one of the two dates:\n" + this.startdatewid.value +
+                      "\n" + this.enddatewid.value );
+        console.log( "Exception parsing dates: " + ex.toString() );
+        return;
+    }
+
+    this.connector.sendHttpRequest( "exposures", { "startdate": startdate, "enddate": enddate },
+                                    function( data ) { self.actually_show_exposures( data ); } );
+}
+
+seechange.Context.prototype.actually_show_exposures = function( data )
+{
+    if ( ! data.hasOwnProperty( "status" ) ) {
+        console.log( "return has no status: " + data.toString() );
+        window.alert( "Unexpected response from server when looking for exposures." );
+        return
+    }
+    let exps = new seechange.ExposureList( this, this.maindiv, data["exposures"], data["startdate"], data["enddate"] );
+    exps.render_page();
+}
+
+// **********************************************************************
+// **********************************************************************
+// **********************************************************************
+
+seechange.ExposureList = function( context, parentdiv, exposures, fromtime, totime )
+{
+    this.context = context;
+    this.parentdiv = parentdiv;
+    this.exposures = exposures;
+    this.fromtime = fromtime;
+    this.totime = totime;
+    this.div = null;
+}
+
+seechange.ExposureList.prototype.render_page = function()
+{
+    let self = this;
+
+    rkWebUtil.wipeDiv( this.parentdiv );
+
+    if ( this.div != null ) {
+        this.parentdiv.appendChild( this.div );
+        return
+    }
+
+    this.div = rkWebUtil.elemaker( "div", this.parentdiv );
+
+    var table, th, tr, td;
+
+    let p = rkWebUtil.elemaker( "p", this.div );
+    rkWebUtil.elemaker( "span", p, { "text": "[Back to exposure search]",
+                                     "classes": [ "link" ],
+                                     "click": () => { self.context.render_page() } } );
+    p.appendChild( document.createTextNode( "  —  " ) );
+    rkWebUtil.elemaker( "span", p, { "text": "[Refresh]",
+                                     "classes": [ "link" ],
+                                     "click": () => { rkWebUtil.wipeDiv( self.div );
+                                                      self.context.show_exposures(); } } );
+
+    let h2 = rkWebUtil.elemaker( "h2", this.div, { "text": "Exposures" } );
+    if ( ( this.fromtime == null ) && ( this.totime == null ) ) {
+        h2.appendChild( document.createTextNode( " from all time" ) );
+    } else if ( this.fromtime == null ) {
+        h2.appendChild( document.createTextNode( " up to " + this.totime ) );
+    } else if ( this.totime == null ) {
+        h2.appendChild( document.createTextNode( " from " + this.fromtime + " on" ) );
+    } else {
+        h2.appendChild( document.createTextNode( " from " + this.fromtime + " to " + this.totime ) );
+    }
+
+    table = rkWebUtil.elemaker( "table", this.div, { "classes": [ "exposurelist" ] } );
+    tr = rkWebUtil.elemaker( "tr", table );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "Exposure" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "MJD" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "target" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "filter" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "t_exp (s)" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "n_images" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "n_cutouts" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "n_sources" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "n_successim" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "n_errors" } );
+
+    this.tablerows = [];
+    let exps = this.exposures;   // For typing convenience...
+    // Remember, in javascript, "i in x" is like python "i in range(len(x))" or "i in x.keys()"
+    let fade = 1;
+    let countdown = 3;
+    for ( let i in exps["name"] ) {
+        let row = rkWebUtil.elemaker( "tr", table, { "classes": [ fade ? "bgfade" : "bgwhite" ] } );
+        this.tablerows.push( row );
+        td = rkWebUtil.elemaker( "td", row );
+        rkWebUtil.elemaker( "a", td, { "text": exps["name"][i],
+                                       "classes": [ "link" ],
+                                       "click": function() {
+                                           self.show_exposure( exps["id"][i],
+                                                               exps["name"][i],
+                                                               exps["mjd"][i],
+                                                               exps["filter"][i],
+                                                               exps["target"][i],
+                                                               exps["exp_time"][i] );
+                                       }
+                                     } );
+        td = rkWebUtil.elemaker( "td", row, { "text": exps["mjd"][i].toFixed(2) } );
+        td = rkWebUtil.elemaker( "td", row, { "text": exps["target"][i] } );
+        td = rkWebUtil.elemaker( "td", row, { "text": exps["filter"][i] } );
+        td = rkWebUtil.elemaker( "td", row, { "text": exps["exp_time"][i] } );
+        td = rkWebUtil.elemaker( "td", row, { "text": exps["n_images"][i] } );
+        td = rkWebUtil.elemaker( "td", row, { "text": exps["n_cutouts"][i] } );
+        td = rkWebUtil.elemaker( "td", row, { "text": exps["n_sources"][i] } );
+        td = rkWebUtil.elemaker( "td", row, { "text": exps["n_successim"][i] } );
+        td = rkWebUtil.elemaker( "td", row, { "text": exps["n_errors"][i] } );
+        countdown -= 1;
+        if ( countdown == 0 ) {
+            countdown = 3;
+            fade = 1 - fade;
+        }
+    }
+}
+
+seechange.ExposureList.prototype.show_exposure = function( id, name, mjd, filter, target, exp_time )
+{
+    let self = this;
+    this.context.connector.sendHttpRequest( "exposure_images/" + id, null,
+                                            (data) => {
+                                                self.actually_show_exposure( id, name, mjd, filter,
+                                                                             target, exp_time, data );
+                                            } );
+}
+
+seechange.ExposureList.prototype.actually_show_exposure = function( id, name, mjd, filter, target, exp_time, data )
+{
+    let exp = new seechange.Exposure( this, this.context, this.parentdiv,
+                                      id, name, mjd, filter, target, exp_time, data );
+    exp.render_page();
+}
+
+
+// **********************************************************************
+// **********************************************************************
+// **********************************************************************
+
+seechange.Exposure = function( exposurelist, context, parentdiv, id, name, mjd, filter, target, exp_time, data )
+{
+    this.exposurelist = exposurelist;
+    this.context = context;
+    this.parentdiv = parentdiv;
+    this.id = id;
+    this.name = name;
+    this.mjd = mjd;
+    this.filter = filter;
+    this.target = target;
+    this.exp_time = exp_time;
+    this.data = data;
+    this.div = null;
+    this.tabs = null;
+    this.imagesdiv = null;
+    this.cutoutsdiv = null;
+    this.cutoutsallimages_checkbox = null;
+    this.cutoutsimage_checkboxes = {};
+    this.cutouts = {};
+    this.cutouts_pngs = {};
+}
+
+// Copy this from models/enums_and_bitflags.py
+seechange.Exposure.process_steps = {
+    1: 'preprocessing',
+    2: 'extraction',
+    3: 'astro_cal',
+    4: 'photo_cal',
+    5: 'subtraction',
+    6: 'detection',
+    7: 'cutting',
+    8: 'measuring',
+};
+
+// Copy this from models/enums_and_bitflags.py
+seechange.Exposure.pipeline_products = {
+    1: 'image',
+    2: 'sources',
+    3: 'psf',
+    5: 'wcs',
+    6: 'zp',
+    7: 'sub_image',
+    8: 'detections',
+    9: 'cutouts',
+    10: 'measurements',
+}
+
+
+seechange.Exposure.prototype.render_page = function()
+{
+    let self = this;
+
+    rkWebUtil.wipeDiv( this.parentdiv );
+
+    if ( this.div != null ) {
+        this.parentdiv.appendChild( this.div );
+        return;
+    }
+
+    this.div = rkWebUtil.elemaker( "div", this.parentdiv );
+
+    var h2, h3, ul, li, table, tr, td, th, hbox, p, span, tiptext, ttspan;
+
+    rkWebUtil.elemaker( "p", this.div, { "text": "[Back to exposure list]",
+                                         "classes": [ "link" ],
+                                         "click": () => { self.exposurelist.render_page(); } } );
+
+    h2 = rkWebUtil.elemaker( "h2", this.div, { "text": "Exposure " + this.name } );
+    ul = rkWebUtil.elemaker( "ul", this.div );
+    li = rkWebUtil.elemaker( "li", ul );
+    li.innerHTML = "<b>target:</b> " + this.target;
+    li = rkWebUtil.elemaker( "li", ul );
+    li.innerHTML = "<b>mjd:</b> " + this.mjd
+    li = rkWebUtil.elemaker( "li", ul );
+    li.innerHTML = "<b>filter:</b> " + this.filter;
+    li = rkWebUtil.elemaker( "li", ul );
+    li.innerHTML = "<b>t_exp (s):</b> " + this.exp_time;
+
+    this.tabs = new rkWebUtil.Tabbed( this.parentdiv );
+
+
+    this.imagesdiv = rkWebUtil.elemaker( "div", null );
+
+    let totncutouts = 0;
+    let totnsources = 0;
+    for ( let i in this.data['id'] ) {
+        totncutouts += this.data['numcutouts'][i];
+        totnsources += this.data['nummeasurements'][i];
+    }
+
+    p = rkWebUtil.elemaker( "p", this.imagesdiv,
+                            { "text": "Exposure has " + this.data.id.length + " completed subtractions." } )
+    p = rkWebUtil.elemaker( "p", this.imagesdiv,
+                            { "text": ( totnsources.toString() + " out of " +
+                                        totncutouts.toString() + " sources pass preliminary cuts." ) } );
+
+    p = rkWebUtil.elemaker( "p", this.imagesdiv );
+
+    this.cutoutsallimages_checkbox =
+        rkWebUtil.elemaker( "input", p, { "attributes":
+                                          { "type": "radio",
+                                            "id": "cutouts_all_images",
+                                            "name": "whichimages_cutouts_checkbox",
+                                            "checked": "checked" } } );
+    rkWebUtil.elemaker( "span", p, { "text": " Show sources for all images" } );
+    p.appendChild( document.createTextNode( "      " ) );
+
+    this.cutoutssansmeasurements_checkbox =
+        rkWebUtil.elemaker( "input", p, { "attributes":
+                                          { "type": "checkbox",
+                                            "id": "cutouts_sans_measurements",
+                                            "name": "cutouts_sans_measurements_checkbox" } } );
+    rkWebUtil.elemaker( "label", p, { "text": "Show cutouts that failed the preliminary cuts",
+                                      "attributes": { "for": "cutouts_sans_measurements_checkbox" } } );
+
+
+    table = rkWebUtil.elemaker( "table", this.imagesdiv, { "classes": [ "exposurelist" ] } );
+    tr = rkWebUtil.elemaker( "tr", table );
+    th = rkWebUtil.elemaker( "th", tr );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "name" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "section" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "α" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "δ" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "b" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "fwhm" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "zp" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "mag_lim" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "n_cutouts" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "n_sources" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "compl. step" } );
+    th = rkWebUtil.elemaker( "th", tr, {} ); // products exist
+    th = rkWebUtil.elemaker( "th", tr, {} ); // error
+    th = rkWebUtil.elemaker( "th", tr, {} ); // warnings
+
+    let fade = 1;
+    let countdown = 3;
+    for ( let i in this.data['id'] ) {
+        tr = rkWebUtil.elemaker( "tr", table, { "classes": [ fade ? "bgfade" : "bgwhite" ] } );
+        td = rkWebUtil.elemaker( "td", tr );
+        this.cutoutsimage_checkboxes[ this.data['id'][i] ] =
+            rkWebUtil.elemaker( "input", td, { "attributes":
+                                               { "type": "radio",
+                                                 "id": this.data['id'][i],
+                                                 "name": "whichimages_cutouts_checkbox" } } )
+        td = rkWebUtil.elemaker( "td", tr, { "text": this.data['name'][i] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": this.data['section_id'][i] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": seechange.nullorfixed( this.data["ra"][i], 4 ) } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": seechange.nullorfixed( this.data["dec"][i], 4 ) } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": seechange.nullorfixed( this.data["gallat"][i], 1 ) } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": seechange.nullorfixed( this.data["fwhm_estimate"][i], 2 ) } );
+        td = rkWebUtil.elemaker( "td", tr,
+                                 { "text": seechange.nullorfixed( this.data["zero_point_estimate"][i], 2 ) } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": seechange.nullorfixed( this.data["lim_mag_estimate"][i], 1 ) } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": this.data["numcutouts"][i] } );
+        td = rkWebUtil.elemaker( "td", tr, { "text": this.data["nummeasurements"][i] } );
+
+        td = rkWebUtil.elemaker( "td", tr );
+        tiptext = "";
+        let laststep = "(none)";
+        for ( let j of Object.keys( seechange.Exposure.process_steps ) ) {
+            if ( this.data["progress_steps_bitflag"][i] & ( 2**j ) ) {
+                tiptext += seechange.Exposure.process_steps[j] + " done<br>";
+                laststep = seechange.Exposure.process_steps[j];
+            } else {
+                tiptext += "(" + seechange.Exposure.process_steps[j] + " not done)<br>";
+            }
+        }
+        span = rkWebUtil.elemaker( "span", td, { "classes": [ "tooltipsource" ],
+                                                 "text": laststep } );
+        ttspan = rkWebUtil.elemaker( "span", span, { "classes": [ "tooltiptext" ] } );
+        ttspan.innerHTML = tiptext;
+
+        td = rkWebUtil.elemaker( "td", tr );
+        tiptext = "Products created:";
+        for ( let j of Object.keys( seechange.Exposure.pipeline_products ) ) {
+            if ( this.data["products_exist_bitflag"][i] & ( 2**j ) )
+                tiptext += "<br>" + seechange.Exposure.pipeline_products[j];
+        }
+        span = rkWebUtil.elemaker( "span", td, { "classes": [ "tooltipsource" ],
+                                                 "text": "data products" } );
+        ttspan = rkWebUtil.elemaker( "span", span, { "classes": [ "tooltiptext" ] } );
+        ttspan.innerHTML = tiptext;
+
+        // Really I should be doing some HTML sanitization here on error message and, below, warnings....
+
+        td = rkWebUtil.elemaker( "td", tr );
+        if ( this.data["error_step"][i] != null ) {
+            span = rkWebUtil.elemaker( "span", td, { "classes": [ "tooltipsource" ],
+                                                     "text": "error" } );
+            tiptext = ( this.data["error_type"][i] + " error in step " +
+                        seechange.Exposure.process_steps[this.data["error_step"][i]] +
+                        " (" + this.data["error_message"][i].replaceAll( "\n", "<br>") + ")" );
+            ttspan = rkWebUtil.elemaker( "span", span, { "classes": [ "tooltiptext" ] } );
+            ttspan.innerHTML = tiptext;
+        }
+
+        td = rkWebUtil.elemaker( "td", tr );
+        if ( ( this.data["warnings"][i] != null ) && ( this.data["warnings"][i].length > 0 ) ) {
+            span = rkWebUtil.elemaker( "span", td, { "classes": [ "tooltipsource" ],
+                                                     "text": "warnings" } );
+            ttspan = rkWebUtil.elemaker( "span", span, { "classes": [ "tooltiptext" ] } );
+            ttspan.innerHTML = this.data["warnings"][i].replaceAll( "\n", "<br>" );
+        }
+    }
+
+
+    this.cutoutsdiv = rkWebUtil.elemaker( "div", null );
+
+    // TODO : buttons for next, prev, etc.
+
+    this.tabs.addTab( "Images", "Images", this.imagesdiv, true );
+    this.tabs.addTab( "Cutouts", "Sources", this.cutoutsdiv, false, ()=>{ self.update_cutouts() } );
+}
+
+
+seechange.Exposure.prototype.update_cutouts = function()
+{
+    var self = this;
+
+    rkWebUtil.wipeDiv( this.cutoutsdiv );
+
+    let withnomeas = this.cutoutssansmeasurements_checkbox.checked ? 1 : 0;
+
+    if ( this.cutoutsallimages_checkbox.checked ) {
+        rkWebUtil.elemaker( "p", this.cutoutsdiv,
+                            { "text": "Sources for all succesfully completed chips" } );
+        let div = rkWebUtil.elemaker( "div", this.cutoutsdiv );
+        rkWebUtil.elemaker( "p", div,
+                            { "text": "...updating cutouts...",
+                              "classes": [ "bold", "italic", "warning" ] } )
+
+        // TODO : offset and limit
+
+        let prop = "cutouts_for_all_images_for_exposure_" + withnomeas;
+        if ( this.cutouts_pngs.hasOwnProperty( prop ) ) {
+            this.show_cutouts_for_image( div, prop, this.cutouts_pngs[ prop ] );
+        }
+        else {
+            this.context.connector.sendHttpRequest(
+                "png_cutouts_for_sub_image/" + this.id + "/0/" + withnomeas,
+                {},
+                (data) => { self.show_cutouts_for_image( div, prop, data ); }
+            );
+        }
+    }
+    else {
+        for ( let i in this.data['id'] ) {
+            if ( this.cutoutsimage_checkboxes[this.data['id'][i]].checked ) {
+                rkWebUtil.elemaker( "p", this.cutoutsdiv,
+                                    { "text": "Sources for chip " + this.data['section_id'][i]
+                                      + " (image " + this.data['name'][i] + ")" } );
+
+                let div = rkWebUtil.elemaker( "div", this.cutoutsdiv );
+                rkWebUtil.elemaker( "p", div,
+                                    { "text": "...updating cutouts...",
+                                      "classes": [ "bold", "italic", "warning" ] } )
+
+                // TODO : offset and limit
+
+                let prop = this.data['id'][i].toString() + "_" + withnomeas;
+
+                if ( this.cutouts_pngs.hasOwnProperty( prop ) ) {
+                    this.show_cutouts_for_image( div, prop, this.cutouts_pngs[ prop ] );
+                }
+                else {
+                    this.context.connector.sendHttpRequest(
+                        "png_cutouts_for_sub_image/" + this.data['subid'][i] + "/1/" + withnomeas,
+                        {},
+                        (data) => { self.show_cutouts_for_image( div, prop, data ); }
+                    );
+                }
+
+                return;
+            }
+        }
+    }
+}
+
+
+seechange.Exposure.prototype.show_cutouts_for_image = function( div, dex, indata )
+{
+    var table, tr, th, td, img;
+    var oversample = 5;
+
+    if ( ! this.cutouts_pngs.hasOwnProperty( dex ) )
+        this.cutouts_pngs[dex] = indata;
+
+    var data = this.cutouts_pngs[dex];
+
+    rkWebUtil.wipeDiv( div );
+
+    table = rkWebUtil.elemaker( "table", div );
+    tr = rkWebUtil.elemaker( "tr", table );
+    th = rkWebUtil.elemaker( "th", tr );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "new" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "ref" } );
+    th = rkWebUtil.elemaker( "th", tr, { "text": "sub" } );
+
+    // Sorting is now done server-side... TODO, think about this
+    // // TODO : sort by r/b, make sort configurable
+    // let dexen = [...Array(data.cutouts.sub_id.length).keys()];
+    // dexen.sort( (a, b) => {
+    //     if ( ( data.cutouts['flux'][a] == null ) && ( data.cutouts['flux'][b] == null ) ) return 0;
+    //     else if ( data.cutouts['flux'][a] == null ) return 1;
+    //     else if ( data.cutouts['flux'][b] == null ) return -1;
+    //     else if ( data.cutouts['flux'][a] > data.cutouts['flux'][b] ) return -1;
+    //     else if ( data.cutouts['flux'][a] < data.cutouts['flux'][b] ) return 1;
+    //     else return 0;
+    // } );
+
+    // for ( let i of dexen ) {
+    for ( let i in data.cutouts.sub_id ) {
+        tr = rkWebUtil.elemaker( "tr", table );
+        td = rkWebUtil.elemaker( "td", tr );
+        if ( data.cutouts.objname[i] != null ) {
+            let text = "Object: " + data.cutouts.objname[i];
+            if ( data.cutouts.is_fake[i] ) text += " [FAKE]";
+            if ( data.cutouts.is_test[i] ) text += " [TEST]";
+            td.appendChild( document.createTextNode( text ) );
+        }
+        td = rkWebUtil.elemaker( "td", tr );
+        img = rkWebUtil.elemaker( "img", td,
+                                  { "attributes":
+                                    { "src": "data:image/png;base64," + data.cutouts['new_png'][i],
+                                      "width": oversample * data.cutouts['w'][i],
+                                      "height": oversample * data.cutouts['h'][i],
+                                      "alt": "new" } } );
+        td = rkWebUtil.elemaker( "td", tr );
+        img = rkWebUtil.elemaker( "img", td,
+                                  { "attributes":
+                                    { "src": "data:image/png;base64," + data.cutouts['ref_png'][i],
+                                      "width": oversample * data.cutouts['w'][i],
+                                      "height": oversample * data.cutouts['h'][i],
+                                      "alt": "ref" } } );
+        td = rkWebUtil.elemaker( "td", tr );
+        img = rkWebUtil.elemaker( "img", td,
+                                  { "attributes":
+                                    { "src": "data:image/png;base64," + data.cutouts['sub_png'][i],
+                                      "width": oversample * data.cutouts['w'][i],
+                                      "height": oversample * data.cutouts['h'][i],
+                                      "alt": "sub" } } );
+
+        td = rkWebUtil.elemaker( "td", tr );
+        let subdiv = rkWebUtil.elemaker( "div", td );
+        // TODO: use "warning" color for low r/b
+        if ( data.cutouts['flux'][i] == null ) td.classList.add( 'bad' );
+        else td.classList.add( 'good' );
+        subdiv.innerHTML = ( "<b>chip:</b> " + data.cutouts.section_id[i] + "<br>" +
+                             // "<b>cutout (α, δ):</b> (" + data.cutouts['ra'][i].toFixed(5) + " , "
+                             // + data.cutouts['dec'][i].toFixed(5) + ")<br>" +
+                             "<b>(α, δ):</b> (" + seechange.nullorfixed( data.cutouts['measra'][i], 5 ) + " , "
+                             + seechange.nullorfixed( data.cutouts['measdec'][i],5 ) + ")<br>" +
+                             "<b>(x, y):</b> (" + data.cutouts['x'][i].toFixed(2) + " , "
+                             + data.cutouts['y'][i].toFixed(2) + ")<br>" +
+                             "<b>Flux:</b> " + seechange.nullorfixed( data.cutouts['flux'][i], 0 )
+                             + " ± " + seechange.nullorfixed( data.cutouts['dflux'][i], 0 )
+                             + "  (aper r=" + seechange.nullorfixed( data.cutouts['aperrad'][i], 2) + " px)"
+                             + "<br>" + "<b>Mag:</b> " + seechange.nullorfixed( data.cutouts['mag'][i], 2 )
+                             + " ± " + seechange.nullorfixed( data.cutouts['dmag'][i], 2 )
+                           );
+    }
+}
+
+// **********************************************************************
+// **********************************************************************
+// **********************************************************************
+// Make this into a module
+
+export { seechange }
+
diff --git a/webap/static/seechange_start.js b/webap/static/seechange_start.js
new file mode 100644
index 00000000..a25589a4
--- /dev/null
+++ b/webap/static/seechange_start.js
@@ -0,0 +1,32 @@
+import { seechange } from "./seechange.js"
+
+// **********************************************************************
+// **********************************************************************
+// **********************************************************************
+// Here is the thing that will make the code run when the document has loaded
+// It only make sense when included in a HTML document after seechange.js.
+
+seechange.started = false
+
+// console.log("About to window.setInterval...");
+seechange.init_interval = window.setInterval(
+    function()
+    {
+        var requestdata, renderer;
+
+        if (document.readyState == "complete")
+        {
+            // console.log( "document.readyState is complete" );
+            if ( !seechange.started )
+            {
+                seechange.started = true;
+                window.clearInterval( seechange.init_interval );
+                renderer = new seechange.Context();
+                renderer.render_page();
+            }
+        }
+    },
+    100
+);
+
+export { }
diff --git a/webap/templates/base.html b/webap/templates/base.html
new file mode 100644
index 00000000..499e5434
--- /dev/null
+++ b/webap/templates/base.html
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  {% block head %}{% endblock %}
+  <link rel="shortcut icon" href="{{ url_for('static', filename='favicon.ico') }}">
+  <title>{% block title %}{% endblock %}</title>
+</head>
+<body>
+  {% block pagebody %}{% endblock %}
+</body>
+</html>
diff --git a/webap/templates/seechange_webap.html b/webap/templates/seechange_webap.html
new file mode 100644
index 00000000..3196d87a
--- /dev/null
+++ b/webap/templates/seechange_webap.html
@@ -0,0 +1,16 @@
+{% extends 'base.html' %}
+
+{% block head %}
+<!-- <link rel="icon" type="image.png" href="static/favicon.png"> -->
+<link rel="stylesheet" href="static/seechange.css">
+<script src="static/seechange.js" type="module"></script>
+<script src="static/seechange_start.js" type="module"></script>
+{% endblock %}
+
+{% block title %}SeeChange Webap{% endblock %}
+
+<h1>SeeChange Webap</h1>
+
+{% block pagebody %}
+<div name="pagebody" id="pagebody"></div>
+{% endblock %}

From d14463fe48927d22e76e8e3125dc4cc633ab3a41 Mon Sep 17 00:00:00 2001
From: Rob Knop <rknop@pobox.com>
Date: Wed, 12 Jun 2024 10:19:36 -0700
Subject: [PATCH 08/11] Add missing webap_secrets/seechange_webap_config.py
 file (#309)

---
 tests/webap_secrets/seechange_webap_config.py |  6 ++++++
 webap/Makefile                                | 16 ----------------
 2 files changed, 6 insertions(+), 16 deletions(-)
 create mode 100644 tests/webap_secrets/seechange_webap_config.py

diff --git a/tests/webap_secrets/seechange_webap_config.py b/tests/webap_secrets/seechange_webap_config.py
new file mode 100644
index 00000000..0539807b
--- /dev/null
+++ b/tests/webap_secrets/seechange_webap_config.py
@@ -0,0 +1,6 @@
+PG_HOST = 'seechange_postgres'
+PG_PORT = 5432
+PG_USER = 'postgres'
+PG_PASS = 'fragile'
+PG_NAME = 'seechange'
+ARCHIVE_DIR = pathlib.Path( '/archive-storage/base/test' )
diff --git a/webap/Makefile b/webap/Makefile
index d0aaf8b4..387ec0f9 100644
--- a/webap/Makefile
+++ b/webap/Makefile
@@ -1,12 +1,5 @@
 INSTALLDIR = install
 
-#PG_HOST = placeholder
-#PG_PORT = placeholder
-#PG_USER = placeholder
-#PG_PASS = placeholder
-#PG_NAME = placeholder
-#ARCHIVE_DIR = placeholder
-
 toinstall = seechange_webap.py \
 	static/seechange.css static/seechange.js static/seechange_start.js static/rkwebutil.js static/favicon.ico \
 	templates/base.html templates/seechange_webap.html
@@ -19,14 +12,5 @@ install: $(patsubst %, $(INSTALLDIR)/%, $(toinstall))
 $(INSTALLDIR)/%: %
 	install -Dp $< $@
 
-#seechange_webap_config.py:
-#	echo import pathlib > seechange_webap_config.py
-#	echo PG_HOST = \"$(PG_HOST)\" >> seechange_webap_config.py
-#	echo PG_PORT = $(PG_PORT) >> seechange_webap_config.py
-#	echo PG_USER = \"$(PG_USER)\" >> seechange_webap_config.py
-#	echo PG_PASS = \"$(PG_PASS)\" >> seechange_webap_config.py
-#	echo PG_NAME = \"$(PG_NAME)\" >> seechange_webap_config.py
-#	echo ARCHIVE_DIR = pathlib.Path\( \"$(ARCHIVE_DIR)\" \) >> seechange_webap_config.py
-
 static/rkwebutil.js: rkwebutil/rkwebutil.js
 	ln -s ../rkwebutil/rkwebutil.js static/rkwebutil.js

From 8ac1ce13dd3cb5607ee2a54fa821cb4d1761c908 Mon Sep 17 00:00:00 2001
From: Guy Nir <37179063+guynir42@users.noreply.github.com>
Date: Wed, 12 Jun 2024 23:36:29 -0700
Subject: [PATCH 09/11] Combine astro_cal and photo_cal into extraction (#296)

Also, add siblings to get_downstreams, split up pipeline tests
---
 .github/workflows/run-pipeline-tests-1.yml    |  64 ++
 ...ine-tests.yml => run-pipeline-tests-2.yml} |   5 +-
 default_config.yaml                           |  68 +-
 docs/overview.md                              |  99 ++-
 docs/setup.md                                 |   2 +-
 improc/alignment.py                           |   2 +-
 improc/photometry.py                          |   5 +-
 improc/sextrsky.py                            |   3 +
 improc/tools.py                               |  10 +-
 models/base.py                                |  44 +-
 models/cutouts.py                             |   2 +-
 models/enums_and_bitflags.py                  |   4 +-
 models/exposure.py                            |   2 +-
 models/image.py                               |  15 +-
 models/measurements.py                        |   2 +-
 models/provenance.py                          |   1 +
 models/psf.py                                 |  55 +-
 models/reference.py                           |   4 +-
 models/source_list.py                         |  53 +-
 models/world_coordinates.py                   |  67 +-
 models/zero_point.py                          |  76 +-
 pipeline/astro_cal.py                         |   8 +-
 pipeline/coaddition.py                        |  97 ++-
 pipeline/cutting.py                           |   2 +-
 pipeline/data_store.py                        | 752 ++++++++----------
 pipeline/detection.py                         |  11 +-
 pipeline/measuring.py                         |   2 +-
 pipeline/parameters.py                        |  44 +-
 pipeline/photo_cal.py                         |   2 +-
 pipeline/preprocessing.py                     |   7 +-
 pipeline/subtraction.py                       |  14 +-
 pipeline/top_level.py                         | 181 +++--
 pipeline/utils.py                             |   0
 tests/fixtures/decam.py                       |   5 +-
 tests/fixtures/pipeline_objects.py            | 266 ++++---
 tests/fixtures/ptf.py                         | 108 ++-
 tests/fixtures/simulated.py                   |   8 +-
 tests/improc/test_alignment.py                |   1 -
 tests/models/test_decam.py                    |   2 -
 tests/models/test_image.py                    |   8 +-
 tests/models/test_measurements.py             |  34 +
 tests/models/test_objects.py                  |   5 +-
 tests/models/test_provenance.py               |   2 +
 tests/models/test_ptf.py                      |   6 +
 tests/models/test_reports.py                  |  20 +-
 tests/models/test_source_list.py              |   4 +-
 tests/pipeline/test_coaddition.py             |  25 +-
 tests/pipeline/test_detection.py              |   1 +
 tests/pipeline/test_extraction.py             |   4 +-
 tests/pipeline/test_measuring.py              |   2 +-
 tests/pipeline/test_pipeline.py               | 209 +++--
 util/util.py                                  |   6 +
 52 files changed, 1411 insertions(+), 1008 deletions(-)
 create mode 100644 .github/workflows/run-pipeline-tests-1.yml
 rename .github/workflows/{run-pipeline-tests.yml => run-pipeline-tests-2.yml} (92%)
 delete mode 100644 pipeline/utils.py

diff --git a/.github/workflows/run-pipeline-tests-1.yml b/.github/workflows/run-pipeline-tests-1.yml
new file mode 100644
index 00000000..af0b121f
--- /dev/null
+++ b/.github/workflows/run-pipeline-tests-1.yml
@@ -0,0 +1,64 @@
+name: Run Pipeline Tests 1
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  tests:
+    name: run tests in docker image
+    runs-on: ubuntu-latest
+    env:
+      REGISTRY: ghcr.io
+      COMPOSE_FILE: tests/docker-compose.yaml
+
+    steps:
+      - name: Dump docker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+
+      - name: checkout code
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: log into github container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: setup docker buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver: docker-container
+
+      - name: bake
+        uses: docker/bake-action@v2.3.0
+        with:
+          workdir: tests
+          load: true
+          files: docker-compose.yaml
+          set: |
+            seechange_postgres.tags=ghcr.io/${{ github.repository_owner }}/seechange-postgres
+            seechange_postgres.cache-from=type=gha,scope=cached-seechange-postgres
+            seechange_postgres.cache-to=type=gha,scope=cached-seechange-postgres,mode=max
+            setuptables.tags=ghcr.io/${{ github.repository_owner }}/runtests
+            setuptables.cache-from=type=gha,scope=cached-seechange
+            setuptables.cache-to=type=gha,scope=cached-seechange,mode=max
+            runtests.tags=ghcr.io/${{ github.repository_owner }}/runtests
+            runtests.cache-from=type=gha,scope=cached-seechange
+            runtests.cache-to=type=gha,scope=cached-seechange,mode=max
+            shell.tags=ghcr.io/${{ github.repository_owner }}/runtests
+            shell.cache-from=type=gha,scope=cached-seechange
+            shell.cache-to=type=gha,scope=cached-seechange,mode=max
+
+      - name: run test
+        run: |
+          df -h
+          shopt -s nullglob
+          TEST_SUBFOLDER=$(ls tests/pipeline/test_{a..o}*.py) docker compose run runtests
diff --git a/.github/workflows/run-pipeline-tests.yml b/.github/workflows/run-pipeline-tests-2.yml
similarity index 92%
rename from .github/workflows/run-pipeline-tests.yml
rename to .github/workflows/run-pipeline-tests-2.yml
index b1b24cbe..a94c2422 100644
--- a/.github/workflows/run-pipeline-tests.yml
+++ b/.github/workflows/run-pipeline-tests-2.yml
@@ -1,4 +1,4 @@
-name: Run Pipeline Tests
+name: Run Pipeline Tests 2
 
 on:
   push:
@@ -59,4 +59,5 @@ jobs:
 
       - name: run test
         run: |
-          TEST_SUBFOLDER=tests/pipeline docker compose run runtests
+          shopt -s nullglob
+          TEST_SUBFOLDER=$(ls tests/pipeline/test_{p..z}*.py) docker compose run runtests
diff --git a/default_config.yaml b/default_config.yaml
index 034e5e44..dcbe863a 100644
--- a/default_config.yaml
+++ b/default_config.yaml
@@ -82,23 +82,24 @@ preprocessing:
   use_sky_subtraction: False
 
 extraction:
-  measure_psf: true
-  threshold: 3.0
-  method: sextractor
-
-astro_cal:
-  cross_match_catalog: gaia_dr3
-  solution_method: scamp
-  max_catalog_mag: [20.0]
-  mag_range_catalog: 4.0
-  min_catalog_stars: 50
-  max_sources_to_use: [2000, 1000, 500, 200]
-
-photo_cal:
-  cross_match_catalog: gaia_dr3
-  max_catalog_mag: [20.0]
-  mag_range_catalog: 4.0
-  min_catalog_stars: 50
+  sources:
+    measure_psf: true
+    threshold: 3.0
+    method: sextractor
+
+  wcs:
+    cross_match_catalog: gaia_dr3
+    solution_method: scamp
+    max_catalog_mag: [20.0]
+    mag_range_catalog: 4.0
+    min_catalog_stars: 50
+    max_sources_to_use: [2000, 1000, 500, 200]
+
+  zp:
+    cross_match_catalog: gaia_dr3
+    max_catalog_mag: [20.0]
+    mag_range_catalog: 4.0
+    min_catalog_stars: 50
 
 subtraction:
   method: zogy
@@ -170,22 +171,23 @@ coaddition:
       ignore_flags: 0
   # The following are used to override the regular "extraction" parameters
   extraction:
-    measure_psf: true
-    threshold: 3.0
-    method: sextractor
-    # The following are used to override the regular "astro_cal" parameters
-  astro_cal:
-    cross_match_catalog: gaia_dr3
-    solution_method: scamp
-    max_catalog_mag: [22.0]
-    mag_range_catalog: 6.0
-    min_catalog_stars: 50
-    # The following are used to override the regular "photo_cal" parameters
-  photo_cal:
-    cross_match_catalog: gaia_dr3
-    max_catalog_mag: [22.0]
-    mag_range_catalog: 6.0
-    min_catalog_stars: 50
+    sources:
+      measure_psf: true
+      threshold: 3.0
+      method: sextractor
+    # The following are used to override the regular astrometric calibration parameters
+    wcs:
+      cross_match_catalog: gaia_dr3
+      solution_method: scamp
+      max_catalog_mag: [22.0]
+      mag_range_catalog: 6.0
+      min_catalog_stars: 50
+    # The following are used to override the regular photometric calibration parameters
+    zp:
+      cross_match_catalog: gaia_dr3
+      max_catalog_mag: [22.0]
+      mag_range_catalog: 6.0
+      min_catalog_stars: 50
 
 
 # DECam
diff --git a/docs/overview.md b/docs/overview.md
index 988d187e..ba782e1c 100644
--- a/docs/overview.md
+++ b/docs/overview.md
@@ -16,7 +16,7 @@ SeeChange consists of a main pipeline that takes raw images and produces a few d
  - Cutouts around the sources detected in the difference images, along with the corresponding 
    image cutouts from the reference and the newly acquired images. 
  - Measurements on those cutouts, including the photometric flux, the shapes, and some 
-   metrics that indicate if the source is astronomical or an artefact (e.g., using deep neural classifiers). 
+   metrics that indicate if the source is astronomical or an artefact (using analytical cuts). 
 
 Additional pipelines for making bias frames, flat frames, and to produce deep coadded references
 are described separately. 
@@ -48,7 +48,7 @@ Additional folders include:
  - `extern`: external packages that are used by SeeChange, including the `nersc-upload-connector` 
    package that is used to connect the archive.
  - `improc: image processing code that is used by the pipeline, generally manipulating images
-   in ways that are not specific to a single point in the pipeline (e.g., image segmentation).
+   in ways that are not specific to a single point in the pipeline (e.g., image alignment or inpainting).
  - `tests`: tests for the pipeline (more on that below). 
  - `utils`: generic utility functions that are used by the pipeline. 
 
@@ -56,7 +56,7 @@ The source code is found in `pipeline`, `models`, `improc` and `utils`.
 Notable files in the `pipeline` folder include `data_store.py` (described below)
 and the `top_level.py` file that defines the `Pipeline` object, 
 which is the main entry point for running the pipeline.
-In `models` we define the `base.py` file, which contains tools for 
+In `models` we have the `base.py` file, which contains tools for 
 database communications, along with some useful mixin classes, 
 and the `instrument.py` file, which contains the `Instrument` 
 base class used to define various instruments from different surveys. 
@@ -71,24 +71,20 @@ Here is a list of the processes and their data products (including the object cl
  - preprocessing: dark, bias, flat, fringe corrections, etc. For large, segmented focal planes,
    will also segment the input raw data into "sections" that usually correspond to individual CCDs. 
    This process takes an `Exposure` object and produces `Image` objects, one for each section/CCD. 
- - extraction: find the sources in the pre-processed image. 
-   This process takes an `Image` object and produces a `SourceList` object, and also a `PSF` object.
- - astro_cal: astrometric calibration, i.e., matching the detected sources' positions
-   to an external catalog, and fitting the WCS solution that maps image pixel coordinates
-   to "real world" coordinates on the sky. Generally we use Gaia DR3 as the reference catalog.
-   This process uses the `SourceList` object and produces an `WorldCoordinates` object.
- - photo_cal: photometric calibration, i.e., matching the detected sources' fluxes
-   to an external catalog, and fitting the photometric zero point (ZP) that maps the 
-   instrumental fluxes to the intrinsic brightness (magnitude) of the stars. 
-   We can use Gaia DR3 for this matching, but this can be configured to use other catalogs.
-    This process uses the `SourceList` object and produces a `ZeroPoint` object.
+ - extraction: find the sources in the pre-processed image, measure their PSF, cross-match them 
+   for astrometric and photometric calibration. 
+   This process takes an `Image` object and produces a `SourceList`, a 'PSF', a 'WorldCoordinates', 
+   and a 'ZeroPoint' object. 
+   The astrometric and photometric steps were integrated into "extraction" to simplify the pipeline.
+   The WorldCoordinates object is a WCS solution that maps image pixel coordinates to sky coordinates.
+   The ZeroPoint object is a photometric solution that maps instrumental fluxes to magnitudes.
  - subtraction: taking a reference image of the same part of the sky (usually a deep coadd)
    and subtracting it from the "new" image (the one being processed by the pipeline). 
    Different algorithms can be used to match the PSFs of the new and reference image
-   (we currently implement HOTPANTS and ZOGY). This process uses the `Image` object, 
-   along with all the other data products produced so far in the pipeline, and another
-   `Image` object for the reference (this image comes with its own set of data products)
-   and produces a subtraction `Image` object. 
+   (we currently implement ZOGY, but HOTPANTS and SFFT will be added later). 
+ - This process uses the `Image` object, along with all the other data products 
+   produced so far in the pipeline, and another `Image` object for the reference 
+   (this image comes with its own set of data products) and produces a subtraction `Image` object. 
  - detection: finding the sources in the difference image. 
    This process uses the difference `Image` object and produces a `SourceList` object.
    This new source list is different from the previous one, as it contains information only
@@ -99,8 +95,8 @@ Here is a list of the processes and their data products (including the object cl
    Additional pixel data could optionally be scraped from other surveys (like PanSTARRS or DECaLS).
    Each source that was detected in the difference image gets a separate `Cutouts` object. 
  - measuring: this part of the pipeline measures the fluxes and shapes of the sources 
-   in the cutouts. It uses a set of analytical cuts to and also a deep neural network classifier 
-   to distinguish between astronomical sources and artefacts. 
+   in the cutouts. It uses a set of analytical cuts to 
+   distinguish between astronomical sources and artefacts. 
    This process uses the list of `Cutouts` objects 
    to produce a list of `Measurements` objects, one for each source. 
 
@@ -138,7 +134,7 @@ ds = DataStore(image_id=123456)
 Note that the `Image` and `Exposure` IDs are internal database identifiers, 
 while the section ID is defined by the instrument used, and usually refers
 to the CCD number or name (it can be an integer or a string). 
-E.g., the DECam sections are named `N1`, `N2`, ... `S1`, S2`, etc.
+E.g., the DECam sections are named `N1`, `N2`, ... `S1`, `S2`, etc.
 
 Once a datastore is initialized, it can be used to query for any data product:
 
@@ -156,21 +152,23 @@ There could be multiple versions of the same data product,
 produced with different parameters or code versions.
 A user may choose to pass a `provenance` input to the `get` methods,
 to specify which version of the data product is requested. 
-If no provenance is specified, the object with the latest provenance is returned. 
+If no provenance is specified, the provenance is loaded either
+from the datastore's general `prov_tree` dictionary, or if it doesn't exist, 
+will just load the most recently created provenance for that pipeline step. 
 
 ```python
 from models.provenance import Provenance
 prov = Provenance(
-   process='photo_cal', 
+   process='extraction', 
    code_version=code_version, 
    parameters=parameters, 
    upstreams=upstream_provs
 )
 # or, using the datastore's tool to get the "right" provenance:
-prov = ds.get_provenance(process='photo_cal', pars_dict=parameters)
+prov = ds.get_provenance(process='extraction', pars_dict=parameters)
 
 # then you can get a specific data product, with the parameters and code version:
-zp = ds.get_zero_point(provenance=prov)
+sources = ds.get_sources(provenance=prov)
 ```
 
 See below for more information about versioning using the provenance model. 
@@ -180,13 +178,17 @@ See below for more information about versioning using the provenance model.
 
 Each part of the pipeline (each process) is conducted using a dedicated object. 
  - preprocessing: using the `Preprocessor` object defined in `pipeline/preprocessing.py`.
- - extraction: using the `Detector` object defined in `pipeline/detection.py`.
- - astro_cal: using the `AstroCalibrator` object defined in `pipeline/astro_cal.py`.
- - photo_cal: using the `PhotoCalibrator` object defined in `pipeline/photo_cal.py`.
- - subtraction: using the `Subtractor` object defined in `pipeline/subtraction.py`.
- - detection: again using the `Detector` object, with a different set of parameters.
- - cutting: using the `Cutter` object defined in `pipeline/cutting.py`.
- - measuring: using the `Measurer` object defined in `pipeline/measuring.py`.
+ - extraction: using the `Detector` object defined in `pipeline/detection.py` to produce the `SourceList` and `PSF` 
+   objects. A sub dictionary keyed by "sources" is used to define the parameters for these objects. 
+   The astrometric and photometric calibration are also done in this step.
+   The astrometric calibration using the `AstroCalibrator` object defined in `pipeline/astro_cal.py`, 
+   with a sub dictionary keyed by "wcs", produces the `WorldCoordinates` object. 
+   The photometric calibration is done using the `PhotoCalibrator` object defined in
+   `pipeline/photo_cal.py`, with a sub dictionary keyed by "zp", produces the `ZeroPoint` object.
+ - subtraction: using the `Subtractor` object defined in `pipeline/subtraction.py`, producing an `Image` object.
+ - detection: again using the `Detector` object, with different parameters, also producing a `SourceList` object.
+ - cutting: using the `Cutter` object defined in `pipeline/cutting.py`, producing a list of `Cutouts` objects.
+ - measuring: using the `Measurer` object defined in `pipeline/measuring.py`, producing a list of `Measurements` objects.
 
 All these objects are initialized as attributes of a top level `Pipeline` object,
 which is defined in `pipeline/top_level.py`. 
@@ -194,13 +196,13 @@ Each of these objects can be configured using a dictionary of parameters.
 
 There are three ways to configure any object in the pipeline. 
 The first is using a `Config` object, which is defined in `util/config.py`.
-This object reads one or more YAML files and stores the parameters in a dictionary heirarchy.
+This object reads one or more YAML files and stores the parameters in a dictionary hierarchy.
 More on how to initialize this object can be found in the `configuration.md` document. 
 Keys in this dictionary can include `pipeline`, `preprocessing`, etc. 
 Each of those keys should map to another dictionary, with parameter choices for that process.
 
 After the config files are read in, the `Pipeline` object can also be initialized using
-a heirarchical dictionary: 
+a hierarchical dictionary: 
 
 ```python
 from pipeline.top_level import Pipeline
@@ -212,7 +214,7 @@ p = Pipeline(
 )
 ```
 
-If only a single object from the pipeline needs to be initialized, 
+If only a single object needs to be initialized, 
 pass the parameters directly to the object's constructor:
 
 ```python
@@ -223,7 +225,7 @@ pp = Preprocessor(
 )
 ```
 
-Finally, after all objects are intialized with their parameters, 
+Finally, after all objects are initialized with their parameters, 
 a user (e.g., in an interactive session) can modify any of the parameters
 using the `pars` attribute of the object. 
 
@@ -256,7 +258,7 @@ The `Provenance` object is defined in `models/provenance.py`.
 
 The `Provenance` object is initialized with the following inputs:
  - `process`: the name of the process that produced this data product ('preprocessing', 'subtraction', etc.).
- - `code_version`: the version of the code that was used to produce this data product.
+ - `code_version`: the version object for the code that was used to produce this data product.
  - `parameters`: a dictionary of parameters that were used to produce this data product.
  - `upstreams`: a list of `Provenance` objects that were used to produce this data product.
 
@@ -275,7 +277,9 @@ Only parameters that affect the product values are included.
 The upstreams are other `Provenance` objects defined for the data products that 
 are an input to the current processing step. 
 The flowchart of the different process steps is defined in `pipeline.datastore.UPSTREAM_STEPS`. 
-E.g., the upstreams for the `photo_cal` object are `['extraction', 'astro_cal']`. 
+E.g., the upstreams for the `subtraction` object are `['preprocessing', 'extraction', 'reference']`.
+Note that the `reference` upstream is replaced by the provenances 
+of the reference's `preprocessing` and `extraction` steps.
 
 When a `Provenance` object has all the required inputs, it will produce a hash identifier
 that is unique to that combination of inputs.
@@ -301,18 +305,19 @@ It is useful to get familiar with the naming convention for different data produ
  - `PSF`: a model of the point spread function (PSF) of an image. 
    This is linked to a single `Image` and will contain the PSF model for that image.
  - `WorldCoordinates`: a set of transformations used to convert between image pixel coordinates and sky coordinates. 
-   This is linked to a single `Image` and will contain the WCS information for that image.
+   This is linked to a single `SourceList` (and from it to an `Image`) and will contain the WCS information for that image.
  - `ZeroPoint`: a photometric solution that converts image flux to magnitudes. 
-   This is linked to a single `Image` and will contain the zeropoint information for that image.
+   This is linked to a single `SourceList` (and from it to an  `Image`) and will contain the zeropoint information for that image.
  - `Object`: a table that contains information about a single astronomical object (real or bogus), 
    such as its RA, Dec, and magnitude. Each `Object` is linked to a list of `Measurements` objects.  
  - `Cutouts`: contain the small pixel stamps around a point in the sky in a new image, reference image, and 
-   subtraction image. Could contain additional, external imaging data from other surveys. 
+   subtraction image. Could contain additional, external imaging data from other surveys.
+   Each `Cutouts` object is linked back to a subtraction based `SourceList`. 
  - `Measurements`: contains measurements made on the information in the `Cutouts`.  
-   These include flux+errors, magnitude+errors, centroid positions, spot width, machine learning scores, etc. 
+   These include flux+errors, magnitude+errors, centroid positions, spot width, analytical cuts, etc. 
  - `Provenance`: A table containing the code version and critical parameters that are unique to this version of the data. 
    Each data product above must link back to a provenance row, so we can recreate the conditions that produced this data. 
- - `Reference`: An object that links a reference `Image`  with a specific field/target, a section ID, 
+ - `Reference`: An object that links a reference `Image` with a specific field/target, a section ID, 
    and a time validity range, that allows users to quickly identify which reference goes with a new image. 
  - `CalibratorFile`: An object that tracks data needed to apply calibration (preprocessing) for a specific instrument.
    The calibration could include an `Image` data file, or a generic non-image `DataFile` object. 
@@ -355,6 +360,12 @@ These include:
    describing the bounding box of the object on the sky.
    This is particularly useful for images but also for catalog excerpts, 
    that span a small region of the sky.
+ - `HasBitFlagBadness`: adds a `_bitflag` and `_upstream_bitflag` columns to the model.
+   These allow flagging of bad data products, either because they are bad themselves, or 
+   because one of their upstreams is bad. It also adds some methods and attributes to access
+   the badness like `badness` and `append_badness()`. 
+   If you change the bitflag of such an object, and it was already used to produce downstream products, 
+   make sure to use `update_downstream_badness()` to recursively update the badness of all downstream products.
 
 Enums and bitflag are stored on the database as integers
 (short integers for Enums and long integers for bitflags).
@@ -381,7 +392,7 @@ some caching of cross-match catalogs also helps speed things up.
 
 When running on a cluster/supercomputer, there is usually an abundance of CPU cores, 
 so running multiple sections at once, or even multiple exposures (each with many sections), 
-is not a problem, and simplfies the processing. 
+is not a problem, and simplifies the processing. 
 
 Additional parallelization can be achieved by using multi-threaded code
 on specific bottlenecks in the pipeline, but this is not yet implemented.
diff --git a/docs/setup.md b/docs/setup.md
index 908bc6e8..d18598b9 100644
--- a/docs/setup.md
+++ b/docs/setup.md
@@ -49,7 +49,7 @@ By default, the volumes with archived files and the database files will still be
    docker compose down -v
 ```
 
-If all is well, the `-v` will delete the volumnes that stored the database and archive files.
+If all is well, the `-v` will delete the volumes that stored the database and archive files.
 
 You can see what volumes docker knows about with
 ```
diff --git a/improc/alignment.py b/improc/alignment.py
index a606d35d..aae18a27 100644
--- a/improc/alignment.py
+++ b/improc/alignment.py
@@ -414,7 +414,7 @@ def _align_swarp( self, image, target, sources, target_sources ):
 
             # re-calculate the source list and PSF for the warped image
             extractor = Detector()
-            extractor.pars.override(sources.provenance.parameters, ignore_addons=True)
+            extractor.pars.override(sources.provenance.parameters['sources'], ignore_addons=True)
             warpedsrc, warpedpsf, _, _ = extractor.extract_sources(warpedim)
             warpedim.sources = warpedsrc
             warpedim.psf = warpedpsf
diff --git a/improc/photometry.py b/improc/photometry.py
index 94fbb004..4ea12415 100644
--- a/improc/photometry.py
+++ b/improc/photometry.py
@@ -419,6 +419,8 @@ def calc_at_position(data, radius, annulus, xgrid, ygrid, cx, cy, local_bg=True,
         the iterative process.
     """
     flux = area = background = variance = norm = cxx = cyy = cxy = 0
+    if np.all(np.isnan(data)):
+        return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
 
     # make a circle-mask based on the centroid position
     if not np.isfinite(cx) or not np.isfinite(cy):
@@ -447,7 +449,8 @@ def calc_at_position(data, radius, annulus, xgrid, ygrid, cx, cy, local_bg=True,
             return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
 
         annulus_map_sum = np.nansum(annulus_map)
-        if annulus_map_sum == 0:  # this should only happen in tests or if the annulus is way too large
+        if annulus_map_sum == 0 or np.all(np.isnan(annulus_map)):
+            # this should only happen in tests or if the annulus is way too large or if all pixels are NaN
             background = 0
             variance = 0
             norm = 0
diff --git a/improc/sextrsky.py b/improc/sextrsky.py
index a457c6d6..255ced03 100644
--- a/improc/sextrsky.py
+++ b/improc/sextrsky.py
@@ -9,6 +9,7 @@
 
 from util.logger import SCLogger
 
+
 def single_sextrsky( imagedata, maskdata=None, sigcut=3 ):
     """Estimate sky and sky sigma of imagedata (ignoreing nonzero maskdata pixels)
 
@@ -66,6 +67,7 @@ def single_sextrsky( imagedata, maskdata=None, sigcut=3 ):
     skysig = 1.4826 * ( np.median( np.abs( imagedata[w] - sky ) ) )
     return sky, skysig
 
+
 def sextrsky( imagedata, maskdata=None, sigcut=3, boxsize=200, filtsize=3 ):
     """Estimate sky using an approximation of the SExtractor algorithm.
 
@@ -178,6 +180,7 @@ def sextrsky( imagedata, maskdata=None, sigcut=3, boxsize=200, filtsize=3 ):
 
 # ======================================================================
 
+
 def main():
     parser = argparse.ArgumentParser( description="Estimate image sky using sextractor algorithm" )
     parser.add_argument( "image", help="Image filename" )
diff --git a/improc/tools.py b/improc/tools.py
index 19c5842b..e37b8ebc 100644
--- a/improc/tools.py
+++ b/improc/tools.py
@@ -52,13 +52,15 @@ def sigma_clipping(values, nsigma=3.0, iterations=5, axis=None, median=False):
             raise ValueError("values must be a vector, image, or cube")
 
     values = values.copy()
-    
-    # first iteration:
-    mean = np.nanmedian(values, axis=axis)
-    rms = np.nanstd(values, axis=axis)
 
     # how many nan values?
     nans = np.isnan(values).sum()
+    if nans == values.size:
+        return np.nan, np.nan
+
+    # first iteration:
+    mean = np.nanmedian(values, axis=axis)
+    rms = np.nanstd(values, axis=axis)
 
     for i in range(iterations):
         # remove pixels that are more than nsigma from the median
diff --git a/models/base.py b/models/base.py
index cb42e354..f1a96801 100644
--- a/models/base.py
+++ b/models/base.py
@@ -45,6 +45,17 @@
 
 # this is the root SeeChange folder
 CODE_ROOT = os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))
+#
+# # printout the list of relevant environmental variables:
+# print("SeeChange environment variables:")
+# for key in [
+#     'INTERACTIVE',
+#     'LIMIT_CACHE_USAGE',
+#     'SKIP_NOIRLAB_DOWNLOADS',
+#     'RUN_SLOW_TESTS',
+#     'SEECHANGE_TRACEMALLOC',
+# ]:
+#     print(f'{key}: {os.getenv(key)}')
 
 
 # This is a list of warnings that are categorically ignored in the pipeline. Beware:
@@ -327,8 +338,13 @@ def get_upstreams(self, session=None):
         """Get all data products that were directly used to create this object (non-recursive)."""
         raise NotImplementedError('get_upstreams not implemented for this class')
 
-    def get_downstreams(self, session=None):
-        """Get all data products that were created directly from this object (non-recursive)."""
+    def get_downstreams(self, session=None, siblings=True):
+        """Get all data products that were created directly from this object (non-recursive).
+
+        This optionally includes siblings: data products that are co-created in the same pipeline step
+        and depend on one another. E.g., a source list and psf have an image upstream and a (subtraction?) image
+        as a downstream, but they are each other's siblings.
+        """
         raise NotImplementedError('get_downstreams not implemented for this class')
 
     def delete_from_database(self, session=None, commit=True, remove_downstreams=False):
@@ -354,11 +370,16 @@ def delete_from_database(self, session=None, commit=True, remove_downstreams=Fal
         if session is None and not commit:
             raise RuntimeError("When session=None, commit must be True!")
 
-        with SmartSession(session) as session:
+        with SmartSession(session) as session, warnings.catch_warnings():
+            warnings.filterwarnings(
+                action='ignore',
+                message=r'.*DELETE statement on table .* expected to delete \d* row\(s\).*',
+            )
+
             need_commit = False
             if remove_downstreams:
                 try:
-                    downstreams = self.get_downstreams()
+                    downstreams = self.get_downstreams(session=session)
                     for d in downstreams:
                         if hasattr(d, 'delete_from_database'):
                             if d.delete_from_database(session=session, commit=False, remove_downstreams=True):
@@ -1676,14 +1697,14 @@ def append_badness(self, value):
         doc='Free text comment about this data product, e.g., why it is bad. '
     )
 
-    def update_downstream_badness(self, session=None, commit=True):
+    def update_downstream_badness(self, session=None, commit=True, siblings=True):
         """Send a recursive command to update all downstream objects that have bitflags.
 
         Since this function is called recursively, it always updates the current
         object's _upstream_bitflag to reflect the state of this object's upstreams,
         before calling the same function on all downstream objects.
 
-        Note that this function will session.add() this object and all its
+        Note that this function will session.merge() this object and all its
         recursive downstreams (to update the changes in bitflag) and will
         commit the new changes on its own (unless given commit=False)
         but only at the end of the recursion.
@@ -1698,6 +1719,11 @@ def update_downstream_badness(self, session=None, commit=True):
             provide a commit=True to commit the changes.
         commit: bool (default True)
             Whether to commit the changes to the database.
+        siblings: bool (default True)
+            Whether to also update the siblings of this object.
+            Default is True. This is usually what you want, but
+            anytime this function calls itself, it uses siblings=False,
+            to avoid infinite recursion.
         """
         # make sure this object is current:
         with SmartSession(session) as session:
@@ -1710,10 +1736,10 @@ def update_downstream_badness(self, session=None, commit=True):
             if hasattr(merged_self, '_upstream_bitflag'):
                 merged_self._upstream_bitflag = new_bitflag
 
-            # recursively do this for all the other objects
-            for downstream in merged_self.get_downstreams(session):
+            # recursively do this for all downstream objects
+            for downstream in merged_self.get_downstreams(session=session, siblings=siblings):
                 if hasattr(downstream, 'update_downstream_badness') and callable(downstream.update_downstream_badness):
-                    downstream.update_downstream_badness(session=session, commit=False)
+                    downstream.update_downstream_badness(session=session, siblings=False, commit=False)
 
             if commit:
                 session.commit()
diff --git a/models/cutouts.py b/models/cutouts.py
index fa98d78b..b35e153e 100644
--- a/models/cutouts.py
+++ b/models/cutouts.py
@@ -669,7 +669,7 @@ def get_upstreams(self, session=None):
         with SmartSession(session) as session:
             return session.scalars(sa.select(SourceList).where(SourceList.id == self.sources_id)).all()
         
-    def get_downstreams(self, session=None):
+    def get_downstreams(self, session=None, siblings=False):
         """Get the downstream Measurements that were made from this Cutouts object. """
         from models.measurements import Measurements
 
diff --git a/models/enums_and_bitflags.py b/models/enums_and_bitflags.py
index c928c0de..a979b657 100644
--- a/models/enums_and_bitflags.py
+++ b/models/enums_and_bitflags.py
@@ -418,12 +418,10 @@ class BitFlagConverter( EnumConverter ):
     _dict_inverse = None
 
 
-# the list of possible processing steps from a section of an exposure up to measurments, r/b scores, and report
+# the list of possible processing steps from a section of an exposure up to measurements, r/b scores, and report
 process_steps_dict = {
     1: 'preprocessing',  # creates an Image from a section of the Exposure
     2: 'extraction',     # creates a SourceList from an Image, and a PSF
-    3: 'astro_cal',      # creates a WorldCoordinates from a SourceList
-    4: 'photo_cal',       # creates a ZeroPoint from a WorldCoordinates
     5: 'subtraction',    # creates a subtraction Image
     6: 'detection',      # creates a SourceList from a subtraction Image
     7: 'cutting',        # creates Cutouts from a subtraction Image
diff --git a/models/exposure.py b/models/exposure.py
index e14e11cb..052d3269 100644
--- a/models/exposure.py
+++ b/models/exposure.py
@@ -736,7 +736,7 @@ def get_upstreams(self, session=None):
         """An exposure does not have any upstreams. """
         return []
 
-    def get_downstreams(self, session=None):
+    def get_downstreams(self, session=None, siblings=False):
         """An exposure has only Image objects as direct downstreams. """
         from models.image import Image
 
diff --git a/models/image.py b/models/image.py
index 7d7f44c8..c2b09e8c 100644
--- a/models/image.py
+++ b/models/image.py
@@ -482,6 +482,7 @@ def __init__(self, *args, **kwargs):
 
         self._instrument_object = None
         self._bitflag = 0
+        self.is_sub = False
 
         if 'header' in kwargs:
             kwargs['_header'] = kwargs.pop('header')
@@ -545,14 +546,14 @@ def merge_all(self, session):
             self.sources.provenance_id = self.sources.provenance.id if self.sources.provenance is not None else None
             new_image.sources = self.sources.merge_all(session=session)
 
-            new_image.wcs = new_image.sources.wcs
-            if new_image.wcs is not None:
+            if new_image.sources.wcs is not None:
+                new_image.wcs = new_image.sources.wcs
                 new_image.wcs.sources = new_image.sources
                 new_image.wcs.sources_id = new_image.sources.id
                 new_image.wcs.provenance_id = new_image.wcs.provenance.id if new_image.wcs.provenance is not None else None
 
-            new_image.zp = new_image.sources.zp
-            if new_image.zp is not None:
+            if new_image.sources.zp is not None:
+                new_image.zp = new_image.sources.zp
                 new_image.zp.sources = new_image.sources
                 new_image.zp.sources_id = new_image.sources.id
                 new_image.zp.provenance_id = new_image.zp.provenance.id if new_image.zp.provenance is not None else None
@@ -1797,7 +1798,7 @@ def get_upstreams(self, session=None):
 
         return upstreams
 
-    def get_downstreams(self, session=None):
+    def get_downstreams(self, session=None, siblings=False):
         """Get all the objects that were created based on this image. """
         # avoids circular import
         from models.source_list import SourceList
@@ -1808,9 +1809,7 @@ def get_downstreams(self, session=None):
         downstreams = []
         with SmartSession(session) as session:
             # get all psfs that are related to this image (regardless of provenance)
-            psfs = session.scalars(
-                sa.select(PSF).where(PSF.image_id == self.id)
-            ).all()
+            psfs = session.scalars(sa.select(PSF).where(PSF.image_id == self.id)).all()
             downstreams += psfs
             if self.psf is not None and self.psf not in psfs:  # if not in the session, could be duplicate!
                 downstreams.append(self.psf)
diff --git a/models/measurements.py b/models/measurements.py
index 59df8e3e..547660f6 100644
--- a/models/measurements.py
+++ b/models/measurements.py
@@ -492,7 +492,7 @@ def get_upstreams(self, session=None):
         with SmartSession(session) as session:
             return session.scalars(sa.select(Cutouts).where(Cutouts.id == self.cutouts_id)).all()
 
-    def get_downstreams(self, session=None):
+    def get_downstreams(self, session=None, siblings=False):
         """Get the downstreams of this Measurements"""
         return []
 
diff --git a/models/provenance.py b/models/provenance.py
index f5022c09..2b9ced8a 100644
--- a/models/provenance.py
+++ b/models/provenance.py
@@ -367,6 +367,7 @@ def merge_concurrent(self, session=None, commit=True):
 
         return output
 
+
 @event.listens_for(Provenance, "before_insert")
 def insert_new_dataset(mapper, connection, target):
     """
diff --git a/models/psf.py b/models/psf.py
index 8e4fc2ac..e8272a1c 100644
--- a/models/psf.py
+++ b/models/psf.py
@@ -1,4 +1,3 @@
-import re
 import pathlib
 
 import numpy as np
@@ -527,7 +526,55 @@ def get_upstreams(self, session=None):
         with SmartSession(session) as session:
             return session.scalars(sa.select(Image).where(Image.id == self.image_id)).all()
         
-    def get_downstreams(self, session=None):
-        """Get the downstreams of this PSF (currently none)"""
-        return []
+    def get_downstreams(self, session=None, siblings=False):
+        """Get the downstreams of this PSF.
+
+        If siblings=True then also include the SourceLists, WCSes, ZPs and background objects
+        that were created at the same time as this PSF.
+        """
+        from models.source_list import SourceList
+        from models.world_coordinates import WorldCoordinates
+        from models.zero_point import ZeroPoint
+        from models.provenance import Provenance
+
+        with SmartSession(session) as session:
+            subs = session.scalars(
+                sa.select(Image).where(
+                    Image.provenance.has(Provenance.upstreams.any(Provenance.id == self.provenance.id)),
+                    Image.upstream_images.any(Image.id == self.image_id),
+                )
+            ).all()
+            output = subs
+
+            if siblings:
+                # There should be exactly one source list, wcs, and zp per PSF, with the same provenance
+                # as they are created at the same time.
+                sources = session.scalars(
+                    sa.select(SourceList).where(
+                        SourceList.image_id == self.image_id, SourceList.provenance_id == self.provenance_id
+                    )
+                ).all()
+                if len(sources) != 1:
+                    raise ValueError(f"Expected exactly one source list for PSF {self.id}, but found {len(sources)}")
+
+                output.append(sources[0])
+
+                # TODO: add background object
+
+                wcs = session.scalars(
+                    sa.select(WorldCoordinates).where(WorldCoordinates.sources_id == sources.id)
+                ).all()
+                if len(wcs) != 1:
+                    raise ValueError(f"Expected exactly one wcs for PSF {self.id}, but found {len(wcs)}")
+
+                output.append(wcs[0])
+
+                zp = session.scalars(sa.select(ZeroPoint).where(ZeroPoint.sources_id == sources.id)).all()
+
+                if len(zp) != 1:
+                    raise ValueError(f"Expected exactly one zp for PSF {self.id}, but found {len(zp)}")
+
+                output.append(zp[0])
+
+        return output
     
diff --git a/models/reference.py b/models/reference.py
index 20aca8d3..780411c2 100644
--- a/models/reference.py
+++ b/models/reference.py
@@ -219,7 +219,7 @@ def load_upstream_products(self, session=None):
 
             sources = session.scalars(
                 sa.select(SourceList).where(
-                    SourceList.image_id == self.image_id,
+                    SourceList.image_id == self.image.id,
                     SourceList.provenance_id.in_(prov_ids),
                 )
             ).all()
@@ -233,7 +233,7 @@ def load_upstream_products(self, session=None):
 
             psfs = session.scalars(
                 sa.select(PSF).where(
-                    PSF.image_id == self.image_id,
+                    PSF.image_id == self.image.id,
                     PSF.provenance_id.in_(prov_ids),
                 )
             ).all()
diff --git a/models/source_list.py b/models/source_list.py
index d6962bf5..2daf1bbd 100644
--- a/models/source_list.py
+++ b/models/source_list.py
@@ -635,7 +635,6 @@ def save(self, **kwargs):
         self.num_sources = len( self.data )
         super().save(fullname, **kwargs)
 
-
     def free( self, ):
         """Free loaded source list memory.
 
@@ -647,7 +646,6 @@ def free( self, ):
         self._data = None
         self._info = None
 
-
     @staticmethod
     def _convert_from_sextractor_to_numpy( arr, copy=False ):
         """Convert from 1-offset to 0-offset coordinates.
@@ -751,25 +749,52 @@ def get_upstreams(self, session=None):
         with SmartSession(session) as session:
             return session.scalars(sa.select(Image).where(Image.id == self.image_id)).all()
 
-    def get_downstreams(self, session=None):
-        """Get all the data products (WCSs and ZPs) that are made using this source list. """
+    def get_downstreams(self, session=None, siblings=False):
+        """Get all the data products that are made using this source list.
+
+        If siblings=True then also include the PSFs, WCSes, ZPs and background objects
+        that were created at the same time as this SourceList.
+        """
+        from models.psf import PSF
         from models.world_coordinates import WorldCoordinates
         from models.zero_point import ZeroPoint
         from models.cutouts import Cutouts
-        from models.psf import PSF
         from models.provenance import Provenance
 
         with SmartSession(session) as session:
-            wcs = session.scalars(sa.select(WorldCoordinates).where(WorldCoordinates.sources_id == self.id)).all()
-            zps = session.scalars(sa.select(ZeroPoint).where(ZeroPoint.sources_id == self.id)).all()
-            cutouts = session.scalars(sa.select(Cutouts).where(Cutouts.sources_id == self.id)).all()
-            subs = session.scalars(sa.select(Image)
-                                   .where(Image.provenance
-                                          .has(Provenance.upstreams
-                                               .any(Provenance.id == self.provenance.id)))).all()
-             
-        return wcs + zps + cutouts + subs
+            subs = session.scalars(
+                sa.select(Image).where(
+                    Image.provenance.has(Provenance.upstreams.any(Provenance.id == self.provenance.id)),
+                    Image.upstream_images.any(Image.id == self.image_id),
+                )
+            ).all()
+            output = subs
+
+            if self.is_sub:
+                cutouts = session.scalars(sa.select(Cutouts).where(Cutouts.sources_id == self.id)).all()
+                output += cutouts
+            elif siblings:  # for "detections" we don't have siblings
+                psfs = session.scalars(
+                    sa.select(PSF).where(PSF.image_id == self.image_id, PSF.provenance_id == self.provenance_id)
+                ).all()
+                if len(psfs) != 1:
+                    raise ValueError(f"Expected exactly one PSF for SourceList {self.id}, but found {len(psfs)}")
+
+                # TODO: add background object
+
+                wcs = session.scalars(sa.select(WorldCoordinates).where(WorldCoordinates.sources_id == self.id)).all()
+                if len(wcs) != 1:
+                    raise ValueError(
+                        f"Expected exactly one WorldCoordinates for SourceList {self.id}, but found {len(wcs)}"
+                    )
+                zps = session.scalars(sa.select(ZeroPoint).where(ZeroPoint.sources_id == self.id)).all()
+                if len(zps) != 1:
+                    raise ValueError(
+                        f"Expected exactly one ZeroPoint for SourceList {self.id}, but found {len(zps)}"
+                    )
+                output += psfs + wcs + zps
 
+        return output
 
     def show(self, **kwargs):
         """Show the source positions on top of the image.
diff --git a/models/world_coordinates.py b/models/world_coordinates.py
index 5e5ad91f..d4676115 100644
--- a/models/world_coordinates.py
+++ b/models/world_coordinates.py
@@ -13,6 +13,7 @@
 
 from models.base import Base, SmartSession, AutoIDMixin, HasBitFlagBadness, FileOnDiskMixin, SeeChangeBase
 from models.enums_and_bitflags import catalog_match_badness_inverse
+from models.image import Image
 from models.source_list import SourceList
 
 
@@ -102,25 +103,57 @@ def get_upstreams(self, session=None):
         with SmartSession(session) as session:
             return session.scalars(sa.select(SourceList).where(SourceList.id == self.sources_id)).all()
         
-    def get_downstreams(self, session=None):
-        """Get the downstreams of this WorldCoordinates"""
-        # get the ZeroPoint that uses the same SourceList as this WCS
+    def get_downstreams(self, session=None, siblings=False):
+        """Get the downstreams of this WorldCoordinates.
+
+        If siblings=True  then also include the SourceLists, PSFs, ZPs and background objects
+        that were created at the same time as this WorldCoordinates.
+        """
+        from models.source_list import SourceList
+        from models.psf import PSF
         from models.zero_point import ZeroPoint
-        from models.image import Image
         from models.provenance import Provenance
-        with SmartSession(session) as session:
-            zps = session.scalars(sa.select(ZeroPoint) 
-                                  .where(ZeroPoint.provenance 
-                                         .has(Provenance.upstreams 
-                                              .any(Provenance.id == self.provenance.id)))).all()
-
-            subs = session.scalars(sa.select(Image)
-                                   .where(Image.provenance
-                                          .has(Provenance.upstreams
-                                               .any(Provenance.id == self.provenance.id)))).all()
-
-        downstreams = zps + subs
-        return downstreams
+
+        with (SmartSession(session) as session):
+            subs = session.scalars(
+                sa.select(Image).where(
+                    Image.provenance.has(Provenance.upstreams.any(Provenance.id == self.provenance.id)),
+                    Image.upstream_images.any(Image.id == self.sources.image_id),
+                )
+            ).all()
+            output = subs
+
+            if siblings:
+                sources = session.scalars(sa.select(SourceList).where(SourceList.id == self.sources_id)).all()
+                if len(sources) > 1:
+                    raise ValueError(
+                        f"Expected exactly one SourceList for WorldCoordinates {self.id}, but found {len(sources)}."
+                    )
+
+                output.append(sources[0])
+
+                psf = session.scalars(
+                    sa.select(PSF).where(
+                        PSF.image_id == sources.image_id, PSF.provenance_id == self.provenance_id
+                    )
+                ).all()
+
+                if len(psf) > 1:
+                    raise ValueError(f"Expected exactly one PSF for WorldCoordinates {self.id}, but found {len(psf)}.")
+
+                output.append(psf[0])
+
+                # TODO: add background object
+
+                zp = session.scalars(sa.select(ZeroPoint).where(ZeroPoint.sources_id == sources.id)).all()
+
+                if len(zp) > 1:
+                    raise ValueError(
+                        f"Expected exactly one ZeroPoint for WorldCoordinates {self.id}, but found {len(zp)}."
+                    )
+                output.append(zp[0])
+
+        return output
 
     def save( self, filename=None, **kwargs ):
         """Write the WCS data to disk.
diff --git a/models/zero_point.py b/models/zero_point.py
index 5daf321e..257a6dc8 100644
--- a/models/zero_point.py
+++ b/models/zero_point.py
@@ -9,6 +9,7 @@
 from models.base import Base, SmartSession, AutoIDMixin, HasBitFlagBadness, FileOnDiskMixin, SeeChangeBase
 from models.enums_and_bitflags import catalog_match_badness_inverse
 from models.world_coordinates import WorldCoordinates
+from models.image import Image
 from models.source_list import SourceList
 
 
@@ -136,29 +137,58 @@ def get_aper_cor( self, rad ):
 
     def get_upstreams(self, session=None):
         """Get the extraction SourceList and WorldCoordinates used to make this ZeroPoint"""
-        from models.provenance import Provenance
         with SmartSession(session) as session:
-            source_list = session.scalars(sa.select(SourceList).where(SourceList.id == self.sources_id)).all()
-
-            wcs_prov_id = None
-            for prov in self.provenance.upstreams:
-                if prov.process == "astro_cal":
-                    wcs_prov_id = prov.id
-            wcs = []
-            if wcs_prov_id is not None:
-                wcs = session.scalars(sa.select(WorldCoordinates) 
-                                    .where(WorldCoordinates.provenance 
-                                            .has(Provenance.id == wcs_prov_id))).all()
-
-        return source_list + wcs
-    
-    def get_downstreams(self, session=None):
-        """Get the downstreams of this ZeroPoint"""
-        from models.image import Image
+            sources = session.scalars(sa.select(SourceList).where(SourceList.id == self.sources_id)).all()
+
+        return sources
+
+    def get_downstreams(self, session=None, siblings=False):
+        """Get the downstreams of this ZeroPoint.
+
+        If siblings=True then also include the SourceLists, PSFs, WCSes, and background objects
+        that were created at the same time as this ZeroPoint.
+        """
+        from models.source_list import SourceList
+        from models.psf import PSF
+        from models.world_coordinates import WorldCoordinates
         from models.provenance import Provenance
+
         with SmartSession(session) as session:
-            subs = session.scalars(sa.select(Image)
-                                    .where(Image.provenance
-                                            .has(Provenance.upstreams
-                                                .any(Provenance.id == self.provenance.id)))).all()
-        return subs
+            subs = session.scalars(
+                sa.select(Image).where(
+                    Image.provenance.has(Provenance.upstreams.any(Provenance.id == self.provenance.id)),
+                    Image.upstream_images.any(Image.id == self.sources.image_id),
+                )
+            ).all()
+            output = subs
+
+            if siblings:
+                sources = session.scalars(sa.select(SourceList).where(SourceList.id == self.sources_id)).all()
+                if len(sources) > 1:
+                    raise ValueError(
+                        f"Expected exactly one SourceList for ZeroPoint {self.id}, but found {len(sources)}."
+                    )
+                output.append(sources[0])
+
+                psf = session.scalars(
+                    sa.select(PSF).where(
+                        PSF.image_id == sources.image_id, PSF.provenance_id == self.provenance_id
+                    )
+                ).all()
+                if len(psf) > 1:
+                    raise ValueError(f"Expected exactly one PSF for ZeroPoint {self.id}, but found {len(psf)}.")
+
+                output.append(psf[0])
+
+                # TODO: add background object
+
+                wcs = session.scalars(
+                    sa.select(WorldCoordinates).where(WorldCoordinates.sources_id == sources.id)
+                ).all()
+
+                if len(wcs) > 1:
+                    raise ValueError(f"Expected exactly one WCS for ZeroPoint {self.id}, but found {len(wcs)}.")
+
+                output.append(wcs[0])
+
+        return output
diff --git a/pipeline/astro_cal.py b/pipeline/astro_cal.py
index 3106eeda..0f74a1b4 100644
--- a/pipeline/astro_cal.py
+++ b/pipeline/astro_cal.py
@@ -268,8 +268,6 @@ def _run_scamp( self, ds, prov, session=None ):
 
         ds.wcs = WorldCoordinates( sources=sources, provenance=prov )
         ds.wcs.wcs = wcs
-        if session is not None:
-            ds.wcs = session.merge( ds.wcs )
 
     # ----------------------------------------------------------------------
 
@@ -294,7 +292,7 @@ def run(self, *args, **kwargs):
             self.pars.do_warning_exception_hangup_injection_here()
 
             # get the provenance for this step:
-            prov = ds.get_provenance(self.pars.get_process_name(), self.pars.get_critical_pars(), session=session)
+            prov = ds.get_provenance('extraction', self.pars.get_critical_pars(), session=session)
 
             # try to find the world coordinates in memory or in the database:
             wcs = ds.get_wcs(prov, session=session)
@@ -316,7 +314,9 @@ def run(self, *args, **kwargs):
                 # update the upstream bitflag
                 sources = ds.get_sources( session=session )
                 if sources is None:
-                    raise ValueError(f'Cannot find a source list corresponding to the datastore inputs: {ds.get_inputs()}')
+                    raise ValueError(
+                        f'Cannot find a source list corresponding to the datastore inputs: {ds.get_inputs()}'
+                    )
                 if ds.wcs._upstream_bitflag is None:
                     ds.wcs._upstream_bitflag = 0
                 ds.wcs._upstream_bitflag |= sources.bitflag
diff --git a/pipeline/coaddition.py b/pipeline/coaddition.py
index 4172e3b6..4082a119 100644
--- a/pipeline/coaddition.py
+++ b/pipeline/coaddition.py
@@ -13,6 +13,7 @@
 from models.image import Image
 
 from pipeline.parameters import Parameters
+from pipeline.data_store import DataStore
 from pipeline.detection import Detector
 from pipeline.astro_cal import AstroCalibrator
 from pipeline.photo_cal import PhotCalibrator
@@ -488,25 +489,32 @@ def __init__(self, **kwargs):
         self.coadder = Coadder(**coadd_config)
 
         # source detection ("extraction" for the regular image!)
-        extraction_config = self.config.value('extraction', {})
-        extraction_config.update(self.config.value('coaddition.extraction', {}))  # override coadd specific pars
-        extraction_config.update(kwargs.get('extraction', {'measure_psf': True}))
+        extraction_config = self.config.value('extraction.sources', {})
+        extraction_config.update(self.config.value('coaddition.extraction.sources', {}))  # override coadd specific pars
+        extraction_config.update(kwargs.get('extraction', {}).get('sources', {}))
+        extraction_config.update({'measure_psf': True})
         self.pars.add_defaults_to_dict(extraction_config)
         self.extractor = Detector(**extraction_config)
 
         # astrometric fit using a first pass of sextractor and then astrometric fit to Gaia
-        astro_cal_config = self.config.value('astro_cal', {})
-        astro_cal_config.update(self.config.value('coaddition.astro_cal', {}))  # override coadd specific pars
-        astro_cal_config.update(kwargs.get('astro_cal', {}))
-        self.pars.add_defaults_to_dict(astro_cal_config)
-        self.astro_cal = AstroCalibrator(**astro_cal_config)
+        astrometor_config = self.config.value('extraction.wcs', {})
+        astrometor_config.update(self.config.value('coaddition.extraction.wcs', {}))  # override coadd specific pars
+        astrometor_config.update(kwargs.get('extraction', {}).get('wcs', {}))
+        self.pars.add_defaults_to_dict(astrometor_config)
+        self.astrometor = AstroCalibrator(**astrometor_config)
 
         # photometric calibration:
-        photo_cal_config = self.config.value('photo_cal', {})
-        photo_cal_config.update(self.config.value('coaddition.photo_cal', {}))  # override coadd specific pars
-        photo_cal_config.update(kwargs.get('photo_cal', {}))
-        self.pars.add_defaults_to_dict(photo_cal_config)
-        self.photo_cal = PhotCalibrator(**photo_cal_config)
+        photometor_config = self.config.value('extraction.zp', {})
+        photometor_config.update(self.config.value('coaddition.extraction.zp', {}))  # override coadd specific pars
+        photometor_config.update(kwargs.get('extraction', {}).get('zp', {}))
+        self.pars.add_defaults_to_dict(photometor_config)
+        self.photometor = PhotCalibrator(**photometor_config)
+
+        # make sure when calling get_critical_pars() these objects will produce the full, nested dictionary
+        siblings = {'sources': self.extractor.pars, 'wcs': self.astrometor.pars, 'zp': self.photometor.pars}
+        self.extractor.pars.add_siblings(siblings)
+        self.astrometor.pars.add_siblings(siblings)
+        self.photometor.pars.add_siblings(siblings)
 
         self.datastore = None  # use this datastore to save the coadd image and all the products
 
@@ -517,7 +525,7 @@ def parse_inputs(self, *args, **kwargs):
         """Parse the possible inputs to the run method.
 
         The possible input types are:
-        - unamed arguments that are all Image objects, to be treated as self.images
+        - unnamed arguments that are all Image objects, to be treated as self.images
         - a list of Image objects, assigned into self.images
         - two lists of Image objects, the second one is a list of aligned images matching the first list,
           such that the two lists are assigned to self.images and self.aligned_images
@@ -568,7 +576,7 @@ def parse_inputs(self, *args, **kwargs):
             raise ValueError('All unnamed arguments must be Image objects. ')
 
         if self.images is None:  # get the images from the DB
-            # TODO: this feels like it could be a useful tool, maybe need to move it Image class? Issue 188
+            # TODO: this feels like it could be a useful tool, maybe need to move it to Image class? Issue 188
             # if no images were given, parse the named parameters
             ra = kwargs.get('ra', None)
             if isinstance(ra, str):
@@ -601,7 +609,7 @@ def parse_inputs(self, *args, **kwargs):
                 provenance_ids = [prov.id]
             provenance_ids = listify(provenance_ids)
 
-            with SmartSession(session) as session:
+            with SmartSession(session) as dbsession:
                 stmt = sa.select(Image).where(
                         Image.mjd >= start_time,
                         Image.mjd <= end_time,
@@ -615,19 +623,64 @@ def parse_inputs(self, *args, **kwargs):
                     stmt = stmt.where(Image.target == target)
                 else:
                     stmt = stmt.where(Image.containing( ra, dec ))
-                self.images = session.scalars(stmt.order_by(Image.mjd.asc())).all()
+                self.images = dbsession.scalars(stmt.order_by(Image.mjd.asc())).all()
+
+        return session
 
     def run(self, *args, **kwargs):
-        self.parse_inputs(*args, **kwargs)
+        session = self.parse_inputs(*args, **kwargs)
         if self.images is None or len(self.images) == 0:
             raise ValueError('No images found matching the given parameters. ')
 
+        self.datastore = DataStore()
+        self.datastore.prov_tree = self.make_provenance_tree(session=session)
+
         # the self.aligned_images is None unless you explicitly pass in the pre-aligned images to save time
-        coadd = self.coadder.run(self.images, self.aligned_images)
+        self.datastore.image = self.coadder.run(self.images, self.aligned_images)
 
-        self.datastore = self.extractor.run(coadd)
-        self.datastore = self.astro_cal.run(self.datastore)
-        self.datastore = self.photo_cal.run(self.datastore)
+        # TODO: add the warnings/exception capturing, runtime/memory tracking (and Report making) as in top_level.py
+        self.datastore = self.extractor.run(self.datastore)
+        self.datastore = self.astrometor.run(self.datastore)
+        self.datastore = self.photometor.run(self.datastore)
 
         return self.datastore.image
 
+    def make_provenance_tree(self, session=None):
+        """Make a (short) provenance tree to use when fetching the provenances of upstreams. """
+        with SmartSession(session) as session:
+            coadd_upstreams = set()
+            code_versions = set()
+            # assumes each image given to the coaddition pipline has sources, psf, background, wcs, zp, all loaded
+            for im in self.images:
+                coadd_upstreams.add(im.provenance)
+                coadd_upstreams.add(im.sources.provenance)
+                code_versions.add(im.provenance.code_version)
+                code_versions.add(im.sources.provenance.code_version)
+
+            code_versions = list(code_versions)
+            code_versions.sort(key=lambda x: x.id)
+            code_version = code_versions[-1]  # choose the most recent ID if there are multiple code versions
+
+            pars_dict = self.coadder.pars.get_critical_pars()
+            coadd_prov = Provenance(
+                code_version=code_version,
+                process='coaddition',
+                upstreams=list(coadd_upstreams),
+                parameters=pars_dict,
+                is_testing="test_parameter" in pars_dict,  # this is a flag for testing purposes
+            )
+            coadd_prov = coadd_prov.merge_concurrent(session=session, commit=True)
+
+            # the extraction pipeline
+            pars_dict = self.extractor.pars.get_critical_pars()
+            extract_prov = Provenance(
+                code_version=code_version,
+                process='extraction',
+                upstreams=[coadd_prov],
+                parameters=pars_dict,
+                is_testing="test_parameter" in pars_dict['sources'],  # this is a flag for testing purposes
+            )
+            extract_prov = extract_prov.merge_concurrent(session=session, commit=True)
+
+        return {'coaddition': coadd_prov, 'extraction': extract_prov}
+
diff --git a/pipeline/cutting.py b/pipeline/cutting.py
index 651dfa5e..a533abbc 100644
--- a/pipeline/cutting.py
+++ b/pipeline/cutting.py
@@ -70,7 +70,7 @@ def run(self, *args, **kwargs):
             self.pars.do_warning_exception_hangup_injection_here()
 
             # get the provenance for this step:
-            prov = ds.get_provenance(self.pars.get_process_name(), self.pars.get_critical_pars(), session=session)
+            prov = ds.get_provenance('cutting', self.pars.get_critical_pars(), session=session)
 
             # try to find some measurements in memory or in the database:
             cutout_list = ds.get_cutouts(prov, session=session)
diff --git a/pipeline/data_store.py b/pipeline/data_store.py
index 5c5d227b..6f185526 100644
--- a/pipeline/data_store.py
+++ b/pipeline/data_store.py
@@ -1,8 +1,9 @@
+import warnings
 import math
 import datetime
 import sqlalchemy as sa
 
-from util.util import get_latest_provenance, parse_session
+from util.util import parse_session
 from util.logger import SCLogger
 
 from models.base import SmartSession, FileOnDiskMixin
@@ -22,9 +23,7 @@
     'exposure': [],  # no upstreams
     'preprocessing': ['exposure'],
     'extraction': ['preprocessing'],
-    'astro_cal': ['extraction'],
-    'photo_cal': ['extraction', 'astro_cal'],
-    'subtraction': ['reference', 'preprocessing', 'extraction', 'astro_cal', 'photo_cal'],
+    'subtraction': ['reference', 'preprocessing', 'extraction'],
     'detection': ['subtraction'],
     'cutting': ['detection'],
     'measuring': ['cutting'],
@@ -36,9 +35,7 @@
     'exposure': 'exposure',
     'preprocessing': 'image',
     'coaddition': 'image',
-    'extraction': ['sources', 'psf'],  # TODO: add background, maybe move wcs and zp in here too? 
-    'astro_cal': 'wcs',
-    'photo_cal': 'zp',
+    'extraction': ['sources', 'psf', 'background', 'wcs', 'zp'],
     'reference': 'reference',
     'subtraction': 'sub_image',
     'detection': 'detections',
@@ -133,6 +130,12 @@ def parse_args(self, *args, **kwargs):
             attributes. These are parsed after the args
             list and can override it!
 
+        Additional things that can get automatically parsed,
+        either by keyword or by the content of one of the args:
+            - provenances / prov_tree: a dictionary of provenances for each process.
+            - session: a sqlalchemy session object to use.
+            -
+
         Returns
         -------
         output_session: sqlalchemy.orm.session.Session or SmartSession
@@ -147,13 +150,31 @@ def parse_args(self, *args, **kwargs):
             return
 
         args, kwargs, output_session = parse_session(*args, **kwargs)
+
         self.session = output_session
 
-        # remove any provenances from the args list
-        for arg in args:
-            if isinstance(arg, Provenance):
-                self.upstream_provs.append(arg)
-        args = [arg for arg in args if not isinstance(arg, Provenance)]
+        # look for a user-given provenance tree
+        provs = [
+            arg for arg in args
+            if isinstance(arg, dict) and all([isinstance(value, Provenance) for value in arg.values()])
+        ]
+        if len(provs) > 0:
+            self.prov_tree = provs[0]
+            # also remove the provenances from the args list
+            args = [
+                arg for arg in args
+                if not isinstance(arg, dict) or not all([isinstance(value, Provenance) for value in arg.values()])
+            ]
+        found_keys = []
+        for key, value in kwargs.items():
+            if key in ['prov', 'provs', 'provenances', 'prov_tree', 'provs_tree', 'provenance_tree']:
+                if not isinstance(value, dict) or not all([isinstance(v, Provenance) for v in value.values()]):
+                    raise ValueError('Provenance tree must be a dictionary of Provenance objects.')
+                self.prov_tree = value
+                found_keys.append(key)
+
+        for key in found_keys:
+            del kwargs[key]
 
         # parse the args list
         arg_types = [type(arg) for arg in args]
@@ -195,17 +216,6 @@ def parse_args(self, *args, **kwargs):
                     raise ValueError(f'image must be an Image object, got {type(val)}')
                 self.image = val
 
-            # check for provenances
-            if key in ['prov', 'provenances', 'upstream_provs', 'upstream_provenances']:
-                new_provs = val
-                if not isinstance(new_provs, list):
-                    new_provs = [new_provs]
-
-                for prov in new_provs:
-                    if not isinstance(prov, Provenance):
-                        raise ValueError(f'Provenance must be a Provenance object, got {type(prov)}')
-                    self.upstream_provs.append(prov)
-
         if self.image is not None:
             for att in ['sources', 'psf', 'wcs', 'zp', 'detections', 'cutouts', 'measurements']:
                 if getattr(self.image, att, None) is not None:
@@ -255,7 +265,7 @@ def __init__(self, *args, **kwargs):
         self._exposure = None  # single image, entire focal plane
         self._section = None  # SensorSection
 
-        self.upstream_provs = None  # provenances to override the upstreams if no upstream objects exist
+        self.prov_tree = None  # provenance dictionary keyed on the process name
 
         # these all need to be added to the products_to_save list
         self.image = None  # single image from one sensor section
@@ -384,15 +394,13 @@ def __setattr__(self, key, value):
                     f'measurements must be a list of Measurement objects, got list with {[type(m) for m in value]}'
                 )
 
-            if key == 'upstream_provs' and not isinstance(value, list):
-                raise ValueError(f'upstream_provs must be a list of Provenance objects, got {type(value)}')
-
-            if key == 'upstream_provs' and not all([isinstance(p, Provenance) for p in value]):
-                raise ValueError(
-                    f'upstream_provs must be a list of Provenance objects, got list with {[type(p) for p in value]}'
-                )
+            if (
+                key == 'prov_tree' and not isinstance(value, dict) and
+                not all([isinstance(v, Provenance) for v in value.values()])
+            ):
+                raise ValueError(f'prov_tree must be a list of Provenance objects, got {value}')
 
-            if key == 'session' and not isinstance(value, (sa.orm.session.Session, SmartSession)):
+            if key == 'session' and not isinstance(value, sa.orm.session.Session):
                 raise ValueError(f'Session must be a SQLAlchemy session or SmartSession, got {type(value)}')
 
         super().__setattr__(key, value)
@@ -415,12 +423,16 @@ def get_inputs(self):
 
         if self.image_id is not None:
             return f'image_id={self.image_id}'
+        if self.image is not None:
+            return f'image={self.image}'
         elif self.exposure_id is not None and self.section_id is not None:
             return f'exposure_id={self.exposure_id}, section_id={self.section_id}'
+        elif self.exposure is not None and self.section_id is not None:
+            return f'exposure={self.exposure}, section_id={self.section_id}'
         else:
             raise ValueError('Could not get inputs for DataStore.')
 
-    def get_provenance(self, process, pars_dict, upstream_provs=None, session=None):
+    def get_provenance(self, process, pars_dict, session=None):
         """Get the provenance for a given process.
         Will try to find a provenance that matches the current code version
         and the parameter dictionary, and if it doesn't find it,
@@ -430,25 +442,32 @@ def get_provenance(self, process, pars_dict, upstream_provs=None, session=None):
         using the DataStore, to get the provenance for a given process,
         or to make it if it doesn't exist.
 
+        Getting upstreams:
+        Will use the prov_tree attribute of the datastore (if it exists)
+        and if not, will try to get the upstream provenances from objects
+        it has in memory already.
+        If it doesn't find an upstream in either places it would use the
+        most recently created provenance as an upstream, but this should
+        rarely happen.
+
+        Note that the output provenance can be different for the given process,
+        if there are new parameters that differ from those used to make this provenance.
+        For example: a prov_tree contains a preprocessing provenance "A",
+        and an extraction provenance "B". This function is called for
+        the "extraction" step, but with some new parameters (different than in "B").
+        The "A" provenance will be used as the upstream, but the output provenance
+        will not be "B" because of the new parameters.
+        This will not change the prov_tree or affect later calls to this function
+        for downstream provenances.
+
         Parameters
         ----------
         process: str
-            The name of the process, e.g., "preprocess", "calibration", "subtraction".
-            Use a Parameter object's get_process_name().
+            The name of the process, e.g., "preprocess", "extraction", "subtraction".
         pars_dict: dict
             A dictionary of parameters used for the process.
             These include the critical parameters for this process.
             Use a Parameter object's get_critical_pars().
-        upstream_provs: list of Provenance objects
-            A list of provenances to use as upstreams for the current
-            provenance that is requested. Any upstreams that are not
-            given will be filled using objects that already exist
-            in the data store, or by getting the most up-to-date
-            provenance from the database.
-            The upstream provenances can be given directly as
-            a function parameter, or using the DataStore constructor.
-            If given as a parameter, it will override the DataStore's
-            self.upstream_provs attribute for that call.
         session: sqlalchemy.orm.session.Session
             An optional session to use for the database query.
             If not given, will use the session stored inside the
@@ -461,9 +480,6 @@ def get_provenance(self, process, pars_dict, upstream_provs=None, session=None):
             The provenance for the given process.
 
         """
-        if upstream_provs is None:
-            upstream_provs = self.upstream_provs
-
         with SmartSession(session, self.session) as session:
             code_version = Provenance.get_code_version(session=session)
             if code_version is None:
@@ -474,34 +490,34 @@ def get_provenance(self, process, pars_dict, upstream_provs=None, session=None):
             # check if we can find the upstream provenances
             upstreams = []
             for name in UPSTREAM_STEPS[process]:
+                prov = None
                 # first try to load an upstream that was given explicitly:
-                obj_names = PROCESS_PRODUCTS[name]
-                if isinstance(obj_names, str):
-                    obj_names = [obj_names]
-                obj = getattr(self, obj_names[0], None)  # only need one object to get the provenance
-                if isinstance(obj, list):
-                    obj = obj[0]  # for cutouts or measurements just use the first one
-                if upstream_provs is not None and name in [p.process for p in upstream_provs]:
-                    prov = [p for p in upstream_provs if p.process == name][0]
-
-                # second, try to get a provenance from objects saved to the store:
-                elif obj is not None and hasattr(obj, 'provenance') and obj.provenance is not None:
-                    prov = obj.provenance
-
-                # last, try to get the latest provenance from the database:
-                else:
-                    prov = get_latest_provenance(name, session=session)
+                if self.prov_tree is not None and name in self.prov_tree:
+                    prov = self.prov_tree[name]
+
+                if prov is None:  # if that fails, see if the correct object exists in memory
+                    obj_names = PROCESS_PRODUCTS[name]
+                    if isinstance(obj_names, str):
+                        obj_names = [obj_names]
+                    obj = getattr(self, obj_names[0], None)  # only need one object to get the provenance
+                    if isinstance(obj, list):
+                        obj = obj[0]  # for cutouts or measurements just use the first one
 
-                # can't find any provenance upstream, therefore
-                # there can't be any provenance for this process
-                if prov is None:
-                    return None
+                    if obj is not None and hasattr(obj, 'provenance') and obj.provenance is not None:
+                        prov = obj.provenance
 
-                upstreams.append(prov)
+                if prov is not None:  # if we don't find one of the upstreams, it will raise an exception
+                    upstreams.append(prov)
 
             if len(upstreams) != len(UPSTREAM_STEPS[process]):
                 raise ValueError(f'Could not find all upstream provenances for process {process}.')
 
+            for u in upstreams:  # check if "reference" is in the list, if so, replace it with its upstreams
+                if u.process == 'reference':
+                    upstreams.remove(u)
+                    for up in u.upstreams:
+                        upstreams.append(up)
+
             # we have a code version object and upstreams, we can make a provenance
             prov = Provenance(
                 process=process,
@@ -510,46 +526,31 @@ def get_provenance(self, process, pars_dict, upstream_provs=None, session=None):
                 upstreams=upstreams,
                 is_testing="test_parameter" in pars_dict,  # this is a flag for testing purposes
             )
-            db_prov = session.scalars(sa.select(Provenance).where(Provenance.id == prov.id)).first()
-            if db_prov is not None:  # only merge if this provenance already exists
-                prov = session.merge(prov)
+            prov = prov.merge_concurrent(session=session, commit=True)
 
         return prov
 
     def _get_provenance_for_an_upstream(self, process, session=None):
-        """
-        Get the provenance for a given process, without knowing
-        the parameters or code version.
-        This simply looks for a matching provenance in the upstream_provs
-        attribute, and if it is not there, it will call the latest provenance
-        (for that process) from the database.
-        This is used to get the provenance of upstream objects,
-        only when those objects are not found in the store.
-        Example: when looking for the upstream provenance of a
-        photo_cal process, the upstream process is preprocess,
-        so this function will look for the preprocess provenance.
-        If the ZP object is from the DB then there must be provenance
-        objects for the Image that was used to create it.
-        If the ZP was just created, the Image should also be
-        in memory even if the provenance is not on DB yet,
-        in which case this function should not be called.
-
-        This will raise if no provenance can be found.
-        """
-        session = self.session if session is None else session
+        """Get the provenance for a given process, without parameters or code version.
+        This is used to get the provenance of upstream objects.
+        Looks for a matching provenance in the prov_tree attribute.
 
-        # see if it is in the upstream_provs
-        if self.upstream_provs is not None:
-            prov_list = [p for p in self.upstream_provs if p.process == process]
-            provenance = prov_list[0] if len(prov_list) > 0 else None
-        else:
-            provenance = None
+        Example:
+        When making a SourceList in the extraction phase, we will want to know the provenance
+        of the Image object (from the preprocessing phase).
+        To get it, we'll call this function with process="preprocessing".
+        If prov_tree is not None, it will provide the provenance for the preprocessing phase.
 
-        # try getting the latest from the database
-        if provenance is None:  # check latest provenance
-            provenance = get_latest_provenance(process, session=session)
+        Will raise if no provenance can be found.
+        """
+        # see if it is in the prov_tree
+        if self.prov_tree is not None:
+            if process in self.prov_tree:
+                return self.prov_tree[process]
+            else:
+                raise ValueError(f'No provenance found for process "{process}" in prov_tree!')
 
-        return provenance
+        return None  # if not found in prov_tree, just return None
 
     def get_raw_exposure(self, session=None):
         """
@@ -573,26 +574,26 @@ def get_image(self, provenance=None, session=None):
         provenances or the local parameters.
         This is the only way to ask for a coadd image.
         If an image with such an id is not found,
-        in memory or in the database, will raise
-        an ValueError.
+        in memory or in the database, will raise a ValueError.
         If exposure_id and section_id are given, will
         load an image that is consistent with
         that exposure and section ids, and also with
         the code version and critical parameters
         (using a matching of provenances).
-        In this case we will only load a regular
-        image, not a coadded image.
+        In this case we will only load a regular image, not a coadd.
         If no matching image is found, will return None.
 
+        Note that this also updates self.image with the found image (or None).
+
         Parameters
         ----------
         provenance: Provenance object
             The provenance to use for the image.
             This provenance should be consistent with
             the current code version and critical parameters.
-            If none is given, will use the latest provenance
-            for the "preprocessing" process.
-        session: sqlalchemy.orm.session.Session or SmartSession
+            If none is given, will use the prov_tree and if that is None,
+            will use the latest provenance for the "preprocessing" process.
+        session: sqlalchemy.orm.session.Session
             An optional session to use for the database query.
             If not given, will use the session stored inside the
             DataStore object; if there is none, will open a new session
@@ -606,9 +607,15 @@ def get_image(self, provenance=None, session=None):
         """
         session = self.session if session is None else session
 
-        process_name = 'preprocessing'
-        if self.image_id is not None:
-            # we were explicitly asked for a specific image id:
+        if (
+                (self.exposure is None or self.section is None) and
+                (self.exposure_id is None or self.section_id is None) and
+                self.image is None and self.image_id is None
+        ):
+            raise ValueError('Cannot get image without one of (exposure_id, section_id), '
+                             '(exposure, section), image, or image_id!')
+
+        if self.image_id is not None:  # we were explicitly asked for a specific image id:
             if isinstance(self.image, Image) and self.image.id == self.image_id:
                 pass  # return self.image at the end of function...
             else:  # not found in local memory, get from DB
@@ -619,63 +626,36 @@ def get_image(self, provenance=None, session=None):
             if self.image is None:
                 raise ValueError(f'Cannot find image with id {self.image_id}!')
 
-        elif self.image is not None:
-            # If an image already exists and image_id is none, we may be
-            # working with a datastore that hasn't been committed to the
-            # database; do a quick check for mismatches.
-            # (If all the ids are None, it'll match even if the actual
-            # objects are wrong, but, oh well.)
-            if (self.exposure_id is not None) and (self.section_id is not None):
-                if ( (self.image.exposure_id is not None and self.image.exposure_id != self.exposure_id) or
-                     (self.image.section_id != self.section_id) ):
-                    raise ValueError( "Image exposure/section id doesn't match what's expected!" )
-            elif self.exposure is not None and self.section is not None:
-                if ( (self.image.exposure_id is not None and self.image.exposure_id != self.exposure.id) or
-                     (self.image.section_id != self.section.identifier) ):
-                    raise ValueError( "Image exposure/section id doesn't match what's expected!" )
-            # If we get here, self.image is presumed to be good
+        else:  # try to get the image based on exposure_id and section_id
+            process = 'preprocessing'
+            if self.image is not None and self.image.provenance is not None:
+                process = self.image.provenance.process  # this will be "coaddition" sometimes!
+            if provenance is None:  # try to get the provenance from the prov_tree
+                provenance = self._get_provenance_for_an_upstream(process, session=session)
 
-        elif self.exposure_id is not None and self.section_id is not None:
-            # If we don't know the image yet
-            # check if self.image is the correct image:
-            if (
-                isinstance(self.image, Image) and self.image.exposure_id == self.exposure_id
-                    and self.image.section_id == str(self.section_id)
-            ):
-                # make sure the image has the correct provenance
-                if self.image is not None:
-                    if self.image.provenance is None:
-                        raise ValueError('Image has no provenance!')
-                    if provenance is not None and provenance.id != self.image.provenance.id:
-                        self.image = None
-                        self.sources = None
-                        self.psf = None
-                        self.wcs = None
-                        self.zp = None
-
-                if provenance is None and self.image is not None:
-                    if self.upstream_provs is not None:
-                        provenances = [p for p in self.upstream_provs if p.process == process_name]
-                    else:
-                        provenances = []
-
-                    if len(provenances) > 1:
-                        raise ValueError(f'More than one "{process_name}" provenance found!')
-                    if len(provenances) == 1:
-                        # a mismatch of provenance and cached image:
-                        if self.image.provenance.id != provenances[0].id:
-                            self.image = None  # this must be an old image, get a new one
-                            self.sources = None
-                            self.psf = None
-                            self.wcs = None
-                            self.zp = None
+            if self.image is not None:
+                # If an image already exists and image_id is none, we may be
+                # working with a datastore that hasn't been committed to the
+                # database; do a quick check for mismatches.
+                # (If all the ids are None, it'll match even if the actual
+                # objects are wrong, but, oh well.)
+                if (
+                    self.exposure_id is not None and self.section_id is not None and
+                    (self.exposure_id != self.image.exposure_id or self.section_id != self.image.section_id)
+                ):
+                    self.image = None
+                if self.exposure is not None and self.image.exposure_id != self.exposure.id:
+                    self.image = None
+                if self.section is not None and str(self.image.section_id) != self.section.identifier:
+                    self.image = None
+                if self.image is not None and provenance is not None and self.image.provenance.id != provenance.id:
+                    self.image = None
+
+                # If we get here, self.image is presumed to be good
 
             if self.image is None:  # load from DB
                 # this happens when the image is required as an upstream for another process (but isn't in memory)
-                if provenance is None:  # check if in upstream_provs/database
-                    provenance = self._get_provenance_for_an_upstream(process_name, session=session)
-
-                if provenance is not None:  # if we can't find a provenance, then we don't need to load from DB
+                if provenance is not None:
                     with SmartSession(session) as session:
                         self.image = session.scalars(
                             sa.select(Image).where(
@@ -685,17 +665,7 @@ def get_image(self, provenance=None, session=None):
                             )
                         ).first()
 
-        elif self.exposure is not None and self.section is not None:
-            # If we don't have exposure and section ids, but we do have an exposure
-            # and a section, we're probably working with a non-committed datastore.
-            # So, extract the image from the exposure.
-            self.image = Image.from_exposure( self.exposure, self.section.identifier )
-
-        else:
-            raise ValueError('Cannot get image without one of (exposure_id, section_id), '
-                             '(exposure, section), image, or image_id!')
-
-        return self.image  # could return none if no image was found
+        return self.image  # can return none if no image was found
 
     def append_image_products(self, image):
         """Append the image products to the image and sources objects.
@@ -712,19 +682,21 @@ def append_image_products(self, image):
                     setattr(image.sources, att, getattr(self, att))
 
     def get_sources(self, provenance=None, session=None):
-        """
-        Get a SourceList from the original image,
-        either from memory or from database.
+        """Get the source list, either from memory or from database.
 
         Parameters
         ----------
         provenance: Provenance object
-            The provenance to use for the source list.
+            The provenance to use to get the source list.
             This provenance should be consistent with
             the current code version and critical parameters.
-            If none is given, will use the latest provenance
+            If none is given, uses the appropriate provenance
+            from the prov_tree dictionary.
+            If prov_tree is None, will use the latest provenance
             for the "extraction" process.
-        session: sqlalchemy.orm.session.Session or SmartSession
+            Usually the provenance is not given when sources are loaded
+            in order to be used as an upstream of the current process.
+        session: sqlalchemy.orm.session.Session
             An optional session to use for the database query.
             If not given, will use the session stored inside the
             DataStore object; if there is none, will open a new session
@@ -738,6 +710,9 @@ def get_sources(self, provenance=None, session=None):
 
         """
         process_name = 'extraction'
+        if provenance is None:  # try to get the provenance from the prov_tree
+            provenance = self._get_provenance_for_an_upstream(process_name, session)
+
         # if sources exists in memory, check the provenance is ok
         if self.sources is not None:
             # make sure the sources object has the correct provenance
@@ -745,34 +720,14 @@ def get_sources(self, provenance=None, session=None):
                 raise ValueError('SourceList has no provenance!')
             if provenance is not None and provenance.id != self.sources.provenance.id:
                 self.sources = None
-                self.wcs = None
-                self.zp = None
 
         # TODO: do we need to test the SourceList Provenance has upstreams consistent with self.image.provenance?
 
-        if provenance is None and self.sources is not None:
-            if self.upstream_provs is not None:
-                provenances = [p for p in self.upstream_provs if p.process == process_name]
-            else:
-                provenances = []
-            if len(provenances) > 1:
-                raise ValueError(f'More than one {process_name} provenance found!')
-            if len(provenances) == 1:
-                # a mismatch of given provenance and self.sources' provenance:
-                if self.sources.provenance.id != provenances[0].id:
-                    self.sources = None  # this must be an old sources object, get a new one
-                    self.wcs = None
-                    self.zp = None
-
         # not in memory, look for it on the DB
         if self.sources is None:
-            # this happens when the source list is required as an upstream for another process (but isn't in memory)
-            if provenance is None:  # check if in upstream_provs/database
-                provenance = self._get_provenance_for_an_upstream(process_name, session )
-
-            if provenance is not None:  # if we can't find a provenance, then we don't need to load from DB
-                with SmartSession(session, self.session) as session:
-                    image = self.get_image(session=session)
+            with SmartSession(session, self.session) as session:
+                image = self.get_image(session=session)
+                if image is not None:
                     self.sources = session.scalars(
                         sa.select(SourceList).where(
                             SourceList.image_id == image.id,
@@ -784,82 +739,74 @@ def get_sources(self, provenance=None, session=None):
         return self.sources
 
     def get_psf(self, provenance=None, session=None):
-        """Get a PSF for the image, either from memory or the database.
+        """Get a PSF, either from memory or from the database.
 
         Parameters
         ----------
         provenance: Provenance object
-          The provenance to use for the PSF.  This provenance should be
-          consistent with the current code version and critical
-          parameters.  If None, will use the latest provenance for the
-          "extraction" process.
-        session: sqlalchemy.orm.session.Sesssion
-          An optional database session.  If not given, will use the
-          session stored in the DataStore object, or open and close a
-          new session if there isn't one.
-
-        Retruns
+            The provenance to use for the PSF.
+            This provenance should be consistent with
+            the current code version and critical parameters.
+            If none is given, uses the appropriate provenance
+            from the prov_tree dictionary.
+            If prov_tree is None, will use the latest provenance
+            for the "extraction" process.
+            Usually the provenance is not given when the psf is loaded
+            in order to be used as an upstream of the current process.
+        session: sqlalchemy.orm.session.Session
+            An optional session to use for the database query.
+            If not given, will use the session stored inside the
+            DataStore object; if there is none, will open a new session
+            and close it at the end of the function.
+
+        Returns
         -------
-        psf: PSF Object
+        psf: PSF object
+            The point spread function object for this image,
+            or None if no matching PSF is found.
 
         """
         process_name = 'extraction'
-        # if psf exists in memory already, check that the provenance is ok
+        if provenance is None:  # try to get the provenance from the prov_tree
+            provenance = self._get_provenance_for_an_upstream(process_name, session)
+
+        # if psf exists in memory, check the provenance is ok
         if self.psf is not None:
+            # make sure the psf object has the correct provenance
             if self.psf.provenance is None:
-                raise ValueError( 'PSF has no provenance!' )
+                raise ValueError('PSF has no provenance!')
             if provenance is not None and provenance.id != self.psf.provenance.id:
                 self.psf = None
-                self.wcs = None
-                self.zp = None
 
-        if provenance is None and self.psf is not None:
-            if self.upstream_provs is not None:
-                provenances = [ p for p in self.upstream_provs if p.process == process_name ]
-            else:
-                provenances = []
-            if len(provenances) > 1:
-                raise ValueError( f"More than one {process_name} provenances found!" )
-            if len(provenances) == 1:
-                # Check for a mismatch of given provenance and self.psf's provenance
-                if self.psf.provenance.id != provenances[0].id:
-                    self.psf = None
-                    self.wcs = None
-                    self.zp = None
-
-        # Didn't have the right psf in memory, look for it in the DB
+        # TODO: do we need to test the PSF Provenance has upstreams consistent with self.image.provenance?
+
+        # not in memory, look for it on the DB
         if self.psf is None:
-            # This happens when the psf is required as an upstream for another process (but isn't in memory)
-            if provenance is None:
-                provenance = self._get_provenance_for_an_upstream( process_name, session )
-
-            # If we can't find a provenance, then we don't need to load from the DB
-            if provenance is not None:
-                with SmartSession(session, self.session) as session:
-                    image = self.get_image( session=session )
+            with SmartSession(session, self.session) as session:
+                image = self.get_image(session=session)
+                if image is not None:
                     self.psf = session.scalars(
-                        sa.select( PSF ).where(
-                            PSF.image_id == image.id,
-                            PSF.provenance.has( id=provenance.id )
-                        )
+                        sa.select(PSF).where(PSF.image_id == image.id, PSF.provenance.has(id=provenance.id))
                     ).first()
 
         return self.psf
 
     def get_wcs(self, provenance=None, session=None):
-        """
-        Get an astrometric solution (in the form of a WorldCoordinates),
-        either from memory or from database.
+        """Get an astrometric solution in the form of a WorldCoordinates object, from memory or from the database.
 
         Parameters
         ----------
         provenance: Provenance object
-            The provenance to use for the wcs.
+            The provenance to use for the WCS.
             This provenance should be consistent with
             the current code version and critical parameters.
-            If none is given, will use the latest provenance
-            for the "astro_cal" process.
-        session: sqlalchemy.orm.session.Session or SmartSession
+            If none is given, uses the appropriate provenance
+            from the prov_tree dictionary.
+            If prov_tree is None, will use the latest provenance
+            for the "extraction" process.
+            Usually the provenance is not given when the wcs is loaded
+            in order to be used as an upstream of the current process.
+        session: sqlalchemy.orm.session.Session
             An optional session to use for the database query.
             If not given, will use the session stored inside the
             DataStore object; if there is none, will open a new session
@@ -868,61 +815,53 @@ def get_wcs(self, provenance=None, session=None):
         Returns
         -------
         wcs: WorldCoordinates object
-            The WCS object, or None if no matching WCS is found.
+            The world coordinates object for this image,
+            or None if no matching WCS is found.
 
         """
-        process_name = 'astro_cal'
-        # make sure the wcs has the correct provenance
+        process_name = 'extraction'
+        if provenance is None:  # try to get the provenance from the prov_tree
+            provenance = self._get_provenance_for_an_upstream(process_name, session)
+
+        # if psf exists in memory, check the provenance is ok
         if self.wcs is not None:
+            # make sure the psf object has the correct provenance
             if self.wcs.provenance is None:
                 raise ValueError('WorldCoordinates has no provenance!')
             if provenance is not None and provenance.id != self.wcs.provenance.id:
                 self.wcs = None
 
-        if provenance is None and self.wcs is not None:
-            if self.upstream_provs is not None:
-                provenances = [p for p in self.upstream_provs if p.process == process_name]
-            else:
-                provenances = []
-            if len(provenances) > 1:
-                raise ValueError(f'More than one "{process_name}" provenance found!')
-            if len(provenances) == 1:
-                # a mismatch of provenance and cached wcs:
-                if self.wcs.provenance.id != provenances[0].id:
-                    self.wcs = None  # this must be an old wcs object, get a new one
+        # TODO: do we need to test the WCS Provenance has upstreams consistent with self.sources.provenance?
 
         # not in memory, look for it on the DB
         if self.wcs is None:
             with SmartSession(session, self.session) as session:
-                # this happens when the wcs is required as an upstream for another process (but isn't in memory)
-                if provenance is None:  # check if in upstream_provs/database
-                    provenance = self._get_provenance_for_an_upstream(process_name, session=session)
-
-                if provenance is not None:  # if None, it means we can't find it on the DB
-                    sources = self.get_sources(session=session)
+                sources = self.get_sources(session=session)
+                if sources is not None and sources.id is not None:
                     self.wcs = session.scalars(
                         sa.select(WorldCoordinates).where(
-                            WorldCoordinates.sources_id == sources.id,
-                            WorldCoordinates.provenance.has(id=provenance.id),
+                            WorldCoordinates.sources_id == sources.id, WorldCoordinates.provenance.has(id=provenance.id)
                         )
                     ).first()
 
         return self.wcs
 
     def get_zp(self, provenance=None, session=None):
-        """
-        Get a photometric calibration (in the form of a ZeroPoint object),
-        either from memory or from database.
+        """Get a photometric solution in the form of a ZeroPoint object, from memory or from the database.
 
         Parameters
         ----------
         provenance: Provenance object
-            The provenance to use for the wcs.
+            The provenance to use for the ZP.
             This provenance should be consistent with
             the current code version and critical parameters.
-            If none is given, will use the latest provenance
-            for the "photo_cal" process.
-        session: sqlalchemy.orm.session.Session or SmartSession
+            If none is given, uses the appropriate provenance
+            from the prov_tree dictionary.
+            If prov_tree is None, will use the latest provenance
+            for the "extraction" process.
+            Usually the provenance is not given when the zp is loaded
+            in order to be used as an upstream of the current process.
+        session: sqlalchemy.orm.session.Session
             An optional session to use for the database query.
             If not given, will use the session stored inside the
             DataStore object; if there is none, will open a new session
@@ -930,43 +869,33 @@ def get_zp(self, provenance=None, session=None):
 
         Returns
         -------
-        wcs: ZeroPoint object
-            The photometric calibration object, or None if no matching ZP is found.
+        zp: ZeroPoint object
+            The zero point object for this image,
+            or None if no matching ZP is found.
+
         """
-        process_name = 'photo_cal'
-        # make sure the zp has the correct provenance
+        process_name = 'extraction'
+        if provenance is None:  # try to get the provenance from the prov_tree
+            provenance = self._get_provenance_for_an_upstream(process_name, session)
+
+        # if psf exists in memory, check the provenance is ok
         if self.zp is not None:
+            # make sure the psf object has the correct provenance
             if self.zp.provenance is None:
                 raise ValueError('ZeroPoint has no provenance!')
             if provenance is not None and provenance.id != self.zp.provenance.id:
                 self.zp = None
 
-        if provenance is None and self.zp is not None:
-            if self.upstream_provs is not None:
-                provenances = [p for p in self.upstream_provs if p.process == process_name]
-            else:
-                provenances = []
-            if len(provenances) > 1:
-                raise ValueError(f'More than one "{process_name}" provenance found!')
-            if len(provenances) == 1:
-                # a mismatch of provenance and cached zp:
-                if self.zp.provenance.id != provenances[0].id:
-                    self.zp = None  # this must be an old zp, get a new one
+        # TODO: do we need to test the ZP Provenance has upstreams consistent with self.sources.provenance?
 
         # not in memory, look for it on the DB
         if self.zp is None:
             with SmartSession(session, self.session) as session:
                 sources = self.get_sources(session=session)
-                # TODO: do we also need the astrometric solution (to query for the ZP)?
-                # this happens when the wcs is required as an upstream for another process (but isn't in memory)
-                if provenance is None:  # check if in upstream_provs/database
-                    provenance = self._get_provenance_for_an_upstream(process_name, session=session)
-
-                if provenance is not None:  # if None, it means we can't find it on the DB
+                if sources is not None and sources.id is not None:
                     self.zp = session.scalars(
                         sa.select(ZeroPoint).where(
-                            ZeroPoint.sources_id == sources.id,
-                            ZeroPoint.provenance.has(id=provenance.id),
+                            ZeroPoint.sources_id == sources.id, ZeroPoint.provenance.has(id=provenance.id)
                         )
                     ).first()
 
@@ -1085,7 +1014,6 @@ def get_reference(self, minovfrac=0.85, must_match_instrument=True, must_match_f
         that matches the other criteria.  Be careful with this.
 
         """
-
         with SmartSession(session, self.session) as session:
             image = self.get_image(session=session)
 
@@ -1146,8 +1074,7 @@ def get_reference(self, minovfrac=0.85, must_match_instrument=True, must_match_f
         return self.reference
 
     def get_subtraction(self, provenance=None, session=None):
-        """
-        Get a subtraction Image, either from memory or from database.
+        """Get a subtraction Image, either from memory or from database.
 
         Parameters
         ----------
@@ -1157,7 +1084,9 @@ def get_subtraction(self, provenance=None, session=None):
             the current code version and critical parameters.
             If none is given, will use the latest provenance
             for the "subtraction" process.
-        session: sqlalchemy.orm.session.Session or SmartSession
+            Usually the provenance is not given when the subtraction is loaded
+            in order to be used as an upstream of the current process.
+        session: sqlalchemy.orm.session.Session
             An optional session to use for the database query.
             If not given, will use the session stored inside the
             DataStore object; if there is none, will open a new session
@@ -1172,23 +1101,18 @@ def get_subtraction(self, provenance=None, session=None):
         """
         process_name = 'subtraction'
         # make sure the subtraction has the correct provenance
+        if provenance is None:  # try to get the provenance from the prov_tree
+            provenance = self._get_provenance_for_an_upstream(process_name, session)
+
+        # if subtraction exists in memory, check the provenance is ok
         if self.sub_image is not None:
+            # make sure the sub_image object has the correct provenance
             if self.sub_image.provenance is None:
-                raise ValueError('Subtraction image has no provenance!')
+                raise ValueError('Subtraction Image has no provenance!')
             if provenance is not None and provenance.id != self.sub_image.provenance.id:
                 self.sub_image = None
 
-        if provenance is None and self.sub_image is not None:
-            if self.upstream_provs is not None:
-                provenances = [p for p in self.upstream_provs if p.process == process_name]
-            else:
-                provenances = []
-            if len(provenances) > 1:
-                raise ValueError(f'More than one "{process_name}" provenance found!')
-            if len(provenances) > 0:
-                # a mismatch of provenance and cached subtraction image:
-                if self.sub_image.provenance.id != provenances[0].id:
-                    self.sub_image = None  # this must be an old subtraction image, need to get a new one
+        # TODO: do we need to test the subtraction Provenance has upstreams consistent with upstream provenances?
 
         # not in memory, look for it on the DB
         if self.sub_image is None:
@@ -1196,27 +1120,22 @@ def get_subtraction(self, provenance=None, session=None):
                 image = self.get_image(session=session)
                 ref = self.get_reference(session=session)
 
-                # this happens when the subtraction is required as an upstream for another process (but isn't in memory)
-                if provenance is None:  # check if in upstream_provs/database
-                    provenance = self._get_provenance_for_an_upstream(process_name, session=session)
-
-                if provenance is not None:  # if None, it means we can't find it on the DB
-                    aliased_table = sa.orm.aliased(image_upstreams_association_table)
-                    self.sub_image = session.scalars(
-                        sa.select(Image).join(
-                            image_upstreams_association_table,
-                            sa.and_(
-                                image_upstreams_association_table.c.upstream_id == ref.image_id,
-                                image_upstreams_association_table.c.downstream_id == Image.id,
-                            )
-                        ).join(
-                            aliased_table,
-                            sa.and_(
-                                aliased_table.c.upstream_id == image.id,
-                                aliased_table.c.downstream_id == Image.id,
-                            )
-                        ).where(Image.provenance.has(id=provenance.id))
-                    ).first()
+                aliased_table = sa.orm.aliased(image_upstreams_association_table)
+                self.sub_image = session.scalars(
+                    sa.select(Image).join(
+                        image_upstreams_association_table,
+                        sa.and_(
+                            image_upstreams_association_table.c.upstream_id == ref.image_id,
+                            image_upstreams_association_table.c.downstream_id == Image.id,
+                        )
+                    ).join(
+                        aliased_table,
+                        sa.and_(
+                            aliased_table.c.upstream_id == image.id,
+                            aliased_table.c.downstream_id == Image.id,
+                        )
+                    ).where(Image.provenance.has(id=provenance.id))
+                ).first()
 
         if self.sub_image is not None:
             self.sub_image.load_upstream_products()
@@ -1225,9 +1144,7 @@ def get_subtraction(self, provenance=None, session=None):
         return self.sub_image
 
     def get_detections(self, provenance=None, session=None):
-        """
-        Get a SourceList for sources from the subtraction image,
-        either from memory or from database.
+        """Get a SourceList for sources from the subtraction image, from memory or from database.
 
         Parameters
         ----------
@@ -1237,7 +1154,9 @@ def get_detections(self, provenance=None, session=None):
             the current code version and critical parameters.
             If none is given, will use the latest provenance
             for the "detection" process.
-        session: sqlalchemy.orm.session.Session or SmartSession
+            Usually the provenance is not given when the subtraction is loaded
+            in order to be used as an upstream of the current process.
+        session: sqlalchemy.orm.session.Session
             An optional session to use for the database query.
             If not given, will use the session stored inside the
             DataStore object; if there is none, will open a new session
@@ -1251,48 +1170,33 @@ def get_detections(self, provenance=None, session=None):
 
         """
         process_name = 'detection'
+        if provenance is None:  # try to get the provenance from the prov_tree
+            provenance = self._get_provenance_for_an_upstream(process_name, session)
+
         # not in memory, look for it on the DB
         if self.detections is not None:
-            # make sure the wcs has the correct provenance
+            # make sure the detections have the correct provenance
             if self.detections.provenance is None:
                 raise ValueError('SourceList has no provenance!')
             if provenance is not None and provenance.id != self.detections.provenance.id:
                 self.detections = None
 
-        if provenance is None and self.detections is not None:
-            if self.upstream_provs is not None:
-                provenances = [p for p in self.upstream_provs if p.process == process_name]
-            else:
-                provenances = []
-            if len(provenances) > 1:
-                raise ValueError(f'More than one "{process_name}" provenance found!')
-            if len(provenances) == 1:
-                # a mismatch of provenance and cached detections:
-                if self.detections.provenance.id != provenances[0].id:
-                    self.detections = None  # this must be an old detections object, need to get a new one
-
         if self.detections is None:
             with SmartSession(session, self.session) as session:
                 sub_image = self.get_subtraction(session=session)
 
-                # this happens when the wcs is required as an upstream for another process (but isn't in memory)
-                if provenance is None:  # check if in upstream_provs/database
-                    provenance = self._get_provenance_for_an_upstream(process_name, session=session)
-
-                if provenance is not None:  # if None, it means we can't find it on the DB
-                    self.detections = session.scalars(
-                        sa.select(SourceList).where(
-                            SourceList.image_id == sub_image.id,
-                            SourceList.is_sub.is_(True),
-                            SourceList.provenance.has(id=provenance.id),
-                        )
-                    ).first()
+                self.detections = session.scalars(
+                    sa.select(SourceList).where(
+                        SourceList.image_id == sub_image.id,
+                        SourceList.is_sub.is_(True),
+                        SourceList.provenance.has(id=provenance.id),
+                    )
+                ).first()
 
         return self.detections
 
     def get_cutouts(self, provenance=None, session=None):
-        """
-        Get a list of Cutouts, either from memory or from database.
+        """Get a list of Cutouts, either from memory or from database.
 
         Parameters
         ----------
@@ -1302,6 +1206,8 @@ def get_cutouts(self, provenance=None, session=None):
             the current code version and critical parameters.
             If none is given, will use the latest provenance
             for the "cutting" process.
+            Usually the provenance is not given when the subtraction is loaded
+            in order to be used as an upstream of the current process.
         session: sqlalchemy.orm.session.Session
             An optional session to use for the database query.
             If not given, will use the session stored inside the
@@ -1315,24 +1221,20 @@ def get_cutouts(self, provenance=None, session=None):
 
         """
         process_name = 'cutting'
-        # make sure the cutouts have the correct provenance
+        if provenance is None:  # try to get the provenance from the prov_tree
+            provenance = self._get_provenance_for_an_upstream(process_name, session)
+
+        # not in memory, look for it on the DB
         if self.cutouts is not None:
-            if any([c.provenance is None for c in self.cutouts]):
-                raise ValueError('One of the Cutouts has no provenance!')
-            if provenance is not None and any([c.provenance.id != provenance.id for c in self.cutouts]):
-                self.cutouts = None
-
-        if provenance is None and self.cutouts is not None:
-            if self.upstream_provs is not None:
-                provenances = [p for p in self.upstream_provs if p.process == process_name]
-            else:
-                provenances = []
-            if len(provenances) > 1:
-                raise ValueError(f'More than one "{process_name}" provenance found!')
-            if len(provenances) == 1:
-                # a mismatch of provenance and cached cutouts:
-                if any([c.provenance.id != provenances[0].id for c in self.cutouts]):
-                    self.cutouts = None  # this must be an old cutouts list, need to get a new one
+            if len(self.cutouts) == 0:
+                self.cutouts = None  # TODO: what about images that actually don't have any detections?
+
+            # make sure the cutouts have the correct provenance
+            if self.cutouts is not None:
+                if self.cutouts[0].provenance is None:
+                    raise ValueError('Cutouts have no provenance!')
+                if provenance is not None and provenance.id != self.cutouts[0].provenance.id:
+                    self.detections = None
 
         # not in memory, look for it on the DB
         if self.cutouts is None:
@@ -1348,23 +1250,17 @@ def get_cutouts(self, provenance=None, session=None):
                 if sub_image.sources is None:
                     return None
 
-                # this happens when the cutouts are required as an upstream for another process (but aren't in memory)
-                if provenance is None:
-                    provenance = self._get_provenance_for_an_upstream(process_name, session=session)
-
-                if provenance is not None:  # if None, it means we can't find it on the DB
-                    self.cutouts = session.scalars(
-                        sa.select(Cutouts).where(
-                            Cutouts.sources_id == sub_image.sources.id,
-                            Cutouts.provenance.has(id=provenance.id),
-                        )
-                    ).all()
+                self.cutouts = session.scalars(
+                    sa.select(Cutouts).where(
+                        Cutouts.sources_id == sub_image.sources.id,
+                        Cutouts.provenance.has(id=provenance.id),
+                    )
+                ).all()
 
         return self.cutouts
 
     def get_measurements(self, provenance=None, session=None):
-        """
-        Get a list of Measurements, either from memory or from database.
+        """Get a list of Measurements, either from memory or from database.
 
         Parameters
         ----------
@@ -1374,6 +1270,8 @@ def get_measurements(self, provenance=None, session=None):
             the current code version and critical parameters.
             If none is given, will use the latest provenance
             for the "measurement" process.
+            Usually the provenance is not given when the subtraction is loaded
+            in order to be used as an upstream of the current process.
         session: sqlalchemy.orm.session.Session
             An optional session to use for the database query.
             If not given, will use the session stored inside the
@@ -1387,6 +1285,9 @@ def get_measurements(self, provenance=None, session=None):
 
         """
         process_name = 'measurement'
+        if provenance is None:  # try to get the provenance from the prov_tree
+            provenance = self._get_provenance_for_an_upstream(process_name, session)
+
         # make sure the measurements have the correct provenance
         if self.measurements is not None:
             if any([m.provenance is None for m in self.measurements]):
@@ -1394,35 +1295,18 @@ def get_measurements(self, provenance=None, session=None):
             if provenance is not None and any([m.provenance.id != provenance.id for m in self.measurements]):
                 self.measurements = None
 
-        if provenance is None and self.measurements is not None:
-            if self.upstream_provs is not None:
-                provenances = [p for p in self.upstream_provs if p.process == process_name]
-            else:
-                provenances = []
-            if len(provenances) > 1:
-                raise ValueError(f'More than one "{process_name}" provenance found!')
-            if len(provenances) == 1:
-                # a mismatch of provenance and cached image:
-                if any([m.provenance.id != provenances[0].id for m in self.measurements]):
-                    self.measurements = None
-
         # not in memory, look for it on the DB
         if self.measurements is None:
             with SmartSession(session, self.session) as session:
                 cutouts = self.get_cutouts(session=session)
                 cutout_ids = [c.id for c in cutouts]
 
-                # this happens when the measurements are required as an upstream (but aren't in memory)
-                if provenance is None:
-                    provenance = self._get_provenance_for_an_upstream(process_name, session=session)
-
-                if provenance is not None:  # if None, it means we can't find it on the DB
-                    self.measurements = session.scalars(
-                        sa.select(Measurements).where(
-                            Measurements.cutouts_id.in_(cutout_ids),
-                            Measurements.provenance.has(id=provenance.id),
-                        )
-                    ).all()
+                self.measurements = session.scalars(
+                    sa.select(Measurements).where(
+                        Measurements.cutouts_id.in_(cutout_ids),
+                        Measurements.provenance.has(id=provenance.id),
+                    )
+                ).all()
 
         return self.measurements
 
@@ -1610,9 +1494,11 @@ def save_and_commit(self, exists_ok=False, overwrite=True, no_archive=False,
             self.products_committed = 'image, sources, psf, wcs, zp'
 
             if self.sub_image is not None:
+                if self.reference is not None:
+                    self.reference = self.reference.merge_all(session)
                 self.sub_image.new_image = self.image  # update with the now-merged image
                 self.sub_image = self.sub_image.merge_all(session)  # merges the upstream_images and downstream products
-                self.sub_image.ref_image.id = self.sub_image.ref_image_id  # just to make sure the ref has an ID for merging
+                self.sub_image.ref_image.id = self.sub_image.ref_image_id
                 self.detections = self.sub_image.sources
 
             session.commit()
@@ -1666,7 +1552,11 @@ def delete_everything(self, session=None, commit=True):
         if session is None and not commit:
             raise ValueError('If session is None, commit must be True')
 
-        with SmartSession( session, self.session ) as session:
+        with SmartSession( session, self.session ) as session, warnings.catch_warnings():
+            warnings.filterwarnings(
+                action='ignore',
+                message=r'.*DELETE statement on table .* expected to delete \d* row\(s\).*',
+            )
             autoflush_state = session.autoflush
             try:
                 # no flush to prevent some foreign keys from being voided before all objects are deleted
@@ -1709,6 +1599,7 @@ def delete_everything(self, session=None, commit=True):
                         session.expunge(obj.provenance)
 
                 session.flush()  # flush to finalize deletion of objects before we delete the Image
+
                 # verify that the objects are in fact deleted by deleting the image at the root of the datastore
                 if self.image is not None and self.image.id is not None:
                     session.execute(sa.delete(Image).where(Image.id == self.image.id))
@@ -1730,6 +1621,7 @@ def delete_everything(self, session=None, commit=True):
                 session.commit()
 
             finally:
+                session.flush()
                 session.autoflush = autoflush_state
 
         self.products_committed = ''  # TODO: maybe not critical, but what happens if we fail to delete some of them?
diff --git a/pipeline/detection.py b/pipeline/detection.py
index 884bc8aa..f7f8629f 100644
--- a/pipeline/detection.py
+++ b/pipeline/detection.py
@@ -121,10 +121,7 @@ def __init__(self, **kwargs):
         self.override(kwargs)
 
     def get_process_name(self):
-        if self.subtraction:
-            return 'detection'
-        else:
-            return 'extraction'
+        return 'detection'
 
 
 class Detector:
@@ -230,11 +227,12 @@ def run(self, *args, **kwargs):
 
                 self.pars.do_warning_exception_hangup_injection_here()
 
-                prov = ds.get_provenance(self.pars.get_process_name(), self.pars.get_critical_pars(), session=session)
                 if ds.sub_image is None and ds.image is not None and ds.image.is_sub:
                     ds.sub_image = ds.image
                     ds.image = ds.sub_image.new_image  # back-fill the image from the sub_image
 
+                prov = ds.get_provenance('detection', self.pars.get_critical_pars(), session=session)
+
                 detections = ds.get_detections(prov, session=session)
 
                 if detections is None:
@@ -280,7 +278,7 @@ def run(self, *args, **kwargs):
                 return ds
 
         else:  # regular image
-            prov = ds.get_provenance(self.pars.get_process_name(), self.pars.get_critical_pars(), session=session)
+            prov = ds.get_provenance('extraction', self.pars.get_critical_pars(), session=session)
             try:
                 t_start = time.perf_counter()
                 if parse_bool(os.getenv('SEECHANGE_TRACEMALLOC')):
@@ -340,7 +338,6 @@ def run(self, *args, **kwargs):
             finally:  # make sure datastore is returned to be used in the next step
                 return ds
 
-
     def extract_sources(self, image):
         """Calls one of the extraction methods, based on self.pars.method. """
         sources = None
diff --git a/pipeline/measuring.py b/pipeline/measuring.py
index b8aa36eb..33566609 100644
--- a/pipeline/measuring.py
+++ b/pipeline/measuring.py
@@ -187,7 +187,7 @@ def run(self, *args, **kwargs):
             self.pars.do_warning_exception_hangup_injection_here()
 
             # get the provenance for this step:
-            prov = ds.get_provenance(self.pars.get_process_name(), self.pars.get_critical_pars(), session=session)
+            prov = ds.get_provenance('measuring', self.pars.get_critical_pars(), session=session)
 
             # try to find some measurements in memory or in the database:
             measurements_list = ds.get_measurements(prov, session=session)
diff --git a/pipeline/parameters.py b/pipeline/parameters.py
index 3f03c6eb..ef5212fd 100644
--- a/pipeline/parameters.py
+++ b/pipeline/parameters.py
@@ -119,6 +119,7 @@ def __init__(self, **kwargs):
         self.__docstrings__ = {}
         self.__critical__ = {}
         self.__aliases__ = {}
+        self.__sibling_parameters__ = {}
 
         self.verbose = self.add_par(
             "verbose", 0, int, "Level of verbosity (0=quiet).", critical=False
@@ -230,6 +231,7 @@ def _get_real_par_name(self, key):
             or "_ignore_case" not in self.__dict__
             or "_remove_underscores" not in self.__dict__
             or "__aliases__" not in self.__dict__
+            or "__sibling_parameters__" not in self.__dict__
         ):
             return key
 
@@ -467,16 +469,50 @@ def augment(self, dictionary, ignore_addons=False):
                     if not ignore_addons and "has no attribute" in str(e):
                         raise e
 
-    def get_critical_pars(self):
+    def add_siblings(self, siblings):
+        """Update the sibling parameters dictionary with other parameter objects.
+
+        Siblings are useful when multiple objects (with multiple Parameter objects)
+        need to produce a nested dictionary of critical parameters.
+        Example:
+            The extractor, astrometor and photometor are all included in the "extraction" step.
+            To produce the provenance for that step we will need a nested dictionary that is keyed
+            something like {'sources': <extractor pars>, 'wcs': <astrometor pars>, 'zp': <photometor pars>}.
+            So we'll add to each of them a siblings dictionary keyed:
+            {'sources': extractor.pars, 'wcs': astrometor.pars, 'zp': photometor.pars}
+            so when each one invokes get_critical_pars() it makes a nested dictionary as expected.
+            To get only the critical parameters for the one object, use get_critical_pars(ignore_siblings=True).
+        """
+        if self.__sibling_parameters__ is None:
+            self.__sibling_parameters__ = {}
+
+        self.__sibling_parameters__.update(siblings)
+
+    def get_critical_pars(self, ignore_siblings=False):
         """
         Get a dictionary of the critical parameters.
 
+        Parameters
+        ----------
+        ignore_siblings: bool
+            If True, will not include sibling parameters.
+            By default, calls the siblings of this object
+            when producing the critical parameters.
+
         Returns
         -------
         dict
             The dictionary of critical parameters.
         """
-        return self.to_dict(critical=True, hidden=True)
+        # if there is no dictionary, or it is empty (or if asked to ignore siblings) just return the critical parameters
+        if ignore_siblings or not self.__sibling_parameters__:
+            return self.to_dict(critical=True, hidden=True)
+        else:  # a dictionary based on keys in __sibling_parameters__ with critical pars sub-dictionaries
+            return {
+                key: value.get_critical_pars(ignore_siblings=True)
+                for key, value
+                in self.__sibling_parameters__.items()
+            }
 
     def to_dict(self, critical=False, hidden=False):
         """
@@ -561,11 +597,11 @@ def show_pars(self, owner_pars=None):
             names.append(name)
 
         if len(defaults) > 0:
-            SCLogger.debug(f" Propagated pars: {', '.join(defaults)}")
+            print(f" Propagated pars: {', '.join(defaults)}")
         if len(names) > 0:
             max_length = max(len(n) for n in names)
             for n, d in zip(names, desc):
-                SCLogger.debug(f" {n:>{max_length}}{d}")
+                print(f" {n:>{max_length}}{d}")
 
     def vprint(self, text, threshold=1):
         """
diff --git a/pipeline/photo_cal.py b/pipeline/photo_cal.py
index b9fa1ce7..365ef85f 100644
--- a/pipeline/photo_cal.py
+++ b/pipeline/photo_cal.py
@@ -248,7 +248,7 @@ def run(self, *args, **kwargs):
             self.pars.do_warning_exception_hangup_injection_here()
 
             # get the provenance for this step:
-            prov = ds.get_provenance(self.pars.get_process_name(), self.pars.get_critical_pars(), session=session)
+            prov = ds.get_provenance('extraction', self.pars.get_critical_pars(), session=session)
 
             # try to find the world coordinates in memory or in the database:
             zp = ds.get_zp(prov, session=session)
diff --git a/pipeline/preprocessing.py b/pipeline/preprocessing.py
index accbddd7..1613319f 100644
--- a/pipeline/preprocessing.py
+++ b/pipeline/preprocessing.py
@@ -176,9 +176,10 @@ def run( self, *args, **kwargs ):
             # We also include any overrides to calibrator files, as that indicates
             # that something individual happened here that's different from
             # normal processing of the image.
-            provdict = dict( self.pars.get_critical_pars() )
-            provdict['preprocessing_steps' ] = self._stepstodo
-            prov = ds.get_provenance(self.pars.get_process_name(), provdict, session=session)
+            # Fix this as part of issue #147
+            # provdict = dict( self.pars.get_critical_pars() )
+            # provdict['preprocessing_steps' ] = self._stepstodo
+            prov = ds.get_provenance('preprocessing', self.pars.get_critical_pars(), session=session)
 
             # check if the image already exists in memory or in the database:
             image = ds.get_image(prov, session=session)
diff --git a/pipeline/subtraction.py b/pipeline/subtraction.py
index f3ba94b2..357efa2c 100644
--- a/pipeline/subtraction.py
+++ b/pipeline/subtraction.py
@@ -251,8 +251,6 @@ def run(self, *args, **kwargs):
 
             # get the provenance for this step:
             with SmartSession(session) as session:
-                prov = ds.get_provenance(self.pars.get_process_name(), self.pars.get_critical_pars(), session=session)
-
                 # look for a reference that has to do with the current image
                 ref = ds.get_reference(session=session)
                 if ref is None:
@@ -260,17 +258,7 @@ def run(self, *args, **kwargs):
                         f'Cannot find a reference image corresponding to the datastore inputs: {ds.get_inputs()}'
                     )
 
-                # manually replace the "reference" provenances with the reference image and its products
-                upstreams = prov.upstreams
-                upstreams = [x for x in upstreams if x.process != 'reference']  # remove reference provenance
-                upstreams.append(ref.image.provenance)
-                upstreams.append(ref.sources.provenance)
-                upstreams.append(ref.psf.provenance)
-                upstreams.append(ref.wcs.provenance)
-                upstreams.append(ref.zp.provenance)
-                prov.upstreams = upstreams  # must re-assign to make sure list items are unique
-                prov.update_id()
-                prov = session.merge(prov)
+                prov = ds.get_provenance('subtraction', self.pars.get_critical_pars(), session=session)
                 sub_image = ds.get_subtraction(prov, session=session)
 
                 if sub_image is None:
diff --git a/pipeline/top_level.py b/pipeline/top_level.py
index b23b70c6..aecd0689 100644
--- a/pipeline/top_level.py
+++ b/pipeline/top_level.py
@@ -16,6 +16,7 @@
 
 from models.base import SmartSession
 from models.provenance import Provenance
+from models.reference import Reference
 from models.exposure import Exposure
 from models.report import Report
 
@@ -29,15 +30,13 @@
 # that come from all the different objects.
 PROCESS_OBJECTS = {
     'preprocessing': 'preprocessor',
-    'extraction': 'extractor',  # the same object also makes the PSF (and background?)
-    # TODO: when joining the astro/photo cal into extraction, use this format:
-    # 'extraction': {
-    #     'sources': 'extractor',
-    #     'astro_cal': 'astro_cal',
-    #     'photo_cal': 'photo_cal',
-    # }
-    'astro_cal': 'astro_cal',
-    'photo_cal': 'photo_cal',
+    'extraction': {
+        'sources': 'extractor',
+        'psf': 'extractor',
+        'background': 'extractor',
+        'wcs': 'astrometor',
+        'zp': 'photometor',
+    },
     'subtraction': 'subtractor',
     'detection': 'detector',
     'cutting': 'cutter',
@@ -76,22 +75,29 @@ def __init__(self, **kwargs):
         self.preprocessor = Preprocessor(**preprocessing_config)
 
         # source detection ("extraction" for the regular image!)
-        extraction_config = self.config.value('extraction', {})
-        extraction_config.update(kwargs.get('extraction', {'measure_psf': True}))
+        extraction_config = self.config.value('extraction.sources', {})
+        extraction_config.update(kwargs.get('extraction', {}).get('sources', {}))
+        extraction_config.update({'measure_psf': True})
         self.pars.add_defaults_to_dict(extraction_config)
         self.extractor = Detector(**extraction_config)
 
         # astrometric fit using a first pass of sextractor and then astrometric fit to Gaia
-        astro_cal_config = self.config.value('astro_cal', {})
-        astro_cal_config.update(kwargs.get('astro_cal', {}))
-        self.pars.add_defaults_to_dict(astro_cal_config)
-        self.astro_cal = AstroCalibrator(**astro_cal_config)
+        astrometor_config = self.config.value('extraction.wcs', {})
+        astrometor_config.update(kwargs.get('extraction', {}).get('wcs', {}))
+        self.pars.add_defaults_to_dict(astrometor_config)
+        self.astrometor = AstroCalibrator(**astrometor_config)
 
         # photometric calibration:
-        photo_cal_config = self.config.value('photo_cal', {})
-        photo_cal_config.update(kwargs.get('photo_cal', {}))
-        self.pars.add_defaults_to_dict(photo_cal_config)
-        self.photo_cal = PhotCalibrator(**photo_cal_config)
+        photometor_config = self.config.value('extraction.zp', {})
+        photometor_config.update(kwargs.get('extraction', {}).get('zp', {}))
+        self.pars.add_defaults_to_dict(photometor_config)
+        self.photometor = PhotCalibrator(**photometor_config)
+
+        # make sure when calling get_critical_pars() these objects will produce the full, nested dictionary
+        siblings = {'sources': self.extractor.pars, 'wcs': self.astrometor.pars, 'zp': self.photometor.pars}
+        self.extractor.pars.add_siblings(siblings)
+        self.astrometor.pars.add_siblings(siblings)
+        self.photometor.pars.add_siblings(siblings)
 
         # reference fetching and image subtraction
         subtraction_config = self.config.value('subtraction', {})
@@ -122,7 +128,12 @@ def override_parameters(self, **kwargs):
         """Override some of the parameters for this object and its sub-objects, using Parameters.override(). """
         for key, value in kwargs.items():
             if key in PROCESS_OBJECTS:
-                getattr(self, PROCESS_OBJECTS[key]).pars.override(value)
+                if isinstance(PROCESS_OBJECTS[key], dict):
+                    for sub_key, sub_value in PROCESS_OBJECTS[key].items():
+                        if sub_key in value:
+                            getattr(self, PROCESS_OBJECTS[key][sub_value]).pars.override(value[sub_key])
+                elif isinstance(PROCESS_OBJECTS[key], str):
+                    getattr(self, PROCESS_OBJECTS[key]).pars.override(value)
             else:
                 self.pars.override({key: value})
 
@@ -255,16 +266,16 @@ def run(self, *args, **kwargs):
             ds.update_report('extraction', session)
 
             # find astrometric solution, save WCS into Image object and FITS headers
-            SCLogger.info(f"astro_cal for image id {ds.image.id}")
-            ds = self.astro_cal.run(ds, session)
-            ds.update_report('astro_cal', session)
+            SCLogger.info(f"astrometor for image id {ds.image.id}")
+            ds = self.astrometor.run(ds, session)
+            ds.update_report('extraction', session)
 
             # cross-match against photometric catalogs and get zero point, save into Image object and FITS headers
-            SCLogger.info(f"photo_cal for image id {ds.image.id}")
-            ds = self.photo_cal.run(ds, session)
-            ds.update_report('photo_cal', session)
+            SCLogger.info(f"photometor for image id {ds.image.id}")
+            ds = self.photometor.run(ds, session)
+            ds.update_report('extraction', session)
 
-            # fetch reference images and subtract them, save SubtractedImage objects to DB and disk
+            # fetch reference images and subtract them, save subtracted Image objects to DB and disk
             SCLogger.info(f"subtractor for image id {ds.image.id}")
             ds = self.subtractor.run(ds, session)
             ds.update_report('subtraction', session)
@@ -279,11 +290,14 @@ def run(self, *args, **kwargs):
             ds = self.cutter.run(ds, session)
             ds.update_report('cutting', session)
 
-            # extract photometry, analytical cuts, and deep learning models on the Cutouts:
+            # extract photometry and analytical cuts
             SCLogger.info(f"measurer for image id {ds.image.id}")
             ds = self.measurer.run(ds, session)
             ds.update_report('measuring', session)
 
+            # measure deep learning models on the cutouts/measurements
+            # TODO: add this...
+
             ds.finalize_report(session)
 
             return ds
@@ -297,7 +311,7 @@ def run_with_session(self):
         with SmartSession() as session:
             self.run(session=session)
 
-    def make_provenance_tree(self, exposure, session=None, commit=True):
+    def make_provenance_tree(self, exposure, reference=None, overrides=None, session=None, commit=True):
         """Use the current configuration of the pipeline and all the objects it has
         to generate the provenances for all the processing steps.
         This will conclude with the reporting step, which simply has an upstreams
@@ -309,6 +323,19 @@ def make_provenance_tree(self, exposure, session=None, commit=True):
         exposure : Exposure
             The exposure to use to get the initial provenance.
             This provenance should be automatically created by the exposure.
+        reference: str, Provenance object or None
+            Can be a string matching a valid reference set. This tells the pipeline which
+            provenance to load for the reference.
+            Instead, can provide either a Reference object with a Provenance
+            or the Provenance object of a reference directly.
+            If not given, will simply load the most recently created reference provenance.
+            # TODO: when we implement reference sets, we will probably not allow this input directly to
+            #  this function anymore. Instead, you will need to define the reference set in the config,
+            #  under the subtraction parameters.
+        overrides: dict, optional
+            A dictionary of provenances to override any of the steps in the pipeline.
+            For example, set overrides={'preprocessing': prov} to use a specific provenance
+            for the basic Image provenance.
         session : SmartSession, optional
             The function needs to work with the database to merge existing provenances.
             If a session is given, it will use that, otherwise it will open a new session,
@@ -325,9 +352,11 @@ def make_provenance_tree(self, exposure, session=None, commit=True):
             keyed according to the different steps in the pipeline.
             The provenances are all merged to the session.
         """
+        if overrides is None:
+            overrides = {}
+
         with SmartSession(session) as session:
             # start by getting the exposure and reference
-            exposure = session.merge(exposure)  # also merges the provenance and code_version
             # TODO: need a better way to find the relevant reference PROVENANCE for this exposure
             #  i.e., we do not look for a valid reference and get its provenance, instead,
             #  we look for a provenance based on our policy (that can be defined in the subtraction parameters)
@@ -343,52 +372,62 @@ def make_provenance_tree(self, exposure, session=None, commit=True):
             #  to create all the references for a given RefSet... we need to make sure we can actually
             #  make that happen consistently (e.g., if you change parameters or start mixing instruments
             #  when you make the references it will create multiple provenances for the same RefSet).
-
-            # for now, use the latest provenance that has to do with references
-            ref_prov = session.scalars(
-                sa.select(Provenance).where(Provenance.process == 'reference').order_by(Provenance.created_at.desc())
-            ).first()
-            provs = {'exposure': exposure.provenance}  # TODO: does this always work on any exposure?
-            code_version = exposure.provenance.code_version
-            is_testing = exposure.provenance.is_testing
+            if isinstance(reference, str):
+                raise NotImplementedError('See issue #287')
+            elif isinstance(reference, Reference):
+                ref_prov = reference.provenance
+            elif isinstance(reference, Provenance):
+                ref_prov = reference
+            elif reference is None:  # use the latest provenance that has to do with references
+                ref_prov = session.scalars(
+                    sa.select(Provenance).where(
+                        Provenance.process == 'reference'
+                    ).order_by(Provenance.created_at.desc())
+                ).first()
+
+            exp_prov = session.merge(exposure.provenance)  # also merges the code_version
+            provs = {'exposure': exp_prov}
+            code_version = exp_prov.code_version
+            is_testing = exp_prov.is_testing
 
             for step in PROCESS_OBJECTS:
-                if isinstance(PROCESS_OBJECTS[step], dict):
-                    parameters = {}
-                    for key, value in PROCESS_OBJECTS[step].items():
-                        parameters[key] = getattr(self, value).pars.get_critical_pars()
+                if step in overrides:
+                    provs[step] = overrides[step]
                 else:
-                    parameters = getattr(self, PROCESS_OBJECTS[step]).pars.get_critical_pars()
-
-                # some preprocessing parameters (the "preprocessing_steps") doesn't come from the
-                # config file, but instead comes from the preprocessing itself.
-                # TODO: fix this as part of issue #147
-                if step == 'preprocessing':
-                    if 'preprocessing_steps' not in parameters:
-                        parameters['preprocessing_steps'] = ['overscan', 'linearity', 'flat', 'fringe']
-
-                # figure out which provenances go into the upstreams for this step
-                up_steps = UPSTREAM_STEPS[step]
-                if isinstance(up_steps, str):
-                    up_steps = [up_steps]
-                upstreams = []
-                for upstream in up_steps:
-                    if upstream == 'reference':
-                        upstreams += ref_prov.upstreams
-                    else:
-                        upstreams.append(provs[upstream])
-
-                provs[step] = Provenance(
-                    code_version=code_version,
-                    process=step,
-                    parameters=parameters,
-                    upstreams=upstreams,
-                    is_testing=is_testing,
-                )
+                    obj_name = PROCESS_OBJECTS[step]
+                    if isinstance(obj_name, dict):
+                        # get the first item of the dictionary and hope its pars object has siblings defined correctly:
+                        obj_name = obj_name.get(list(obj_name.keys())[0])
+                    parameters = getattr(self, obj_name).pars.get_critical_pars()
+
+                    # some preprocessing parameters (the "preprocessing_steps") don't come from the
+                    # config file, but instead come from the preprocessing itself.
+                    # TODO: fix this as part of issue #147
+                    # if step == 'preprocessing':
+                    #     parameters['preprocessing_steps'] = ['overscan', 'linearity', 'flat', 'fringe']
+
+                    # figure out which provenances go into the upstreams for this step
+                    up_steps = UPSTREAM_STEPS[step]
+                    if isinstance(up_steps, str):
+                        up_steps = [up_steps]
+                    upstreams = []
+                    for upstream in up_steps:
+                        if upstream == 'reference':
+                            upstreams += ref_prov.upstreams
+                        else:
+                            upstreams.append(provs[upstream])
+
+                    provs[step] = Provenance(
+                        code_version=code_version,
+                        process=step,
+                        parameters=parameters,
+                        upstreams=upstreams,
+                        is_testing=is_testing,
+                    )
 
                 provs[step] = provs[step].merge_concurrent(session=session, commit=commit)
 
-            # if commit:
-            #     session.commit()
+            if commit:
+                session.commit()
 
             return provs
diff --git a/pipeline/utils.py b/pipeline/utils.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/tests/fixtures/decam.py b/tests/fixtures/decam.py
index e2c41b8b..8e70108f 100644
--- a/tests/fixtures/decam.py
+++ b/tests/fixtures/decam.py
@@ -268,7 +268,7 @@ def decam_datastore(
         decam_exposure,
         'N1',
         cache_dir=decam_cache_dir,
-        cache_base_name='115/c4d_20221104_074232_N1_g_Sci_FVOSOC',
+        cache_base_name='115/c4d_20221104_074232_N1_g_Sci_VCOACQ',
         save_original_image=True
     )
     # This save is redundant, as the datastore_factory calls save_and_commit
@@ -449,9 +449,6 @@ def decam_reference(decam_ref_datastore):
             upstreams=[
                 ds.image.provenance,
                 ds.sources.provenance,
-                ds.psf.provenance,
-                ds.wcs.provenance,
-                ds.zp.provenance,
             ],
             is_testing=True,
         )
diff --git a/tests/fixtures/pipeline_objects.py b/tests/fixtures/pipeline_objects.py
index 94c4aea1..93be2623 100644
--- a/tests/fixtures/pipeline_objects.py
+++ b/tests/fixtures/pipeline_objects.py
@@ -25,7 +25,7 @@
 from pipeline.detection import Detector
 from pipeline.astro_cal import AstroCalibrator
 from pipeline.photo_cal import PhotCalibrator
-from pipeline.coaddition import Coadder
+from pipeline.coaddition import Coadder, CoaddPipeline
 from pipeline.subtraction import Subtractor
 from pipeline.cutting import Cutter
 from pipeline.measuring import Measurer
@@ -36,6 +36,7 @@
 
 from improc.bitmask_tools import make_saturated_flag
 
+
 @pytest.fixture(scope='session')
 def preprocessor_factory(test_config):
 
@@ -61,7 +62,7 @@ def preprocessor(preprocessor_factory):
 def extractor_factory(test_config):
 
     def make_extractor():
-        extr = Detector(**test_config.value('extraction'))
+        extr = Detector(**test_config.value('extraction.sources'))
         extr.pars._enforce_no_new_attrs = False
         extr.pars.test_parameter = extr.pars.add_par(
             'test_parameter', 'test_value', str, 'parameter to define unique tests', critical=True
@@ -82,7 +83,7 @@ def extractor(extractor_factory):
 def astrometor_factory(test_config):
 
     def make_astrometor():
-        astrom = AstroCalibrator(**test_config.value('astro_cal'))
+        astrom = AstroCalibrator(**test_config.value('extraction.wcs'))
         astrom.pars._enforce_no_new_attrs = False
         astrom.pars.test_parameter = astrom.pars.add_par(
             'test_parameter', 'test_value', str, 'parameter to define unique tests', critical=True
@@ -103,7 +104,7 @@ def astrometor(astrometor_factory):
 def photometor_factory(test_config):
 
     def make_photometor():
-        photom = PhotCalibrator(**test_config.value('photo_cal'))
+        photom = PhotCalibrator(**test_config.value('extraction.zp'))
         photom.pars._enforce_no_new_attrs = False
         photom.pars.test_parameter = photom.pars.add_par(
             'test_parameter', 'test_value', str, 'parameter to define unique tests', critical=True
@@ -242,12 +243,20 @@ def make_pipeline():
         p = Pipeline(**test_config.value('pipeline'))
         p.preprocessor = preprocessor_factory()
         p.extractor = extractor_factory()
-        p.astro_cal = astrometor_factory()
-        p.photo_cal = photometor_factory()
+        p.astrometor = astrometor_factory()
+        p.photometor = photometor_factory()
+
+        # make sure when calling get_critical_pars() these objects will produce the full, nested dictionary
+        siblings = {'sources': p.extractor.pars, 'wcs': p.astrometor.pars, 'zp': p.photometor.pars}
+        p.extractor.pars.add_siblings(siblings)
+        p.astrometor.pars.add_siblings(siblings)
+        p.photometor.pars.add_siblings(siblings)
+
         p.subtractor = subtractor_factory()
         p.detector = detector_factory()
         p.cutter = cutter_factory()
         p.measurer = measurer_factory()
+
         return p
 
     return make_pipeline
@@ -258,6 +267,37 @@ def pipeline_for_tests(pipeline_factory):
     return pipeline_factory()
 
 
+@pytest.fixture(scope='session')
+def coadd_pipeline_factory(
+        coadder_factory,
+        extractor_factory,
+        astrometor_factory,
+        photometor_factory,
+        test_config,
+):
+    def make_pipeline():
+        p = CoaddPipeline(**test_config.value('pipeline'))
+        p.coadder = coadder_factory()
+        p.extractor = extractor_factory()
+        p.astrometor = astrometor_factory()
+        p.photometor = photometor_factory()
+
+        # make sure when calling get_critical_pars() these objects will produce the full, nested dictionary
+        siblings = {'sources': p.extractor.pars, 'wcs': p.astrometor.pars, 'zp': p.photometor.pars}
+        p.extractor.pars.add_siblings(siblings)
+        p.astrometor.pars.add_siblings(siblings)
+        p.photometor.pars.add_siblings(siblings)
+
+        return p
+
+    return make_pipeline
+
+
+@pytest.fixture
+def coadd_pipeline_for_tests(coadd_pipeline_factory):
+    return coadd_pipeline_factory()
+
+
 @pytest.fixture(scope='session')
 def datastore_factory(data_dir, pipeline_factory):
     """Provide a function that returns a datastore with all the products based on the given exposure and section ID.
@@ -303,6 +343,7 @@ def make_datastore(
 
         with SmartSession(session) as session:
             code_version = session.merge(code_version)
+
             if ds.image is not None:  # if starting from an externally provided Image, must merge it first
                 ds.image = ds.image.merge_all(session)
 
@@ -329,19 +370,20 @@ def make_datastore(
                         shutil.copy2( cache_path, ds.path_to_original_image )
 
                     # add the preprocessing steps from instrument (TODO: remove this as part of Issue #142)
-                    preprocessing_steps = ds.image.instrument_object.preprocessing_steps
-                    prep_pars = p.preprocessor.pars.get_critical_pars()
-                    prep_pars['preprocessing_steps'] = preprocessing_steps
+                    # preprocessing_steps = ds.image.instrument_object.preprocessing_steps
+                    # prep_pars = p.preprocessor.pars.get_critical_pars()
+                    # prep_pars['preprocessing_steps'] = preprocessing_steps
 
                     upstreams = [ds.exposure.provenance] if ds.exposure is not None else []  # images without exposure
                     prov = Provenance(
                         code_version=code_version,
                         process='preprocessing',
                         upstreams=upstreams,
-                        parameters=prep_pars,
+                        parameters=p.preprocessor.pars.get_critical_pars(),
                         is_testing=True,
                     )
                     prov = session.merge(prov)
+                    session.commit()
 
                     # if Image already exists on the database, use that instead of this one
                     existing = session.scalars(sa.select(Image).where(Image.filepath == ds.image.filepath)).first()
@@ -355,6 +397,7 @@ def make_datastore(
                             ):
                                 setattr(existing, key, value)
                         ds.image = existing  # replace with the existing row
+
                     ds.image.provenance = prov
 
                     # make sure this is saved to the archive as well
@@ -362,7 +405,7 @@ def make_datastore(
 
             if ds.image is None:  # make the preprocessed image
                 SCLogger.debug('making preprocessed image. ')
-                ds = p.preprocessor.run(ds)
+                ds = p.preprocessor.run(ds, session)
                 ds.image.provenance.is_testing = True
                 if bad_pixel_map is not None:
                     ds.image.flags |= bad_pixel_map
@@ -410,22 +453,54 @@ def make_datastore(
                 ds.image.bkg_mean_estimate = backgrounder.globalback
                 ds.image.bkg_rms_estimate = backgrounder.globalrms
 
-            ############# extraction to create sources / PSF #############
+            # TODO: move the code below here up to above preprocessing, once we have reference sets
+            try:  # check if this datastore can load a reference
+                # this is a hack to tell the datastore that the given image's provenance is the right one to use
+                ref = ds.get_reference(session=session)
+                ref_prov = ref.provenance
+            except ValueError as e:
+                if 'No reference image found' in str(e):
+                    ref = None
+                    # make a placeholder reference just to be able to make a provenance tree
+                    # this doesn't matter in this case, because if there is no reference
+                    # then the datastore is returned without a subtraction, so all the
+                    # provenances that have the reference provenances as upstream will
+                    # not even exist.
+
+                    # TODO: we really should be working in a state where there is a reference set
+                    #  that has one provenance attached to it, that exists before we start up
+                    #  the pipeline. Here we are doing the opposite: we first check if a specific
+                    #  reference exists, and only then chose the provenance based on the available ref.
+                    # TODO: once we have a reference that is independent of the image, we can move this
+                    #  code that makes the prov_tree up to before preprocessing
+                    ref_prov = Provenance(
+                        process='reference',
+                        code_version=code_version,
+                        parameters={},
+                        upstreams=[],
+                        is_testing=True,
+                    )
+                else:
+                    raise e  # if any other error comes up, raise it
+
+            ############# extraction to create sources / PSF / WCS / ZP #############
             if (   ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
                    ( cache_dir is not None ) and ( cache_base_name is not None )
                 ):
-                # try to get the SourceList from cache
+                # try to get the SourceList, PSF, WCS and ZP from cache
                 prov = Provenance(
                     code_version=code_version,
                     process='extraction',
                     upstreams=[ds.image.provenance],
-                    parameters=p.extractor.pars.get_critical_pars(),
+                    parameters=p.extractor.pars.get_critical_pars(),  # the siblings will be loaded automatically
                     is_testing=True,
                 )
                 prov = session.merge(prov)
+                session.commit()
+
                 cache_name = f'{cache_base_name}.sources_{prov.id[:6]}.fits.json'
-                cache_path = os.path.join(cache_dir, cache_name)
-                if os.path.isfile(cache_path):
+                sources_cache_path = os.path.join(cache_dir, cache_name)
+                if os.path.isfile(sources_cache_path):
                     SCLogger.debug('loading source list from cache. ')
                     ds.sources = copy_from_cache(SourceList, cache_dir, cache_name)
 
@@ -452,8 +527,8 @@ def make_datastore(
 
                 # try to get the PSF from cache
                 cache_name = f'{cache_base_name}.psf_{prov.id[:6]}.fits.json'
-                cache_path = os.path.join(cache_dir, cache_name)
-                if os.path.isfile(cache_path):
+                psf_cache_path = os.path.join(cache_dir, cache_name)
+                if os.path.isfile(psf_cache_path):
                     SCLogger.debug('loading PSF from cache. ')
                     ds.psf = copy_from_cache(PSF, cache_dir, cache_name)
 
@@ -478,40 +553,25 @@ def make_datastore(
                     # make sure this is saved to the archive as well
                     ds.psf.save(verify_md5=False, overwrite=True)
 
-            if ds.sources is None or ds.psf is None:  # make the source list from the regular image
-                SCLogger.debug('extracting sources. ')
-                ds = p.extractor.run(ds)
-                ds.sources.save()
-                ds.psf.save(overwrite=True)
-                if not os.getenv( "LIMIT_CACHE_USAGE" ):
-                    copy_to_cache(ds.sources, cache_dir)
-                    output_path = copy_to_cache(ds.psf, cache_dir)
-                    if cache_dir is not None and cache_base_name is not None and output_path != cache_path:
-                        warnings.warn(f'cache path {cache_path} does not match output path {output_path}')
-
-            ############## astro_cal to create wcs ################
-            if cache_dir is not None and cache_base_name is not None:
-                prov = Provenance(
-                        code_version=code_version,
-                        process='astro_cal',
-                        upstreams=[ds.sources.provenance],
-                        parameters=p.astro_cal.pars.get_critical_pars(),
-                        is_testing=True,
-                    )
+                ############## astro_cal to create wcs ################
                 cache_name = f'{cache_base_name}.wcs_{prov.id[:6]}.txt.json'
-                cache_path = os.path.join(cache_dir, cache_name)
-                if os.path.isfile(cache_path):
+                wcs_cache_path = os.path.join(cache_dir, cache_name)
+                if os.path.isfile(wcs_cache_path):
                     SCLogger.debug('loading WCS from cache. ')
                     ds.wcs = copy_from_cache(WorldCoordinates, cache_dir, cache_name)
                     prov = session.merge(prov)
 
                     # check if WCS already exists on the database
-                    existing = session.scalars(
-                        sa.select(WorldCoordinates).where(
-                            WorldCoordinates.sources_id == ds.sources.id,
-                            WorldCoordinates.provenance_id == prov.id
-                        )
-                    ).first()
+                    if ds.sources is not None:
+                        existing = session.scalars(
+                            sa.select(WorldCoordinates).where(
+                                WorldCoordinates.sources_id == ds.sources.id,
+                                WorldCoordinates.provenance_id == prov.id
+                            )
+                        ).first()
+                    else:
+                        existing = None
+
                     if existing is not None:
                         # overwrite the existing row data using the JSON cache file
                         for key in sa.inspect(ds.wcs).mapper.columns.keys():
@@ -528,41 +588,25 @@ def make_datastore(
                     # make sure this is saved to the archive as well
                     ds.wcs.save(verify_md5=False, overwrite=True)
 
-            if ds.wcs is None:  # make the WCS
-                SCLogger.debug('Running astrometric calibration')
-                ds = p.astro_cal.run(ds)
-                ds.wcs.save()
-                if ( ( cache_dir is not None ) and ( cache_base_name is not None ) and
-                     ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) ):
-                    output_path = copy_to_cache(ds.wcs, cache_dir)
-                    if output_path != cache_path:
-                        warnings.warn(f'cache path {cache_path} does not match output path {output_path}')
+                ########### photo_cal to create zero point ############
 
-            ########### photo_cal to create zero point ############
-            if (   ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
-                   ( cache_dir is not None ) and ( cache_base_name is not None )
-                ):
                 cache_name = cache_base_name + '.zp.json'
-                cache_path = os.path.join(cache_dir, cache_name)
-                if os.path.isfile(cache_path):
+                zp_cache_path = os.path.join(cache_dir, cache_name)
+                if os.path.isfile(zp_cache_path):
                     SCLogger.debug('loading zero point from cache. ')
                     ds.zp = copy_from_cache(ZeroPoint, cache_dir, cache_name)
-                    prov = Provenance(
-                        code_version=code_version,
-                        process='photo_cal',
-                        upstreams=[ds.sources.provenance, ds.wcs.provenance],
-                        parameters=p.photo_cal.pars.get_critical_pars(),
-                        is_testing=True,
-                    )
-                    prov = session.merge(prov)
 
                     # check if ZP already exists on the database
-                    existing = session.scalars(
-                        sa.select(ZeroPoint).where(
-                            ZeroPoint.sources_id == ds.sources.id,
-                            ZeroPoint.provenance_id == prov.id
-                        )
-                    ).first()
+                    if ds.sources is not None:
+                        existing = session.scalars(
+                            sa.select(ZeroPoint).where(
+                                ZeroPoint.sources_id == ds.sources.id,
+                                ZeroPoint.provenance_id == prov.id
+                            )
+                        ).first()
+                    else:
+                        existing = None
+
                     if existing is not None:
                         # overwrite the existing row data using the JSON cache file
                         for key in sa.inspect(ds.zp).mapper.columns.keys():
@@ -577,23 +621,43 @@ def make_datastore(
                     ds.zp.provenance = prov
                     ds.zp.sources = ds.sources
 
-            if ds.zp is None:  # make the zero point
+            if ds.sources is None or ds.psf is None or ds.wcs is None or ds.zp is None:  # redo extraction
+                SCLogger.debug('extracting sources. ')
+                ds = p.extractor.run(ds, session)
+
+                ds.sources.save()
+                if cache_dir is not None and cache_base_name is not None:
+                    output_path = copy_to_cache(ds.sources, cache_dir)
+                    if cache_dir is not None and cache_base_name is not None and output_path != sources_cache_path:
+                        warnings.warn(f'cache path {sources_cache_path} does not match output path {output_path}')
+
+                ds.psf.save(overwrite=True)
+                if cache_dir is not None and cache_base_name is not None:
+                    output_path = copy_to_cache(ds.psf, cache_dir)
+                    if cache_dir is not None and cache_base_name is not None and output_path != psf_cache_path:
+                        warnings.warn(f'cache path {psf_cache_path} does not match output path {output_path}')
+
+                SCLogger.debug('Running astrometric calibration')
+                ds = p.astrometor.run(ds, session)
+                ds.wcs.save()
+                if ((cache_dir is not None) and (cache_base_name is not None) and
+                        (not os.getenv("LIMIT_CACHE_USAGE"))):
+                    output_path = copy_to_cache(ds.wcs, cache_dir)
+                    if output_path != wcs_cache_path:
+                        warnings.warn(f'cache path {wcs_cache_path} does not match output path {output_path}')
+
                 SCLogger.debug('Running photometric calibration')
-                ds = p.photo_cal.run(ds)
-                if ( ( cache_dir is not None ) and ( cache_base_name is not None ) and
-                     ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) ):
+                ds = p.photometor.run(ds, session)
+                if (   ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
+                       ( cache_dir is not None ) and ( cache_base_name is not None )
+                ):
                     output_path = copy_to_cache(ds.zp, cache_dir, cache_name)
-                    if output_path != cache_path:
-                        warnings.warn(f'cache path {cache_path} does not match output path {output_path}')
+                    if output_path != zp_cache_path:
+                        warnings.warn(f'cache path {zp_cache_path} does not match output path {output_path}')
 
             ds.save_and_commit(session=session)
-
-            try:  # if no reference is found, simply return the datastore without the rest of the products
-                ref = ds.get_reference()  # first make sure this actually manages to find the reference image
-            except ValueError as e:
-                if 'No reference image found' in str(e):
-                    return ds
-                raise e  # if any other error comes up, raise it
+            if ref is None:
+                return ds  # if no reference is found, simply return the datastore without the rest of the products
 
             # try to find the subtraction image in the cache
             if cache_dir is not None:
@@ -603,16 +667,15 @@ def make_datastore(
                     upstreams=[
                         ds.image.provenance,
                         ds.sources.provenance,
-                        ds.wcs.provenance,
-                        ds.zp.provenance,
                         ref.image.provenance,
                         ref.sources.provenance,
-                        ref.wcs.provenance,
-                        ref.zp.provenance,
                     ],
                     parameters=p.subtractor.pars.get_critical_pars(),
                     is_testing=True,
                 )
+                prov = session.merge(prov)
+                session.commit()
+
                 sub_im = Image.from_new_and_ref(ds.image, ref.image)
                 sub_im.provenance = prov
                 cache_sub_name = sub_im.invent_filepath()
@@ -704,7 +767,7 @@ def make_datastore(
                             ds.sub_image._aligned_images = [image_aligned_new, image_aligned_ref]
 
             if ds.sub_image is None:  # no hit in the cache
-                ds = p.subtractor.run(ds)
+                ds = p.subtractor.run(ds, session)
                 ds.sub_image.save(verify_md5=False)  # make sure it is also saved to archive
                 if not os.getenv( "LIMIT_CACHE_USAGE" ):
                     copy_to_cache(ds.sub_image, cache_dir)
@@ -728,6 +791,9 @@ def make_datastore(
                 parameters=p.detector.pars.get_critical_pars(),
                 is_testing=True,
             )
+            prov = session.merge(prov)
+            session.commit()
+
             cache_name = os.path.join(cache_dir, cache_sub_name + f'.sources_{prov.id[:6]}.npy.json')
             if ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and ( os.path.isfile(cache_name) ):
                 SCLogger.debug('loading detections from cache. ')
@@ -737,7 +803,7 @@ def make_datastore(
                 ds.sub_image.sources = ds.detections
                 ds.detections.save(verify_md5=False)
             else:  # cannot find detections on cache
-                ds = p.detector.run(ds)
+                ds = p.detector.run(ds, session)
                 ds.detections.save(verify_md5=False)
                 if not os.getenv( "LIMIT_CACHE_USAGE" ):
                     copy_to_cache(ds.detections, cache_dir, cache_name)
@@ -750,6 +816,9 @@ def make_datastore(
                 parameters=p.cutter.pars.get_critical_pars(),
                 is_testing=True,
             )
+            prov = session.merge(prov)
+            session.commit()
+
             cache_name = os.path.join(cache_dir, cache_sub_name + f'.cutouts_{prov.id[:6]}.h5')
             if ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and ( os.path.isfile(cache_name) ):
                 SCLogger.debug('loading cutouts from cache. ')
@@ -759,7 +828,7 @@ def make_datastore(
                 [setattr(c, 'sources', ds.detections) for c in ds.cutouts]
                 Cutouts.save_list(ds.cutouts)  # make sure to save to archive as well
             else:  # cannot find cutouts on cache
-                ds = p.cutter.run(ds)
+                ds = p.cutter.run(ds, session)
                 Cutouts.save_list(ds.cutouts)
                 if not os.getenv( "LIMIT_CACHE_USAGE" ):
                     copy_list_to_cache(ds.cutouts, cache_dir)
@@ -772,10 +841,13 @@ def make_datastore(
                 parameters=p.measurer.pars.get_critical_pars(),
                 is_testing=True,
             )
+            prov = session.merge(prov)
+            session.commit()
 
             cache_name = os.path.join(cache_dir, cache_sub_name + f'.measurements_{prov.id[:6]}.json')
 
-            if os.path.isfile(cache_name):  # note that the cache contains ALL the measurements, not only the good ones
+            if ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and ( os.path.isfile(cache_name) ):
+                # note that the cache contains ALL the measurements, not only the good ones
                 SCLogger.debug('loading measurements from cache. ')
                 ds.all_measurements = copy_list_from_cache(Measurements, cache_dir, cache_name)
                 [setattr(m, 'provenance', prov) for m in ds.all_measurements]
@@ -791,7 +863,7 @@ def make_datastore(
                 [m.associate_object(session) for m in ds.measurements]  # create or find an object for each measurement
                 # no need to save list because Measurements is not a FileOnDiskMixin!
             else:  # cannot find measurements on cache
-                ds = p.measurer.run(ds)
+                ds = p.measurer.run(ds, session)
                 copy_list_to_cache(ds.all_measurements, cache_dir, cache_name)  # must provide filepath!
 
             ds.save_and_commit(session=session)
diff --git a/tests/fixtures/ptf.py b/tests/fixtures/ptf.py
index a08ded5f..1fb671da 100644
--- a/tests/fixtures/ptf.py
+++ b/tests/fixtures/ptf.py
@@ -1,3 +1,6 @@
+import uuid
+import warnings
+
 import pytest
 import os
 import shutil
@@ -169,7 +172,7 @@ def ptf_datastore(datastore_factory, ptf_exposure, ptf_ref, ptf_cache_dir, ptf_b
         ptf_exposure,
         11,
         cache_dir=ptf_cache_dir,
-        cache_base_name='187/PTF_20110429_040004_11_R_Sci_5F5TAU',
+        cache_base_name='187/PTF_20110429_040004_11_R_Sci_QTD4UW',
         overrides={'extraction': {'threshold': 5}},
         bad_pixel_map=ptf_bad_pixel_map,
     )
@@ -234,6 +237,7 @@ def factory(start_date='2009-04-04', end_date='2013-03-03', max_images=None):
         for url in urls:
             exp = ptf_downloader(url)
             exp.instrument_object.fetch_sections()
+            exp.md5sum = uuid.uuid4()  # this will save some memory as the exposures are not saved to archive
             try:
                 # produce an image
                 ds = datastore_factory(
@@ -265,7 +269,7 @@ def factory(start_date='2009-04-04', end_date='2013-03-03', max_images=None):
                 SCLogger.debug(f'Error processing {url}')  # this will also leave behind exposure and image data on disk only
                 raise e
                 # SCLogger.debug(e)  # TODO: should we be worried that some of these images can't complete their processing?
-                continue
+                # continue
 
             images.append(ds.image)
             if max_images is not None and len(images) >= max_images:
@@ -351,7 +355,7 @@ def ptf_aligned_images(request, ptf_cache_dir, data_dir, code_version):
             image.save()
             filepath = copy_to_cache(image, cache_dir)
             if image.psf.filepath is None:  # save only PSF objects that haven't been saved yet
-                image.psf.save()
+                image.psf.save(overwrite=True)
             if not os.getenv( "LIMIT_CACHE_USAGE" ):
                copy_to_cache(image.psf, cache_dir)
                copy_to_cache(image.zp, cache_dir, filepath=filepath[:-len('.image.fits.json')]+'.zp.json')
@@ -378,43 +382,49 @@ def ptf_aligned_images(request, ptf_cache_dir, data_dir, code_version):
 
     # must delete these here, as the cleanup for the getfixturevalue() happens after pytest_sessionfinish!
     if 'ptf_reference_images' in locals():
-        with SmartSession() as session:
+        with SmartSession() as session, warnings.catch_warnings():
+            warnings.filterwarnings(
+                action='ignore',
+                message=r'.*DELETE statement on table .* expected to delete \d* row\(s\).*',
+            )
+            # warnings.filterwarnings(
+            #     'ignore',
+            #     message=r".*Object of type .* not in session, .* operation along .* won't proceed.*"
+            # )
             for image in ptf_reference_images:
                 image = session.merge(image)
-                image.exposure.delete_from_disk_and_database(commit=False, session=session)
-                image.delete_from_disk_and_database(commit=False, session=session, remove_downstreams=True)
+                image.exposure.delete_from_disk_and_database(commit=False, session=session, remove_downstreams=True)
+                # image.delete_from_disk_and_database(commit=False, session=session, remove_downstreams=True)
             session.commit()
 
 
 @pytest.fixture
-def ptf_ref(ptf_reference_images, ptf_aligned_images, coadder, ptf_cache_dir, data_dir, code_version):
-    pipe = CoaddPipeline()
-    pipe.coadder = coadder  # use this one that has a test_parameter defined
+def ptf_ref(
+        ptf_reference_images,
+        ptf_aligned_images,
+        coadd_pipeline_for_tests,
+        ptf_cache_dir,
+        data_dir,
+        code_version
+):
+    pipe = coadd_pipeline_for_tests
 
     # build up the provenance tree
     with SmartSession() as session:
         code_version = session.merge(code_version)
         im = ptf_reference_images[0]
-        upstream_provs = [im.provenance, im.sources.provenance, im.psf.provenance, im.wcs.provenance, im.zp.provenance]
+        upstream_provs = [im.provenance, im.sources.provenance]
         im_prov = Provenance(
             process='coaddition',
-            parameters=coadder.pars.get_critical_pars(),
+            parameters=pipe.coadder.pars.get_critical_pars(),
             upstreams=upstream_provs,
             code_version=code_version,
             is_testing=True,
         )
 
-        cache_base_name = f'187/PTF_20090405_073932_11_R_ComSci_{im_prov.id[:6]}_u-ywhkxr'
+        cache_base_name = f'187/PTF_20090405_073932_11_R_ComSci_{im_prov.id[:6]}_u-wswtff'
 
-        psf_prov = Provenance(
-            process='extraction',
-            parameters=pipe.extractor.pars.get_critical_pars(),
-            upstreams=[im_prov],
-            code_version=code_version,
-            is_testing=True,
-        )
-
-        # this is the same provenance as psf_prov (see Issue #176)
+        # this provenance is used for sources, psf, wcs, zp
         sources_prov = Provenance(
             process='extraction',
             parameters=pipe.extractor.pars.get_critical_pars(),
@@ -422,31 +432,18 @@ def ptf_ref(ptf_reference_images, ptf_aligned_images, coadder, ptf_cache_dir, da
             code_version=code_version,
             is_testing=True,
         )
+    extensions = [
+        'image.fits',
+        f'psf_{sources_prov.id[:6]}.fits',
+        f'sources_{sources_prov.id[:6]}.fits',
+        f'wcs_{sources_prov.id[:6]}.txt',
+        'zp'
+    ]
+    filenames = [os.path.join(ptf_cache_dir, cache_base_name) + f'.{ext}.json' for ext in extensions]
 
-        wcs_prov = Provenance(
-            process='astro_cal',
-            parameters=pipe.astro_cal.pars.get_critical_pars(),
-            upstreams=[sources_prov],
-            code_version=code_version,
-            is_testing=True,
-        )
-
-        zp_prov = Provenance(
-            process='photo_cal',
-            parameters=pipe.photo_cal.pars.get_critical_pars(),
-            upstreams=[sources_prov, wcs_prov],
-            code_version=code_version,
-            is_testing=True,
-        )
-
-    extensions = ['image.fits', f'psf_{psf_prov.id[:6]}.fits', f'sources_{sources_prov.id[:6]}.fits', 'wcs', 'zp']
-    if not os.getenv( "LIMIT_CACHE_USAGE" ):
-        filenames = [os.path.join(ptf_cache_dir, cache_base_name) + f'.{ext}.json' for ext in extensions]
-    else:
-        filenames = []
-    if (     ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
-             ( all([os.path.isfile(filename) for filename in filenames]) )
-        ):  # can load from cache
+    if ( not os.getenv( "LIMIT_CACHE_USAGE" ) and
+         all([os.path.isfile(filename) for filename in filenames])
+    ):  # can load from cache
         # get the image:
         coadd_image = copy_from_cache(Image, ptf_cache_dir, cache_base_name + '.image.fits')
         # we must load these images in order to save the reference image with upstreams
@@ -456,26 +453,28 @@ def ptf_ref(ptf_reference_images, ptf_aligned_images, coadder, ptf_cache_dir, da
         assert coadd_image.provenance_id == coadd_image.provenance.id
 
         # get the PSF:
-        coadd_image.psf = copy_from_cache(PSF, ptf_cache_dir, cache_base_name + f'.psf_{psf_prov.id[:6]}.fits')
-        coadd_image.psf.provenance = psf_prov
+        coadd_image.psf = copy_from_cache(PSF, ptf_cache_dir, cache_base_name + f'.psf_{sources_prov.id[:6]}.fits')
+        coadd_image.psf.provenance = sources_prov
         assert coadd_image.psf.provenance_id == coadd_image.psf.provenance.id
 
         # get the source list:
-        coadd_image.sources =copy_from_cache(
+        coadd_image.sources = copy_from_cache(
             SourceList, ptf_cache_dir, cache_base_name + f'.sources_{sources_prov.id[:6]}.fits'
         )
         coadd_image.sources.provenance = sources_prov
         assert coadd_image.sources.provenance_id == coadd_image.sources.provenance.id
 
         # get the WCS:
-        coadd_image.wcs = copy_from_cache(WorldCoordinates, ptf_cache_dir, cache_base_name + '.wcs')
-        coadd_image.wcs.provenance = wcs_prov
+        coadd_image.wcs = copy_from_cache(
+            WorldCoordinates, ptf_cache_dir, cache_base_name + f'.wcs_{sources_prov.id[:6]}.txt'
+        )
+        coadd_image.wcs.provenance = sources_prov
         coadd_image.sources.wcs = coadd_image.wcs
         assert coadd_image.wcs.provenance_id == coadd_image.wcs.provenance.id
 
         # get the zero point:
         coadd_image.zp = copy_from_cache(ZeroPoint, ptf_cache_dir, cache_base_name + '.zp')
-        coadd_image.zp.provenance = zp_prov
+        coadd_image.zp.provenance = sources_prov
         coadd_image.sources.zp = coadd_image.zp
         assert coadd_image.zp.provenance_id == coadd_image.zp.provenance.id
 
@@ -492,7 +491,7 @@ def ptf_ref(ptf_reference_images, ptf_aligned_images, coadder, ptf_cache_dir, da
             copy_to_cache(pipe.datastore.image, ptf_cache_dir)
             copy_to_cache(pipe.datastore.sources, ptf_cache_dir)
             copy_to_cache(pipe.datastore.psf, ptf_cache_dir)
-            copy_to_cache(pipe.datastore.wcs, ptf_cache_dir, cache_base_name + '.wcs.json')
+            copy_to_cache(pipe.datastore.wcs, ptf_cache_dir)
             copy_to_cache(pipe.datastore.zp, ptf_cache_dir, cache_base_name + '.zp.json')
 
     with SmartSession() as session:
@@ -510,9 +509,8 @@ def ptf_ref(ptf_reference_images, ptf_aligned_images, coadder, ptf_cache_dir, da
     yield ref
 
     with SmartSession() as session:
-        coadd_image = session.merge(coadd_image)
-        coadd_image.delete_from_disk_and_database(commit=False, session=session, remove_downstreams=True)
-        session.commit()
+        coadd_image = coadd_image.merge_all(session=session)
+        coadd_image.delete_from_disk_and_database(commit=True, session=session, remove_downstreams=True)
         ref_in_db = session.scalars(sa.select(Reference).where(Reference.id == ref.id)).first()
         assert ref_in_db is None  # should have been deleted by cascade when image is deleted
 
diff --git a/tests/fixtures/simulated.py b/tests/fixtures/simulated.py
index 97e7c61b..5d62bc94 100644
--- a/tests/fixtures/simulated.py
+++ b/tests/fixtures/simulated.py
@@ -431,13 +431,16 @@ def sim_image_list(
 
     yield images
 
-    with SmartSession() as session:
+    with SmartSession() as session, warnings.catch_warnings():
+        warnings.filterwarnings(
+            action='ignore',
+            message=r'.*DELETE statement on table .* expected to delete \d* row\(s\).*',
+        )
         for im in images:
             im = im.merge_all(session)
             exp = im.exposure
             im.delete_from_disk_and_database(session=session, commit=False, remove_downstreams=True)
             exp.delete_from_disk_and_database(session=session, commit=False)
-
         session.commit()
 
 
@@ -617,7 +620,6 @@ def sim_sub_image_list(
 
     with SmartSession() as session:
         for sub in sub_images:
-            # sub = sub.merge_all(session)
             sub.delete_from_disk_and_database(session=session, commit=False, remove_downstreams=True)
         session.commit()
 
diff --git a/tests/improc/test_alignment.py b/tests/improc/test_alignment.py
index c31fa50a..8ac5968b 100644
--- a/tests/improc/test_alignment.py
+++ b/tests/improc/test_alignment.py
@@ -1,4 +1,3 @@
-import logging
 import warnings
 
 import pytest
diff --git a/tests/models/test_decam.py b/tests/models/test_decam.py
index 890fac29..1c0c2ed8 100644
--- a/tests/models/test_decam.py
+++ b/tests/models/test_decam.py
@@ -21,8 +21,6 @@
 import util.radec
 from util.logger import SCLogger
 
-from tests.conftest import CODE_ROOT
-
 
 def test_decam_exposure(decam_filename):
     assert os.path.isfile(decam_filename)
diff --git a/tests/models/test_image.py b/tests/models/test_image.py
index aafae540..f74f151d 100644
--- a/tests/models/test_image.py
+++ b/tests/models/test_image.py
@@ -587,14 +587,14 @@ def test_image_badness(sim_image1):
         session.commit()
 
         # a manual way to propagate bitflags downstream
-        sim_image1.exposure.update_downstream_badness(session)  # make sure the downstreams get the new badness
+        sim_image1.exposure.update_downstream_badness(session=session)  # make sure the downstreams get the new badness
         session.commit()
         assert sim_image1.bitflag == 2 ** 5 + 2 ** 3 + 2 ** 1  # saturation bit is 3
         assert sim_image1.badness == 'banding, saturation, bright sky'
 
         # adding the same keyword on the exposure and the image makes no difference
         sim_image1.exposure.badness = 'Banding'
-        sim_image1.exposure.update_downstream_badness(session)  # make sure the downstreams get the new badness
+        sim_image1.exposure.update_downstream_badness(session=session)  # make sure the downstreams get the new badness
         session.commit()
         assert sim_image1.bitflag == 2 ** 5 + 2 ** 1
         assert sim_image1.badness == 'banding, bright sky'
@@ -642,7 +642,7 @@ def test_multiple_images_badness(
 
             # note that this image is not directly bad, but the exposure has banding
             sim_image3.exposure.badness = 'banding'
-            sim_image3.exposure.update_downstream_badness(session)
+            sim_image3.exposure.update_downstream_badness(session=session)
             session.commit()
 
             assert sim_image3.badness == 'banding'
@@ -761,7 +761,7 @@ def test_multiple_images_badness(
             # try to add some badness to one of the underlying exposures
             sim_image1.exposure.badness = 'shaking'
             session.add(sim_image1)
-            sim_image1.exposure.update_downstream_badness(session)
+            sim_image1.exposure.update_downstream_badness(session=session)
             session.commit()
 
             assert 'shaking' in sim_image1.badness
diff --git a/tests/models/test_measurements.py b/tests/models/test_measurements.py
index ea2912d2..430598ab 100644
--- a/tests/models/test_measurements.py
+++ b/tests/models/test_measurements.py
@@ -73,8 +73,39 @@ def test_measurements_attributes(measurer, ptf_datastore):
     # TODO: add test for limiting magnitude (issue #143)
 
 
+@pytest.mark.skip(reason="This test fails on GA but not locally, see issue #306")
+# @pytest.mark.flaky(max_runs=3)
 def test_filtering_measurements(ptf_datastore):
+    # printout the list of relevant environmental variables:
+    import os
+    print("SeeChange environment variables:")
+    for key in [
+        'INTERACTIVE',
+        'LIMIT_CACHE_USAGE',
+        'SKIP_NOIRLAB_DOWNLOADS',
+        'RUN_SLOW_TESTS',
+        'SEECHANGE_TRACEMALLOC',
+    ]:
+        print(f'{key}: {os.getenv(key)}')
+
     measurements = ptf_datastore.measurements
+    from pprint import pprint
+    print('measurements: ')
+    pprint(measurements)
+
+    if hasattr(ptf_datastore, 'all_measurements'):
+        idx = [m.cutouts.index_in_sources for m in measurements]
+        chosen = np.array(ptf_datastore.all_measurements)[idx]
+        pprint([(m, m.is_bad, m.cutouts.sub_nandata[12, 12]) for m in chosen])
+
+    print(f'new image values: {ptf_datastore.image.data[250, 240:250]}')
+    print(f'ref_image values: {ptf_datastore.ref_image.data[250, 240:250]}')
+    print(f'sub_image values: {ptf_datastore.sub_image.data[250, 240:250]}')
+
+    print(f'number of images in ref image: {len(ptf_datastore.ref_image.upstream_images)}')
+    for i, im in enumerate(ptf_datastore.ref_image.upstream_images):
+        print(f'upstream image {i}: {im.data[250, 240:250]}')
+
     m = measurements[0]  # grab the first one as an example
 
     # test that we can filter on some measurements properties
@@ -170,6 +201,7 @@ def test_measurements_cannot_be_saved_twice(ptf_datastore):
                 session.delete(m2)
                 session.commit()
 
+
 def test_threshold_flagging(ptf_datastore, measurer):
 
     measurements = ptf_datastore.measurements
@@ -209,6 +241,7 @@ def test_threshold_flagging(ptf_datastore, measurer):
     m.disqualifier_scores['negatives'] = 0.9 # a value that would fail both (earlier)
     assert measurer.compare_measurement_to_thresholds(m) == "delete"
 
+
 def test_deletion_thresh_is_non_critical(ptf_datastore, measurer):
 
     # hard code in the thresholds to ensure no problems arise
@@ -240,6 +273,7 @@ def test_deletion_thresh_is_non_critical(ptf_datastore, measurer):
 
     assert m1.provenance.id == m2.provenance.id
 
+
 def test_measurements_forced_photometry(ptf_datastore):
     offset_max = 2.0
     for m in ptf_datastore.measurements:
diff --git a/tests/models/test_objects.py b/tests/models/test_objects.py
index 2a8056fa..f52807d7 100644
--- a/tests/models/test_objects.py
+++ b/tests/models/test_objects.py
@@ -29,7 +29,7 @@ def test_object_creation():
         assert re.match(r'\w+\d{4}\w+', obj2.name)
 
 
-@pytest.mark.flaky(max_runs=3)
+@pytest.mark.flaky(max_runs=5)
 def test_lightcurves_from_measurements(sim_lightcurves):
     for lc in sim_lightcurves:
         expected_flux = []
@@ -46,7 +46,7 @@ def test_lightcurves_from_measurements(sim_lightcurves):
             assert measured_flux[i] == pytest.approx(expected_flux[i], abs=expected_error[i] * 3)
 
 
-@pytest.mark.flaky(max_runs=3)
+@pytest.mark.flaky(max_runs=5)
 def test_filtering_measurements_on_object(sim_lightcurves):
     assert len(sim_lightcurves) > 0
     assert len(sim_lightcurves[0]) > 3
@@ -214,6 +214,7 @@ def test_filtering_measurements_on_object(sim_lightcurves):
         found = obj.get_measurements_list(prov_hash_list=[prov.id, measurements[0].provenance.id])
         assert set([m.id for m in found]) == set(new_id_list)
 
+
 def test_separate_good_and_bad_objects(measurer, ptf_datastore):
     measurements = ptf_datastore.measurements
     m = measurements[0]  # grab the first one as an example
diff --git a/tests/models/test_provenance.py b/tests/models/test_provenance.py
index 85fb3817..615c3318 100644
--- a/tests/models/test_provenance.py
+++ b/tests/models/test_provenance.py
@@ -169,6 +169,8 @@ def test_unique_provenance_hash(code_version):
                 session.add(p2)
                 session.commit()
             assert 'duplicate key value violates unique constraint "pk_provenances"' in str(e)
+            session.rollback()
+            session.refresh(code_version)
 
     finally:
         if 'pid' in locals():
diff --git a/tests/models/test_ptf.py b/tests/models/test_ptf.py
index 83019340..28a8661b 100644
--- a/tests/models/test_ptf.py
+++ b/tests/models/test_ptf.py
@@ -3,6 +3,8 @@
 from models.source_list import SourceList
 from models.world_coordinates import WorldCoordinates
 from models.zero_point import ZeroPoint
+from models.cutouts import Cutouts
+from models.measurements import Measurements
 
 
 def test_get_ptf_exposure(ptf_exposure):
@@ -20,6 +22,10 @@ def test_ptf_datastore(ptf_datastore):
     assert isinstance(ptf_datastore.sources, SourceList)
     assert isinstance(ptf_datastore.wcs, WorldCoordinates)
     assert isinstance(ptf_datastore.zp, ZeroPoint)
+    assert isinstance(ptf_datastore.sub_image, Image)
+    assert isinstance(ptf_datastore.detections, SourceList)
+    assert all([isinstance(c, Cutouts) for c in ptf_datastore.cutouts])
+    assert all([isinstance(m, Measurements) for m in ptf_datastore.measurements])
 
     # using that bad row of pixels from the mask image
     assert all(ptf_datastore.image.flags[0:120, 94] > 0)
diff --git a/tests/models/test_reports.py b/tests/models/test_reports.py
index bb052e41..d395ec80 100644
--- a/tests/models/test_reports.py
+++ b/tests/models/test_reports.py
@@ -29,17 +29,13 @@ def test_report_bitflags(decam_exposure, decam_reference, decam_default_calibrat
     assert report.progress_steps_bitflag == 2 ** 1 + 2 ** 2
     assert report.progress_steps == 'preprocessing, extraction'
 
-    report.append_progress('photo_cal')
-    assert report.progress_steps_bitflag == 2 ** 1 + 2 ** 2 + 2 ** 4
-    assert report.progress_steps == 'preprocessing, extraction, photo_cal'
-
     report.append_progress('preprocessing')  # appending it again makes no difference
-    assert report.progress_steps_bitflag == 2 ** 1 + 2 ** 2 + 2 ** 4
-    assert report.progress_steps == 'preprocessing, extraction, photo_cal'
+    assert report.progress_steps_bitflag == 2 ** 1 + 2 ** 2
+    assert report.progress_steps == 'preprocessing, extraction'
 
     report.append_progress('subtraction, cutting')  # append two at a time
-    assert report.progress_steps_bitflag == 2 ** 1 + 2 ** 2 + 2 ** 4 + 2 ** 5 + 2 ** 7
-    assert report.progress_steps == 'preprocessing, extraction, photo_cal, subtraction, cutting'
+    assert report.progress_steps_bitflag == 2 ** 1 + 2 ** 2 + 2 ** 5 + 2 ** 7
+    assert report.progress_steps == 'preprocessing, extraction, subtraction, cutting'
 
     # test that the products exist flag is working
     assert report.products_exist_bitflag == 0
@@ -101,8 +97,8 @@ def test_measure_runtime_memory(decam_exposure, decam_reference, pipeline_for_te
 
     assert p.preprocessor.has_recalculated
     assert p.extractor.has_recalculated
-    assert p.astro_cal.has_recalculated
-    assert p.photo_cal.has_recalculated
+    assert p.astrometor.has_recalculated
+    assert p.photometor.has_recalculated
     assert p.subtractor.has_recalculated
     assert p.detector.has_recalculated
     assert p.cutter.has_recalculated
@@ -110,7 +106,7 @@ def test_measure_runtime_memory(decam_exposure, decam_reference, pipeline_for_te
 
     measured_time = 0
     peak_memory = 0
-    for step in PROCESS_OBJECTS.keys():  # also make sure all the keys are present in both dictionaries
+    for step in ds.runtimes.keys():  # also make sure all the keys are present in both dictionaries
         measured_time += ds.runtimes[step]
         if parse_bool(os.getenv('SEECHANGE_TRACEMALLOC')):
             peak_memory = max(peak_memory, ds.memory_usages[step])
@@ -133,7 +129,7 @@ def test_measure_runtime_memory(decam_exposure, decam_reference, pipeline_for_te
         assert rep.success
         assert rep.process_runtime == ds.runtimes
         assert rep.process_memory == ds.memory_usages
-        # 'preprocessing, extraction, astro_cal, photo_cal, subtraction, detection, cutting, measuring'
+        # 'preprocessing, extraction, subtraction, detection, cutting, measuring'
         assert rep.progress_steps == ', '.join(PROCESS_OBJECTS.keys())
         assert rep.products_exist == 'image, sources, psf, wcs, zp, sub_image, detections, cutouts, measurements'
         assert rep.products_committed == ''  # we don't save the data store objects at any point?
diff --git a/tests/models/test_source_list.py b/tests/models/test_source_list.py
index 7a72dbbb..46bafc5c 100644
--- a/tests/models/test_source_list.py
+++ b/tests/models/test_source_list.py
@@ -33,7 +33,7 @@ def test_source_list_bitflag(sim_sources):
         # now add a badness to the image and exposure
         sim_sources.image.badness = 'Saturation'
         sim_sources.image.exposure.badness = 'Banding'
-        sim_sources.image.exposure.update_downstream_badness(session)
+        sim_sources.image.exposure.update_downstream_badness(session=session)
         session.add(sim_sources.image)
         session.commit()
 
@@ -71,7 +71,7 @@ def test_source_list_bitflag(sim_sources):
 
         # removing the badness from the exposure is updated directly to the source list
         sim_sources.image.exposure.bitflag = 0
-        sim_sources.image.exposure.update_downstream_badness(session)
+        sim_sources.image.exposure.update_downstream_badness(session=session)
         session.add(sim_sources.image)
         session.commit()
 
diff --git a/tests/pipeline/test_coaddition.py b/tests/pipeline/test_coaddition.py
index 0cbe1b36..5cbb194d 100644
--- a/tests/pipeline/test_coaddition.py
+++ b/tests/pipeline/test_coaddition.py
@@ -20,7 +20,6 @@
 from pipeline.astro_cal import AstroCalibrator
 from pipeline.photo_cal import PhotCalibrator
 
-from util.logger import SCLogger
 
 def estimate_psf_width(data, sz=15, upsampling=25):
     """Extract a bright star and estimate its FWHM.
@@ -300,7 +299,7 @@ def test_coaddition_run(coadder, ptf_reference_images, ptf_aligned_images):
     assert ref_image.instrument == 'PTF'
     assert ref_image.telescope == 'P48'
     assert ref_image.filter == 'R'
-    assert ref_image.section_id == '11'
+    assert str(ref_image.section_id) == '11'
 
     assert isinstance(ref_image.info, dict)
     assert isinstance(ref_image.header, fits.Header)
@@ -343,10 +342,10 @@ def test_coaddition_pipeline_inputs(ptf_reference_images):
     assert pipe.coadder.pars.method == 'zogy'
     assert isinstance(pipe.extractor, Detector)
     assert pipe.extractor.pars.threshold == 3.0
-    assert isinstance(pipe.astro_cal, AstroCalibrator)
-    assert pipe.astro_cal.pars.max_catalog_mag == [22.0]
-    assert isinstance(pipe.photo_cal, PhotCalibrator)
-    assert pipe.photo_cal.pars.max_catalog_mag == [22.0]
+    assert isinstance(pipe.astrometor, AstroCalibrator)
+    assert pipe.astrometor.pars.max_catalog_mag == [22.0]
+    assert isinstance(pipe.photometor, PhotCalibrator)
+    assert pipe.photometor.pars.max_catalog_mag == [22.0]
 
     # make a new pipeline with modified parameters
     pipe = CoaddPipeline(pipeline={'date_range': 5}, coaddition={'method': 'naive'})
@@ -368,7 +367,7 @@ def test_coaddition_pipeline_inputs(ptf_reference_images):
         instrument="PTF",
         filter="R",
         section_id="11",
-        provenance_ids='5F5TAUCJJEXKX6I5H4CJ',
+        provenance_ids=ptf_reference_images[0].provenance_id,
     )
 
     # without giving a start/end time, all these images will not be selected!
@@ -380,7 +379,7 @@ def test_coaddition_pipeline_inputs(ptf_reference_images):
         instrument="PTF",
         filter="R",
         section_id="11",
-        provenance_ids='5F5TAUCJJEXKX6I5H4CJ',
+        provenance_ids=ptf_reference_images[0].provenance_id,
         start_time='2000-01-01',
         end_time='2007-01-01',
     )
@@ -392,7 +391,7 @@ def test_coaddition_pipeline_inputs(ptf_reference_images):
         instrument="PTF",
         filter="R",
         section_id="11",
-        provenance_ids='5F5TAUCJJEXKX6I5H4CJ',
+        provenance_ids=ptf_reference_images[0].provenance_id,
         start_time='2000-01-01',
     )
     im_ids = set([im.id for im in pipe.images])
@@ -412,7 +411,7 @@ def test_coaddition_pipeline_inputs(ptf_reference_images):
         instrument="PTF",
         filter="R",
         section_id="11",
-        provenance_ids='5F5TAUCJJEXKX6I5H4CJ',
+        provenance_ids=ptf_reference_images[0].provenance_id,
         start_time='2000-01-01',
     )
 
@@ -436,7 +435,7 @@ def test_coaddition_pipeline_outputs(ptf_reference_images, ptf_aligned_images):
         assert coadd_image.instrument == 'PTF'
         assert coadd_image.telescope == 'P48'
         assert coadd_image.filter == 'R'
-        assert coadd_image.section_id == '11'
+        assert str(coadd_image.section_id) == '11'
         assert coadd_image.start_mjd == min([im.start_mjd for im in ptf_reference_images])
         assert coadd_image.end_mjd == max([im.end_mjd for im in ptf_reference_images])
         assert coadd_image.provenance_id is not None
@@ -467,8 +466,8 @@ def test_coaddition_pipeline_outputs(ptf_reference_images, ptf_aligned_images):
         # zogy background noise is normalized by construction
         assert bkg_zogy == pytest.approx(1.0, abs=0.1)
 
-        # S/N should be sqrt(N) better
-        assert snr_zogy == pytest.approx(mean_snr * np.sqrt(len(ptf_reference_images)), rel=0.1)
+        # S/N should be sqrt(N) better # TODO: why is the zogy S/N 15% better than expected??
+        assert snr_zogy == pytest.approx(mean_snr * np.sqrt(len(ptf_reference_images)), rel=0.2)
 
     finally:
         if 'coadd_image' in locals():
diff --git a/tests/pipeline/test_detection.py b/tests/pipeline/test_detection.py
index ed6fcaaf..2965c86f 100644
--- a/tests/pipeline/test_detection.py
+++ b/tests/pipeline/test_detection.py
@@ -69,6 +69,7 @@ def make_template_bank(imsize=15, psf_sigma=1.0):
 
 def test_detection_ptf_supernova(detector, ptf_subtraction1, blocking_plots, cache_dir):
     ds = detector.run(ptf_subtraction1)
+
     try:
         assert ds.detections is not None
         assert ds.detections.num_sources > 0
diff --git a/tests/pipeline/test_extraction.py b/tests/pipeline/test_extraction.py
index 3fa443bd..bfd35413 100644
--- a/tests/pipeline/test_extraction.py
+++ b/tests/pipeline/test_extraction.py
@@ -384,12 +384,12 @@ def test_warnings_and_exceptions(decam_datastore, extractor):
     with pytest.warns(UserWarning) as record:
         extractor.run(decam_datastore)
     assert len(record) > 0
-    assert any("Warning injected by pipeline parameters in process 'extraction'." in str(w.message) for w in record)
+    assert any("Warning injected by pipeline parameters in process 'detection'." in str(w.message) for w in record)
 
     extractor.pars.inject_warnings = 0
     extractor.pars.inject_exceptions = 1
     with pytest.raises(Exception) as excinfo:
         ds = extractor.run(decam_datastore)
         ds.reraise()
-    assert "Exception injected by pipeline parameters in process 'extraction'." in str(excinfo.value)
+    assert "Exception injected by pipeline parameters in process 'detection'." in str(excinfo.value)
     ds.read_exception()
\ No newline at end of file
diff --git a/tests/pipeline/test_measuring.py b/tests/pipeline/test_measuring.py
index 620ccb0b..6d529548 100644
--- a/tests/pipeline/test_measuring.py
+++ b/tests/pipeline/test_measuring.py
@@ -248,7 +248,7 @@ def test_propagate_badness(decam_datastore):
         # find the index of the cutout that corresponds to the measurement
         idx = [i for i, c in enumerate(ds.cutouts) if c.id == ds.measurements[0].cutouts_id][0]
         ds.cutouts[idx].badness = 'cosmic ray'
-        ds.cutouts[idx].update_downstream_badness(session)
+        ds.cutouts[idx].update_downstream_badness(session=session)
         m = session.merge(ds.measurements[0])
 
         assert m.badness == 'cosmic ray'  # note that this does not change disqualifier_scores!
diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py
index 0dd25b9b..1d94b996 100644
--- a/tests/pipeline/test_pipeline.py
+++ b/tests/pipeline/test_pipeline.py
@@ -152,39 +152,54 @@ def test_parameters( test_config ):
     # Verify that we can override from the yaml config file
     pipeline = Pipeline()
     assert not pipeline.preprocessor.pars['use_sky_subtraction']
-    assert pipeline.astro_cal.pars['cross_match_catalog'] == 'gaia_dr3'
-    assert pipeline.astro_cal.pars['catalog'] == 'gaia_dr3'
+    assert pipeline.astrometor.pars['cross_match_catalog'] == 'gaia_dr3'
+    assert pipeline.astrometor.pars['catalog'] == 'gaia_dr3'
     assert pipeline.subtractor.pars['method'] == 'zogy'
 
-    # Verify that manual override works for all parts of pipeline
-    overrides = { 'preprocessing': { 'steps': [ 'overscan', 'linearity'] },
-                  # 'extraction': # Currently has no parameters defined
-                  'astro_cal': { 'cross_match_catalog': 'override' },
-                  'photo_cal': { 'cross_match_catalog': 'override' },
-                  'subtraction': { 'method': 'override' },
-                  'detection': { 'threshold': 3.14 },
-                  'cutting': { 'cutout_size': 666 },
-                  'measuring': { 'chosen_aperture': 1 }
-                 }
-    pipelinemodule = { 'preprocessing': 'preprocessor',
-                       'subtraction': 'subtractor',
-                       'detection': 'detector',
-                       'cutting': 'cutter',
-                       'measuring': 'measurer'
-                      }
-
     # TODO: this is based on a temporary "example_pipeline_parameter" that will be removed later
     pipeline = Pipeline( pipeline={ 'example_pipeline_parameter': -999 } )
     assert pipeline.pars['example_pipeline_parameter'] == -999
 
+    # Verify that manual override works for all parts of pipeline
+    overrides = {
+        'preprocessing': { 'steps': [ 'overscan', 'linearity'] },
+        'extraction': {
+            'sources': {'threshold': 3.14 },
+            'wcs': {'cross_match_catalog': 'override'},
+            'zp': {'cross_match_catalog': 'override'},
+        },
+        'subtraction': { 'method': 'override' },
+        'detection': { 'threshold': 3.14 },
+        'cutting': { 'cutout_size': 666 },
+        'measuring': { 'chosen_aperture': 1 }
+    }
+    pipelinemodule = {
+        'preprocessing': 'preprocessor',
+        'extraction': 'extractor',
+        'astro_cal': 'astrometor',
+        'photo_cal': 'photometor',
+        'subtraction': 'subtractor',
+        'detection': 'detector',
+        'cutting': 'cutter',
+        'measuring': 'measurer'
+    }
+
+    def check_override( new_values_dict, pars ):
+        for key, value in new_values_dict.items():
+            if pars[key] != value:
+                return False
+        return True
+
     pipeline = Pipeline( **overrides )
-    for module, subst in overrides.items():
-        if module in pipelinemodule:
-            pipelinemod = getattr( pipeline, pipelinemodule[module] )
-        else:
-            pipelinemod = getattr( pipeline, module )
-        for key, val in subst.items():
-            assert pipelinemod.pars[key] == val
+
+    assert check_override(overrides['preprocessing'], pipeline.preprocessor.pars)
+    assert check_override(overrides['extraction']['sources'], pipeline.extractor.pars)
+    assert check_override(overrides['extraction']['wcs'], pipeline.astrometor.pars)
+    assert check_override(overrides['extraction']['zp'], pipeline.photometor.pars)
+    assert check_override(overrides['subtraction'], pipeline.subtractor.pars)
+    assert check_override(overrides['detection'], pipeline.detector.pars)
+    assert check_override(overrides['cutting'], pipeline.cutter.pars)
+    assert check_override(overrides['measuring'], pipeline.measurer.pars)
 
 
 def test_data_flow(decam_exposure, decam_reference, decam_default_calibrators, archive):
@@ -210,7 +225,7 @@ def test_data_flow(decam_exposure, decam_reference, decam_default_calibrators, a
             provs = session.scalars(sa.select(Provenance)).all()
             assert len(provs) > 0
             prov_processes = [p.process for p in provs]
-            expected_processes = ['preprocessing', 'extraction', 'astro_cal', 'photo_cal', 'subtraction', 'detection']
+            expected_processes = ['preprocessing', 'extraction', 'subtraction', 'detection', 'cutting', 'measuring']
             for process in expected_processes:
                 assert process in prov_processes
 
@@ -298,8 +313,8 @@ def test_bitflag_propagation(decam_exposure, decam_reference, decam_default_cali
         ds.cutouts = None
         ds.measurements = None
 
-        ds.sources._bitflag = 2**17  # bitflag 2**17 is 'many sources'
-        desired_bitflag = 2**1 + 2**17 # bitflag for 'banding' and 'many sources'
+        ds.sources._bitflag = 2 ** 17  # bitflag 2**17 is 'many sources'
+        desired_bitflag = 2 ** 1 + 2 ** 17  # bitflag for 'banding' and 'many sources'
         ds = p.run(ds)
 
         assert ds.sources.bitflag == desired_bitflag
@@ -309,7 +324,7 @@ def test_bitflag_propagation(decam_exposure, decam_reference, decam_default_cali
         assert ds.detections._upstream_bitflag == desired_bitflag
         for cutout in ds.cutouts:
             assert cutout._upstream_bitflag == desired_bitflag
-        assert ds.image.bitflag == 2 # not in the downstream of sources
+        assert ds.image.bitflag == 2  # not in the downstream of sources
 
         # test part 3: test update_downstream_badness() function by adding and removing flags
         # and observing propagation
@@ -320,17 +335,18 @@ def test_bitflag_propagation(decam_exposure, decam_reference, decam_default_cali
             ds.image = session.merge(ds.image)
 
             # add a bitflag and check that it appears in downstreams
-            ds.image._bitflag = 16  # 16=2**4 is the bitflag for 'bad subtraction'
+
+            ds.image._bitflag = 2 ** 4  # bitflag for 'bad subtraction'
             session.add(ds.image)
             session.commit()
-            ds.image.exposure.update_downstream_badness(session)
+            ds.image.exposure.update_downstream_badness(session=session)
             session.commit()
 
             desired_bitflag = 2 ** 1 + 2 ** 4 + 2 ** 17  # 'banding' 'bad subtraction' 'many sources'
             assert ds.exposure.bitflag == 2 ** 1
             assert ds.image.bitflag == 2 ** 1 + 2 ** 4  # 'banding' and 'bad subtraction'
             assert ds.sources.bitflag == desired_bitflag
-            assert ds.psf.bitflag == 2 ** 1 + 2 ** 4  # pending psf re-structure, only downstream of image
+            assert ds.psf.bitflag == 2 ** 1 + 2 ** 4
             assert ds.wcs.bitflag == desired_bitflag
             assert ds.zp.bitflag == desired_bitflag
             assert ds.sub_image.bitflag == desired_bitflag
@@ -341,13 +357,13 @@ def test_bitflag_propagation(decam_exposure, decam_reference, decam_default_cali
             # remove the bitflag and check that it disappears in downstreams
             ds.image._bitflag = 0  # remove 'bad subtraction'
             session.commit()
-            ds.image.exposure.update_downstream_badness(session)
+            ds.image.exposure.update_downstream_badness(session=session)
             session.commit()
             desired_bitflag = 2 ** 1 + 2 ** 17  # 'banding' 'many sources'
             assert ds.exposure.bitflag == 2 ** 1
             assert ds.image.bitflag == 2 ** 1  # just 'banding' left on image
             assert ds.sources.bitflag == desired_bitflag
-            assert ds.psf.bitflag == 2 ** 1  # pending psf re-structure, only downstream of image
+            assert ds.psf.bitflag == 2 ** 1
             assert ds.wcs.bitflag == desired_bitflag
             assert ds.zp.bitflag == desired_bitflag
             assert ds.sub_image.bitflag == desired_bitflag
@@ -389,8 +405,8 @@ def test_get_upstreams_and_downstreams(decam_exposure, decam_reference, decam_de
             assert [upstream.id for upstream in ds.image.get_upstreams(session)] == [ds.exposure.id]
             assert [upstream.id for upstream in ds.sources.get_upstreams(session)] == [ds.image.id]
             assert [upstream.id for upstream in ds.wcs.get_upstreams(session)] == [ds.sources.id]
-            assert [upstream.id for upstream in ds.psf.get_upstreams(session)] == [ds.image.id] # until PSF upstreams settled
-            assert [upstream.id for upstream in ds.zp.get_upstreams(session)] == [ds.sources.id, ds.wcs.id]
+            assert [upstream.id for upstream in ds.psf.get_upstreams(session)] == [ds.image.id]
+            assert [upstream.id for upstream in ds.zp.get_upstreams(session)] == [ds.sources.id]
             assert [upstream.id for upstream in ds.sub_image.get_upstreams(session)] == [ref.image.id,
                                                                                   ref.image.sources.id,
                                                                                   ref.image.psf.id,
@@ -413,14 +429,14 @@ def test_get_upstreams_and_downstreams(decam_exposure, decam_reference, decam_de
 
             # test get_downstreams
             assert [downstream.id for downstream in ds.exposure.get_downstreams(session)] == [ds.image.id]
-            assert [downstream.id for downstream in ds.image.get_downstreams(session)] == [ds.psf.id,
+            assert set([downstream.id for downstream in ds.image.get_downstreams(session)]) == set([ds.psf.id,
                                                                                     ds.sources.id,
                                                                                     ds.wcs.id,
                                                                                     ds.zp.id,
-                                                                                    ds.sub_image.id]
-            assert [downstream.id for downstream in ds.sources.get_downstreams(session)] == [ds.wcs.id, ds.zp.id, ds.sub_image.id]
-            assert [downstream.id for downstream in ds.psf.get_downstreams(session)] == [] # until PSF downstreams settled
-            assert [downstream.id for downstream in ds.wcs.get_downstreams(session)] == [ds.zp.id, ds.sub_image.id]
+                                                                                    ds.sub_image.id])
+            assert [downstream.id for downstream in ds.sources.get_downstreams(session)] == [ds.sub_image.id]
+            assert [downstream.id for downstream in ds.psf.get_downstreams(session)] == [ds.sub_image.id]
+            assert [downstream.id for downstream in ds.wcs.get_downstreams(session)] == [ds.sub_image.id]
             assert [downstream.id for downstream in ds.zp.get_downstreams(session)] == [ds.sub_image.id]
             assert [downstream.id for downstream in ds.sub_image.get_downstreams(session)] == [ds.detections.id]
             assert np.all(np.isin([downstream.id for downstream in ds.detections.get_downstreams(session)], cutout_ids))
@@ -432,7 +448,6 @@ def test_get_upstreams_and_downstreams(decam_exposure, decam_reference, decam_de
             for measurement in ds.measurements:
                 assert [downstream.id for downstream in measurement.get_downstreams(session)] == []
 
-
     finally:
         if 'ds' in locals():
             ds.delete_everything()
@@ -502,8 +517,8 @@ def test_provenance_tree(pipeline_for_tests, decam_exposure, decam_datastore, de
     assert ds.image.provenance_id == provs['preprocessing'].id
     assert ds.sources.provenance_id == provs['extraction'].id
     assert ds.psf.provenance_id == provs['extraction'].id
-    assert ds.wcs.provenance_id == provs['astro_cal'].id
-    assert ds.zp.provenance_id == provs['photo_cal'].id
+    assert ds.wcs.provenance_id == provs['extraction'].id
+    assert ds.zp.provenance_id == provs['extraction'].id
     assert ds.sub_image.provenance_id == provs['subtraction'].id
     assert ds.detections.provenance_id == provs['detection'].id
     assert ds.cutouts[0].provenance_id == provs['cutting'].id
@@ -522,48 +537,74 @@ def test_provenance_tree(pipeline_for_tests, decam_exposure, decam_datastore, de
 def test_inject_warnings_errors(decam_datastore, decam_reference, pipeline_for_tests):
     from pipeline.top_level import PROCESS_OBJECTS
     p = pipeline_for_tests
-    for process, obj in PROCESS_OBJECTS.items():
-        # first reset all warnings and errors
-        for _, obj2 in PROCESS_OBJECTS.items():
-            getattr(p, obj2).pars.inject_exceptions = False
-            getattr(p, obj2).pars.inject_warnings = False
 
-        # set the warning:
-        getattr(p, obj).pars.inject_warnings = True
+    obj_to_process_name = {
+        'preprocessor': 'preprocessing',
+        'extractor': 'detection',
+        'astrometor': 'astro_cal',
+        'photometor': 'photo_cal',
+        'subtractor': 'subtraction',
+        'detector': 'detection',
+        'cutter': 'cutting',
+        'measurer': 'measuring',
+    }
+    for process, objects in PROCESS_OBJECTS.items():
+        if isinstance(objects, str):
+            objects = [objects]
+        elif isinstance(objects, dict):
+            objects = list(set(objects.values()))  # e.g., "extractor", "astrometor", "photometor"
 
-        # run the pipeline
-        ds = p.run(decam_datastore)
-        expected = f"{process}: <class 'UserWarning'> Warning injected by pipeline parameters in process '{process}'"
-        assert expected in ds.report.warnings
-
-        # these are used to find the report later on
-        exp_id = ds.exposure_id
-        sec_id = ds.section_id
-        prov_id = ds.report.provenance_id
-
-        # set the error instead
-        getattr(p, obj).pars.inject_warnings = False
-        getattr(p, obj).pars.inject_exceptions = True
-        # run the pipeline again, this time with an exception
-
-        with pytest.raises(RuntimeError, match=f"Exception injected by pipeline parameters in process '{process}'"):
+        # first reset all warnings and errors
+        for obj in objects:
+            for _, objects2 in PROCESS_OBJECTS.items():
+                if isinstance(objects2, str):
+                    objects2 = [objects2]
+                elif isinstance(objects2, dict):
+                    objects2 = list(set(objects2.values()))  # e.g., "extractor", "astrometor", "photometor"
+                for obj2 in objects2:
+                    getattr(p, obj2).pars.inject_exceptions = False
+                    getattr(p, obj2).pars.inject_warnings = False
+
+            # set the warning:
+            getattr(p, obj).pars.inject_warnings = True
+
+            # run the pipeline
             ds = p.run(decam_datastore)
-
-        # fetch the report object
-        with SmartSession() as session:
-            reports = session.scalars(
-                sa.select(Report).where(
-                    Report.exposure_id == exp_id,
-                    Report.section_id == sec_id,
-                    Report.provenance_id == prov_id
-                ).order_by(Report.start_time.desc())
-            ).all()
-            report = reports[0]  # the last report is the one we just generated
-            assert len(reports) - 1 == report.num_prev_reports
-            assert not report.success
-            assert report.error_step == process
-            assert report.error_type == 'RuntimeError'
-            assert 'Exception injected by pipeline parameters' in report.error_message
+            expected = (f"{process}: <class 'UserWarning'> Warning injected by pipeline parameters "
+                        f"in process '{obj_to_process_name[obj]}'")
+            assert expected in ds.report.warnings
+
+            # these are used to find the report later on
+            exp_id = ds.exposure_id
+            sec_id = ds.section_id
+            prov_id = ds.report.provenance_id
+
+            # set the error instead
+            getattr(p, obj).pars.inject_warnings = False
+            getattr(p, obj).pars.inject_exceptions = True
+            # run the pipeline again, this time with an exception
+
+            with pytest.raises(
+                    RuntimeError,
+                    match=f"Exception injected by pipeline parameters in process '{obj_to_process_name[obj]}'"
+            ):
+                ds = p.run(decam_datastore)
+
+            # fetch the report object
+            with SmartSession() as session:
+                reports = session.scalars(
+                    sa.select(Report).where(
+                        Report.exposure_id == exp_id,
+                        Report.section_id == sec_id,
+                        Report.provenance_id == prov_id
+                    ).order_by(Report.start_time.desc())
+                ).all()
+                report = reports[0]  # the last report is the one we just generated
+                assert len(reports) - 1 == report.num_prev_reports
+                assert not report.success
+                assert report.error_step == process
+                assert report.error_type == 'RuntimeError'
+                assert 'Exception injected by pipeline parameters' in report.error_message
 
 
 def test_multiprocessing_make_provenances_and_exposure(decam_exposure, decam_reference, pipeline_for_tests):
diff --git a/util/util.py b/util/util.py
index 8ba0f9ea..8c68ccac 100644
--- a/util/util.py
+++ b/util/util.py
@@ -92,6 +92,7 @@ def remove_empty_folders(path, remove_root=True):
         if remove_root and not any(path.iterdir()):
             path.rmdir()
 
+
 def get_git_hash():
     """
     Get the commit hash of the current git repo.
@@ -112,6 +113,7 @@ def get_git_hash():
 
     return git_hash
 
+
 def get_latest_provenance(process_name, session=None):
     """
     Find the provenance object that fits the process_name
@@ -147,6 +149,7 @@ def get_latest_provenance(process_name, session=None):
 
     return prov
 
+
 def parse_dateobs(dateobs=None, output='datetime'):
     """
     Parse the dateobs, that can be a float, string, datetime or Time object.
@@ -194,6 +197,7 @@ def parse_dateobs(dateobs=None, output='datetime'):
     else:
         raise ValueError(f'Unknown output type {output}')
 
+
 def parse_session(*args, **kwargs):
     """
     Parse the arguments and keyword arguments to find a SmartSession or SQLAlchemy session.
@@ -235,6 +239,7 @@ def parse_session(*args, **kwargs):
 
     return args, kwargs, session
 
+
 def read_fits_image(filename, ext=0, output='data'):
     """
     Read a standard FITS file's image data and header.
@@ -284,6 +289,7 @@ def read_fits_image(filename, ext=0, output='data'):
     else:
         raise ValueError(f'Unknown output type "{output}", use "data", "header" or "both"')
 
+
 def save_fits_image_file(filename, data, header, extname=None, overwrite=True, single_file=False,
                          just_update_header=False):
     """Save a single dataset (image data, weight, flags, etc) to a FITS file.

From 9d7d9773c82389876ccfb46cbf564bfdc3039891 Mon Sep 17 00:00:00 2001
From: Guy Nir <37179063+guynir42@users.noreply.github.com>
Date: Thu, 13 Jun 2024 22:35:58 -0700
Subject: [PATCH 10/11] Preprocessing steps (#305)

Separate preprocessing steps to those already done on the exposure (in the instrument class) and those that are required in the config.
---
 .github/workflows/run-pipeline-tests-1.yml |   1 -
 default_config.yaml                        |   2 +-
 models/decam.py                            |   3 +-
 models/enums_and_bitflags.py               |  25 ++---
 models/image.py                            |  13 +++
 models/instrument.py                       |  32 +++---
 models/ptf.py                              |   4 +-
 pipeline/parameters.py                     |   2 +-
 pipeline/preprocessing.py                  | 119 ++++++++-------------
 tests/improc/test_tools.py                 |   1 +
 tests/pipeline/test_pipeline.py            |   2 -
 tests/pipeline/test_preprocessing.py       |  25 +----
 12 files changed, 102 insertions(+), 127 deletions(-)

diff --git a/.github/workflows/run-pipeline-tests-1.yml b/.github/workflows/run-pipeline-tests-1.yml
index af0b121f..38132c7e 100644
--- a/.github/workflows/run-pipeline-tests-1.yml
+++ b/.github/workflows/run-pipeline-tests-1.yml
@@ -59,6 +59,5 @@ jobs:
 
       - name: run test
         run: |
-          df -h
           shopt -s nullglob
           TEST_SUBFOLDER=$(ls tests/pipeline/test_{a..o}*.py) docker compose run runtests
diff --git a/default_config.yaml b/default_config.yaml
index dcbe863a..4f676164 100644
--- a/default_config.yaml
+++ b/default_config.yaml
@@ -79,7 +79,7 @@ catalog_gaiadr3:
 pipeline: {}
 
 preprocessing:
-  use_sky_subtraction: False
+  steps_required: [ 'overscan', 'linearity', 'flat', 'fringe' ]
 
 extraction:
   sources:
diff --git a/models/decam.py b/models/decam.py
index 31d18ca2..53430041 100644
--- a/models/decam.py
+++ b/models/decam.py
@@ -124,7 +124,8 @@ def __init__(self, **kwargs):
         # will apply kwargs to attributes, and register instrument in the INSTRUMENT_INSTANCE_CACHE
         Instrument.__init__(self, **kwargs)
 
-        self.preprocessing_steps = [ 'overscan', 'linearity', 'flat', 'fringe' ]
+        self.preprocessing_steps_available = [ 'overscan', 'linearity', 'flat', 'fringe' ]
+        self.preprocessing_steps_done = []
 
     @classmethod
     def get_section_ids(cls):
diff --git a/models/enums_and_bitflags.py b/models/enums_and_bitflags.py
index a979b657..85f62131 100644
--- a/models/enums_and_bitflags.py
+++ b/models/enums_and_bitflags.py
@@ -325,6 +325,19 @@ def string_to_bitflag(value, dictionary):
         return output
 
 
+# bitflag for image preprocessing steps that have been done
+image_preprocessing_dict = {
+    0: 'overscan',
+    1: 'zero',
+    2: 'dark',
+    3: 'linearity',
+    4: 'flat',
+    5: 'fringe',
+    6: 'illumination'
+}
+image_preprocessing_inverse = {EnumConverter.c(v):k for k, v in image_preprocessing_dict.items()}
+
+
 # these are the ways an Image or Exposure are allowed to be bad
 image_badness_dict = {
     1: 'banding',
@@ -389,18 +402,6 @@ class BadnessConverter( EnumConverter ):
     _dict_filtered = None
     _dict_inverse = None
 
-# bitflag for image preprocessing steps that have been done
-image_preprocessing_dict = {
-    0: 'overscan',
-    1: 'zero',
-    2: 'dark',
-    3: 'linearity',
-    4: 'flat',
-    5: 'fringe',
-    6: 'illumination'
-}
-image_preprocessing_inverse = {EnumConverter.c(v):k for k, v in image_preprocessing_dict.items()}
-
 # bitflag used in flag images
 flag_image_bits = {
     0: 'bad pixel',        # Bad pixel flagged by the instrument
diff --git a/models/image.py b/models/image.py
index c2b09e8c..d1efd3f1 100644
--- a/models/image.py
+++ b/models/image.py
@@ -34,6 +34,10 @@
 from models.enums_and_bitflags import (
     ImageFormatConverter,
     ImageTypeConverter,
+    string_to_bitflag,
+    bitflag_to_string,
+    image_preprocessing_dict,
+    image_preprocessing_inverse,
     image_badness_inverse,
 )
 
@@ -361,6 +365,15 @@ def mid_mjd(self):
         doc='Bitflag specifying which preprocessing steps have been completed for the image.'
     )
 
+    @property
+    def preprocessing_done(self):
+        """Return a list of the names of preprocessing steps that have been completed for this image."""
+        return bitflag_to_string(self.preproc_bitflag, image_preprocessing_dict)
+
+    @preprocessing_done.setter
+    def preprocessing_done(self, value):
+        self.preproc_bitflag = string_to_bitflag(value, image_preprocessing_inverse)
+
     astro_cal_done = sa.Column(
         sa.BOOLEAN,
         nullable=False,
diff --git a/models/instrument.py b/models/instrument.py
index 0193d1d4..dea62d94 100644
--- a/models/instrument.py
+++ b/models/instrument.py
@@ -382,15 +382,22 @@ def __init__(self, **kwargs):
         self._dateobs_for_sections = getattr(self, '_dateobs_for_sections', None)  # dateobs when sections were loaded
         self._dateobs_range_days = getattr(self, '_dateobs_range_days', 1.0)  # how many days from dateobs to reload
 
-        # List of the preprocessing steps to apply to images from this
-        # instrument, in order overscan must always be first.  The
-        # values here (in the Instrument class) are all possible values.
-        # Subclasses should redefine this with the subset that they
-        # actually need.  If a subclass has to add a new preprocessing
-        # step, then it should add that step to this list, and (if it's
-        # a step that includes a calibraiton image or datafile) to the
-        # CalibratorTypeConverter dict in enums_and_bitflags.py
-        self.preprocessing_steps = [ 'overscan', 'zero','dark', 'linearity', 'flat', 'fringe', 'illumination' ]
+        # List of the preprocessing steps that can be applied to exposures from this
+        # instrument, in order.  'overscan' must always be first.
+        # All preprocessing steps that are available for an instrument are listed under preprocessing_steps_available.
+        # Use image_preprocessing_dict, defined in the enums_and_bitflags file to see all possible values.
+        # Subclasses of Instrument should redefine this with the subset that they
+        # actually need to apply. So, if an instrument has exposures
+        # that already have overscan removed, that instrument should remove 'overscan' from this list.
+        # If a subclass has to add a new preprocessing step,
+        # then it should add that step to enum_and_bitflags.image_preprocessing_dict,
+        # and (if it's a step that includes a calibraiton image or datafile)
+        # to the CalibratorTypeConverter dict in enums_and_bitflags.
+        self.preprocessing_steps_available = ['overscan', 'zero', 'dark', 'linearity', 'flat', 'fringe', 'illumination']
+        # a list of preprocessing steps that are pre-applied to the exposure data
+        self.preprocessing_steps_done = []
+        self.preprocessing_step_skip_by_filter = {}  # e.g., {'g': ['fringe', 'illumination']} will skip those for g
+
         # nofile_steps are ones that don't have an associated file
         self.preprocessing_nofile_steps = [ 'overscan' ]
 
@@ -1457,7 +1464,7 @@ def preprocessing_calibrator_files( self, calibset, flattype, section, filter, m
         construction).
 
         """
-
+        section = str(section)
         SCLogger.debug( f'Looking for calibrators for {calibset} {section}' )
 
         if ( calibset == 'externally_supplied' ) != ( flattype == 'externally_supplied' ):
@@ -1475,7 +1482,7 @@ def preprocessing_calibrator_files( self, calibset, flattype, section, filter, m
         expdatetime = pytz.utc.localize( astropy.time.Time( mjd, format='mjd' ).datetime )
 
         with SmartSession(session) as session:
-            for calibtype in self.preprocessing_steps:
+            for calibtype in self.preprocessing_steps_available:
                 if calibtype in self.preprocessing_nofile_steps:
                     continue
 
@@ -1895,7 +1902,8 @@ def __init__(self, **kwargs):
         Instrument.__init__(self, **kwargs)
 
         # DemoInstrument doesn't know how to preprocess
-        self.preprocessing_steps = []
+        self.preprocessing_steps_available = []
+        self.preprocessing_steps_done = ['overscan', 'linearity', 'flat', 'fringe']
 
     @classmethod
     def get_section_ids(cls):
diff --git a/models/ptf.py b/models/ptf.py
index 55a8729c..00914ca0 100644
--- a/models/ptf.py
+++ b/models/ptf.py
@@ -26,7 +26,9 @@ def __init__(self, **kwargs):
         # will apply kwargs to attributes, and register instrument in the INSTRUMENT_INSTANCE_CACHE
         Instrument.__init__(self, **kwargs)
 
-        self.preprocessing_steps = []
+        # we are using preprocessed data as the exposures, so everything is already done
+        self.preprocessing_steps_available = []
+        self.preprocessing_steps_done = ['overscan', 'linearity', 'flat', 'fringe']
 
     @classmethod
     def get_section_ids(cls):
diff --git a/pipeline/parameters.py b/pipeline/parameters.py
index ef5212fd..a58c3977 100644
--- a/pipeline/parameters.py
+++ b/pipeline/parameters.py
@@ -333,7 +333,7 @@ def __setattr__(self, key, value):
             and not isinstance(value, self.__typecheck__[real_key])
         ):
             raise TypeError(
-                f'Parameter "{key}" must be of type {self.__typecheck__[real_key]}'
+                f'Parameter "{key}" must be of type {self.__typecheck__[real_key]}, got {type(value)} instead. '
             )
         super().__setattr__(real_key, value)
 
diff --git a/pipeline/preprocessing.py b/pipeline/preprocessing.py
index 1613319f..1b70999d 100644
--- a/pipeline/preprocessing.py
+++ b/pipeline/preprocessing.py
@@ -23,17 +23,16 @@ class ParsPreprocessor(Parameters):
     def __init__(self, **kwargs):
         super().__init__()
 
-        self.use_sky_subtraction = self.add_par('use_sky_subtraction', False, bool, 'Apply sky subtraction. ',
-                                                critical=True)
-        self.add_par( 'steps', None, ( list, None ), "Steps to do; don't specify, or pass None, to do all." )
-        self.add_par( 'calibset', None, ( str, None ),
-                      ( "One of the CalibratorSetConverter enum; "
-                        "the calibrator set to use.  Defaults to the instrument default" ),
-                      critical = True )
+        self.add_par( 'steps_required', [], list, "Steps that need to be done to each exposure" )
+
+        self.add_par( 'calibset', 'externally_supplied', str,
+                      "The calibrator set to use.  Choose one of the CalibratorSetConverter enum. ",
+                      critical=True )
         self.add_alias( 'calibrator_set', 'calibset' )
-        self.add_par( 'flattype', None, ( str, None ),
-                      ( "One of the FlatTypeConverter enum; defaults to the instrument default" ),
-                      critical = True )
+
+        self.add_par( 'flattype', 'externally_supplied', str,
+                      "One of the FlatTypeConverter enum. ",
+                      critical=True )
 
         self._enforce_no_new_attrs = True
 
@@ -75,10 +74,6 @@ def __init__(self, **kwargs):
         # the object did any work or just loaded from DB or datastore
         self.has_recalculated = False
 
-        # TODO : remove this if/when we actually put sky subtraction in run()
-        if self.pars.use_sky_subtraction:
-            raise NotImplementedError( "Sky subtraction in preprocessing isn't implemented." )
-
     def run( self, *args, **kwargs ):
         """Run preprocessing for a given exposure and section_identifier.
 
@@ -90,9 +85,6 @@ def run( self, *args, **kwargs ):
           - Exposure, section_identifier
         Passing just an image won't work.
 
-        kwargs can also include things that override the preprocessing
-        behavior.  (TODO: document this)
-
         Returns
         -------
         DataStore
@@ -105,9 +97,6 @@ def run( self, *args, **kwargs ):
         except Exception as e:
             return DataStore.catch_failure_to_parse(e, *args)
 
-        # This is here just for testing purposes
-        self._ds = ds  # TODO: is there a reason not to just use the output datastore?
-
         try:  # catch any exceptions and save them in the datastore
             t_start = time.perf_counter()
             if parse_bool(os.getenv('SEECHANGE_TRACEMALLOC')):
@@ -124,41 +113,21 @@ def run( self, *args, **kwargs ):
             if ( self.instrument is None ) or ( self.instrument.name != ds.exposure.instrument ):
                 self.instrument = ds.exposure.instrument_object
 
-            # The only reason these are saved in self, rather than being
-            # local variables, is so that tests can probe them
-            self._calibset = None
-            self._flattype = None
-            self._stepstodo = None
-
-            if 'calibset' in kwargs:
-                self._calibset = kwargs['calibset']
-            elif 'calibratorset' in kwargs:
-                self._calibset = kwargs['calibrator_set']
-            elif self.pars.calibset is not None:
-                self._calibset = self.pars.calibset
-            else:
-                self._calibset = cfg.value( f'{self.instrument.name}.calibratorset',
-                                            default=cfg.value( 'instrument_default.calibratorset' ) )
-
-            if 'flattype' in kwargs:
-                self._flattype = kwargs['flattype']
-            elif self.pars.flattype is not None:
-                self._flattype = self.pars.flattype
-            else:
-                self._flattype = cfg.value( f'{self.instrument.name}.flattype',
-                                            default=cfg.value( 'instrument_default.flattype' ) )
-
-            if 'steps' in kwargs:
-                self._stepstodo = [ s for s in self.instrument.preprocessing_steps if s in kwargs['steps'] ]
-            elif self.pars.steps is not None:
-                self._stepstodo = [ s for s in self.instrument.preprocessing_steps if s in self.pars.steps ]
-            else:
-                self._stepstodo = self.instrument.preprocessing_steps
-
+            # check that all required steps can be done (or have been done) by the instrument:
+            known_steps = self.instrument.preprocessing_steps_available
+            known_steps += self.instrument.preprocessing_steps_done
+            known_steps = set(known_steps)
+            needed_steps = set(self.pars.steps_required)
+            if not needed_steps.issubset(known_steps):
+                raise ValueError(
+                    f'Missing some preprocessing steps {needed_steps - known_steps} '
+                    f'for instrument {self.instrument.name}'
+                )
+            
             # Get the calibrator files
             SCLogger.debug("preprocessing: getting calibrator files")
-            preprocparam = self.instrument.preprocessing_calibrator_files( self._calibset,
-                                                                           self._flattype,
+            preprocparam = self.instrument.preprocessing_calibrator_files( self.pars.calibset,
+                                                                           self.pars.flattype,
                                                                            ds.section_id,
                                                                            ds.exposure.filter_short,
                                                                            ds.exposure.mjd,
@@ -167,18 +136,6 @@ def run( self, *args, **kwargs ):
             SCLogger.debug("preprocessing: got calibrator files")
 
             # get the provenance for this step, using the current parameters:
-            # Provenance includes not just self.pars.get_critical_pars(),
-            # but also the steps that were performed.  Reason: we may well
-            # load non-flatfielded images in the database for purposes of
-            # collecting images used for later building flats.  We will then
-            # flatfield those images.  The two images in the database must have
-            # different provenances.
-            # We also include any overrides to calibrator files, as that indicates
-            # that something individual happened here that's different from
-            # normal processing of the image.
-            # Fix this as part of issue #147
-            # provdict = dict( self.pars.get_critical_pars() )
-            # provdict['preprocessing_steps' ] = self._stepstodo
             prov = ds.get_provenance('preprocessing', self.pars.get_critical_pars(), session=session)
 
             # check if the image already exists in memory or in the database:
@@ -194,17 +151,27 @@ def run( self, *args, **kwargs ):
             if image.preproc_bitflag is None:
                 image.preproc_bitflag = 0
 
-            required_bitflag = 0
-            for step in self._stepstodo:
-                required_bitflag |= string_to_bitflag( step, image_preprocessing_inverse )
-
-            if image._data is None:  # in case we are skipping all preprocessing steps
-                image.data = image.raw_data
-
-            if image.preproc_bitflag != required_bitflag:
+            needed_steps -= set(self.instrument.preprocessing_steps_done)
+            filter_skips = self.instrument.preprocessing_step_skip_by_filter.get(ds.exposure.filter_short, [])
+            if not isinstance(filter_skips, list):
+                raise ValueError(f'Filter skips parameter for {ds.exposure.filter_short} must be a list')
+            filter_skips = set(filter_skips)
+            needed_steps -= filter_skips
+
+            if image._data is None:  # in case we skip all preprocessing steps
+                image.data = image.raw_data 
+            
+            # the image keeps track of the steps already done to it in image.preproc_bitflag,
+            # which is translated into a list of keywords when calling image.preprocessing_done
+            # this includes the things that already were applied in the exposure
+            # (i.e., the instrument's preprocessing_steps_done) but does not
+            # include the things that were skipped for this filter
+            # (i.e., the instrument's preprocessing_step_skip_by_filter)
+            already_done = set(image.preprocessing_done.split(', ') if image.preprocessing_done else [])
+            if not needed_steps.issubset(already_done):  # still things to do here
                 self.has_recalculated = True
                 # Overscan is always first (as it reshapes the image)
-                if 'overscan' in self._stepstodo:
+                if 'overscan' in needed_steps:
                     SCLogger.debug('preprocessing: overscan and trim')
                     image.data = self.instrument.overscan_and_trim( image )
                     # Update the header ra/dec calculations now that we know the real width/height
@@ -212,7 +179,7 @@ def run( self, *args, **kwargs ):
                     image.preproc_bitflag |= string_to_bitflag( 'overscan', image_preprocessing_inverse )
 
                 # Apply steps in the order expected by the instrument
-                for step in self._stepstodo:
+                for step in needed_steps:
                     if step == 'overscan':
                         continue
                     SCLogger.debug(f"preprocessing: {step}")
@@ -274,7 +241,7 @@ def run( self, *args, **kwargs ):
 
                     image.preproc_bitflag |= string_to_bitflag( step, image_preprocessing_inverse )
 
-                # Get the Instrument standard bad pixel mask for this image
+            # Get the Instrument standard bad pixel mask for this image
             if image._flags is None or image._weight is None:
                 image._flags = self.instrument.get_standard_flags_image( ds.section_id )
 
diff --git a/tests/improc/test_tools.py b/tests/improc/test_tools.py
index 23bf0e10..aefef7d7 100644
--- a/tests/improc/test_tools.py
+++ b/tests/improc/test_tools.py
@@ -1,6 +1,7 @@
 from astropy.io import fits
 from improc.tools import strip_wcs_keywords
 
+
 def test_strip_wcs_keywords():
     hdr = fits.Header()
 
diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py
index 1d94b996..df045e5b 100644
--- a/tests/pipeline/test_pipeline.py
+++ b/tests/pipeline/test_pipeline.py
@@ -151,7 +151,6 @@ def test_parameters( test_config ):
 
     # Verify that we can override from the yaml config file
     pipeline = Pipeline()
-    assert not pipeline.preprocessor.pars['use_sky_subtraction']
     assert pipeline.astrometor.pars['cross_match_catalog'] == 'gaia_dr3'
     assert pipeline.astrometor.pars['catalog'] == 'gaia_dr3'
     assert pipeline.subtractor.pars['method'] == 'zogy'
@@ -537,7 +536,6 @@ def test_provenance_tree(pipeline_for_tests, decam_exposure, decam_datastore, de
 def test_inject_warnings_errors(decam_datastore, decam_reference, pipeline_for_tests):
     from pipeline.top_level import PROCESS_OBJECTS
     p = pipeline_for_tests
-
     obj_to_process_name = {
         'preprocessor': 'preprocessing',
         'extractor': 'detection',
diff --git a/tests/pipeline/test_preprocessing.py b/tests/pipeline/test_preprocessing.py
index 7cf3b928..0095042d 100644
--- a/tests/pipeline/test_preprocessing.py
+++ b/tests/pipeline/test_preprocessing.py
@@ -26,11 +26,11 @@ def test_preprocessing(
     assert not preprocessor.has_recalculated
 
     # Check some Preprocesor internals
-    assert preprocessor._calibset == 'externally_supplied'
-    assert preprocessor._flattype == 'externally_supplied'
-    assert preprocessor._stepstodo == [ 'overscan', 'linearity', 'flat', 'fringe' ]
-    assert preprocessor._ds.exposure.filter[:1] == 'g'
-    assert preprocessor._ds.section_id == 'N1'
+    assert preprocessor.pars.calibset == 'externally_supplied'
+    assert preprocessor.pars.flattype == 'externally_supplied'
+    assert preprocessor.pars.steps_required == [ 'overscan', 'linearity', 'flat', 'fringe' ]
+    ds.exposure.filter[:1] == 'g'
+    ds.section_id == 'N1'
     assert set( preprocessor.stepfiles.keys() ) == { 'flat', 'linearity' }
 
     # Make sure that the BSCALE and BZERO keywords got stripped
@@ -89,21 +89,6 @@ def test_preprocessing(
     # TODO : other checks that preprocessing did what it was supposed to do?
     # (Look at image header once we have HISTORY adding in there.)
 
-    # Test some overriding
-    # clear these caches
-    preprocessor.instrument = None
-    preprocessor.stepfilesids = {}
-    preprocessor.stepfiles = {}
-
-    ds = preprocessor.run( decam_exposure, 'N1', steps=['overscan', 'linearity'] )
-    assert preprocessor.has_recalculated
-    assert preprocessor._calibset == 'externally_supplied'
-    assert preprocessor._flattype == 'externally_supplied'
-    assert preprocessor._stepstodo == [ 'overscan', 'linearity' ]
-    assert preprocessor._ds.exposure.filter[:1] == 'g'
-    assert preprocessor._ds.section_id == 'N1'
-    assert set( preprocessor.stepfiles.keys() ) == { 'linearity' }
-
 
 def test_warnings_and_exceptions(decam_exposure, preprocessor, decam_default_calibrators, archive):
     preprocessor.pars.inject_warnings = 1

From 8b4a87c8f0ee287087b17f17de6fd28097f57c06 Mon Sep 17 00:00:00 2001
From: Guy Nir <37179063+guynir42@users.noreply.github.com>
Date: Thu, 20 Jun 2024 23:55:55 -0700
Subject: [PATCH 11/11] Background model (#308)

Add Background as an Image data product, add backgrounding and Backgrounder as a configurable pipeline step.
---
 ...6_10_1132-a375526c8260_background_table.py |  86 ++++
 default_config.yaml                           |  19 +-
 docs/troubleshooting_sqla.md                  | 147 ++++++
 improc/alignment.py                           |  98 +++-
 improc/photometry.py                          |  37 +-
 improc/simulator.py                           |   5 +-
 models/background.py                          | 433 ++++++++++++++++++
 models/base.py                                |   4 +
 models/cutouts.py                             |   2 +-
 models/enums_and_bitflags.py                  |  58 ++-
 models/exposure.py                            |   1 +
 models/image.py                               | 106 +++--
 models/instrument.py                          |   2 +
 models/measurements.py                        |  35 +-
 models/provenance.py                          |   1 +
 models/psf.py                                 |  22 +-
 models/reference.py                           |  22 +-
 models/report.py                              |  54 ++-
 models/source_list.py                         |  42 +-
 models/world_coordinates.py                   |  27 +-
 models/zero_point.py                          |  20 +-
 pipeline/backgrounding.py                     | 169 +++++++
 pipeline/coaddition.py                        |  32 +-
 pipeline/cutting.py                           |   4 +-
 pipeline/data_store.py                        | 112 ++++-
 pipeline/detection.py                         | 161 +++++--
 pipeline/measuring.py                         |  40 +-
 pipeline/subtraction.py                       |  13 +-
 pipeline/top_level.py                         |  53 ++-
 requirements.txt                              |   2 +-
 tests/conftest.py                             |   1 -
 tests/fixtures/decam.py                       |   5 +-
 tests/fixtures/pipeline_objects.py            |  98 +++-
 tests/fixtures/ptf.py                         |  41 +-
 tests/fixtures/simulated.py                   |   2 +-
 tests/improc/test_alignment.py                |  19 +-
 tests/improc/test_zogy.py                     |   3 +-
 tests/models/test_background.py               | 102 +++++
 tests/models/test_measurements.py             |  27 +-
 tests/models/test_objects.py                  |   2 +-
 tests/models/test_provenance.py               |  66 +++
 tests/models/test_reports.py                  |  92 ++--
 tests/models/test_source_list.py              |  15 +-
 tests/pipeline/test_backgrounding.py          |  52 +++
 tests/pipeline/test_coaddition.py             |  92 ++--
 tests/pipeline/test_extraction.py             | 147 ++----
 tests/pipeline/test_measuring.py              |  51 +--
 tests/pipeline/test_photo_cal.py              |   2 +-
 tests/pipeline/test_pipeline.py               |  48 +-
 tests/pipeline/test_subtraction.py            |   6 +-
 util/util.py                                  |   3 +-
 51 files changed, 2133 insertions(+), 548 deletions(-)
 create mode 100644 alembic/versions/2024_06_10_1132-a375526c8260_background_table.py
 create mode 100644 docs/troubleshooting_sqla.md
 create mode 100644 models/background.py
 create mode 100644 pipeline/backgrounding.py
 create mode 100644 tests/models/test_background.py
 create mode 100644 tests/pipeline/test_backgrounding.py

diff --git a/alembic/versions/2024_06_10_1132-a375526c8260_background_table.py b/alembic/versions/2024_06_10_1132-a375526c8260_background_table.py
new file mode 100644
index 00000000..4b5d328f
--- /dev/null
+++ b/alembic/versions/2024_06_10_1132-a375526c8260_background_table.py
@@ -0,0 +1,86 @@
+"""background table
+
+Revision ID: a375526c8260
+Revises: a7dde2327dde
+Create Date: 2024-06-10 11:32:39.717922
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'a375526c8260'
+down_revision = 'a7dde2327dde'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('backgrounds',
+    sa.Column('_format', sa.SMALLINT(), nullable=False),
+    sa.Column('_method', sa.SMALLINT(), nullable=False),
+    sa.Column('image_id', sa.BigInteger(), nullable=False),
+    sa.Column('value', sa.Float(), nullable=False),
+    sa.Column('noise', sa.Float(), nullable=False),
+    sa.Column('provenance_id', sa.String(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('modified', sa.DateTime(), nullable=False),
+    sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
+    sa.Column('filepath_extensions', postgresql.ARRAY(sa.Text(), zero_indexes=True), nullable=True),
+    sa.Column('md5sum', sa.UUID(), nullable=True),
+    sa.Column('md5sum_extensions', postgresql.ARRAY(sa.UUID(), zero_indexes=True), nullable=True),
+    sa.Column('filepath', sa.Text(), nullable=False),
+    sa.Column('_bitflag', sa.BIGINT(), nullable=False),
+    sa.Column('description', sa.Text(), nullable=True),
+    sa.Column('_upstream_bitflag', sa.BIGINT(), nullable=False),
+    sa.ForeignKeyConstraint(['image_id'], ['images.id'], name='backgrounds_image_id_fkey', ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['provenance_id'], ['provenances.id'], name='backgrounds_provenance_id_fkey', ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index('backgrounds_image_id_provenance_index', 'backgrounds', ['image_id', 'provenance_id'], unique=True)
+    op.create_index(op.f('ix_backgrounds__bitflag'), 'backgrounds', ['_bitflag'], unique=False)
+    op.create_index(op.f('ix_backgrounds__upstream_bitflag'), 'backgrounds', ['_upstream_bitflag'], unique=False)
+    op.create_index(op.f('ix_backgrounds_created_at'), 'backgrounds', ['created_at'], unique=False)
+    op.create_index(op.f('ix_backgrounds_filepath'), 'backgrounds', ['filepath'], unique=True)
+    op.create_index(op.f('ix_backgrounds_id'), 'backgrounds', ['id'], unique=False)
+    op.create_index(op.f('ix_backgrounds_image_id'), 'backgrounds', ['image_id'], unique=False)
+    op.create_index(op.f('ix_backgrounds_noise'), 'backgrounds', ['noise'], unique=False)
+    op.create_index(op.f('ix_backgrounds_provenance_id'), 'backgrounds', ['provenance_id'], unique=False)
+    op.create_index(op.f('ix_backgrounds_value'), 'backgrounds', ['value'], unique=False)
+    op.add_column('source_lists', sa.Column('inf_aper_num', sa.SMALLINT(), nullable=True))
+    op.add_column('source_lists', sa.Column('best_aper_num', sa.SMALLINT(), nullable=True))
+    op.drop_column('source_lists', '_inf_aper_num')
+
+    op.add_column('measurements', sa.Column('bkg_mean', sa.REAL(), nullable=False))
+    op.add_column('measurements', sa.Column('bkg_std', sa.REAL(), nullable=False))
+    op.add_column('measurements', sa.Column('bkg_pix', sa.REAL(), nullable=False))
+    op.drop_column('measurements', 'background')
+    op.drop_column('measurements', 'background_err')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('measurements', sa.Column('background_err', sa.REAL(), autoincrement=False, nullable=False))
+    op.add_column('measurements', sa.Column('background', sa.REAL(), autoincrement=False, nullable=False))
+    op.drop_column('measurements', 'bkg_pix')
+    op.drop_column('measurements', 'bkg_std')
+    op.drop_column('measurements', 'bkg_mean')
+
+    op.add_column('source_lists', sa.Column('_inf_aper_num', sa.SMALLINT(), autoincrement=False, nullable=True))
+    op.drop_column('source_lists', 'best_aper_num')
+    op.drop_column('source_lists', 'inf_aper_num')
+    op.drop_index(op.f('ix_backgrounds_value'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds_provenance_id'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds_noise'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds_image_id'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds_id'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds_filepath'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds_created_at'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds__upstream_bitflag'), table_name='backgrounds')
+    op.drop_index(op.f('ix_backgrounds__bitflag'), table_name='backgrounds')
+    op.drop_index('backgrounds_image_id_provenance_index', table_name='backgrounds')
+    op.drop_table('backgrounds')
+    # ### end Alembic commands ###
diff --git a/default_config.yaml b/default_config.yaml
index 4f676164..68658624 100644
--- a/default_config.yaml
+++ b/default_config.yaml
@@ -83,10 +83,21 @@ preprocessing:
 
 extraction:
   sources:
+    method: sextractor
     measure_psf: true
+    apertures: [1.0, 2.0, 3.0, 5.0]
+    inf_aper_num: -1
+    best_aper_num: 0
+    aperunit: fwhm
+    separation_fwhm: 1.0
     threshold: 3.0
-    method: sextractor
-
+    subtraction: false
+  bg:
+    format: map
+    method: sep
+    poly_order: 1
+    sep_box_size: 128
+    sep_filt_size: 3
   wcs:
     cross_match_catalog: gaia_dr3
     solution_method: scamp
@@ -94,7 +105,6 @@ extraction:
     mag_range_catalog: 4.0
     min_catalog_stars: 50
     max_sources_to_use: [2000, 1000, 500, 200]
-
   zp:
     cross_match_catalog: gaia_dr3
     max_catalog_mag: [20.0]
@@ -118,7 +128,7 @@ cutting:
 measuring:
   annulus_radii: [10, 15]
   annulus_units: pixels
-  chosen_aperture: 0
+  use_annulus_for_centroids: true
   analytical_cuts: ['negatives', 'bad pixels', 'offsets', 'filter bank']
   outlier_sigma: 3.0
   bad_pixel_radius: 3.0
@@ -175,6 +185,7 @@ coaddition:
       measure_psf: true
       threshold: 3.0
       method: sextractor
+      background_method: zero
     # The following are used to override the regular astrometric calibration parameters
     wcs:
       cross_match_catalog: gaia_dr3
diff --git a/docs/troubleshooting_sqla.md b/docs/troubleshooting_sqla.md
new file mode 100644
index 00000000..d754f28a
--- /dev/null
+++ b/docs/troubleshooting_sqla.md
@@ -0,0 +1,147 @@
+## Troubleshooting SQLAlchemy
+
+Here is a growing list of common issues and solutions for SQLAlchemy.
+
+#### Adding this object again causes a unique constraint violation
+
+This is a common issue when you are trying to add an object to the session that is already on the DB. 
+Instead, use merge, and make sure to assign the merged object to a variable (often with the same name) 
+and keep using that. There's no real advantage to using `session.add()` over `session.merge()`. 
+
+Example: 
+
+```python
+obj = session.merge(obj)
+```
+
+#### Related objects get added to the session (and database) when they are not supposed to
+
+This is a hard one, where a complex web of relationships is causing SQLAlchemy to add objects to the session 
+when they are not supposed to.
+This happens when you `session.merge()` an object, not just on `session.add()`. 
+This is especially tricky when you are trying to delete a parent, so you merge it first, 
+and then you end up adding the children instead. 
+Usually the relationship will merge and then delete the children using cascades, 
+but some complex relationships may not work that way. 
+If you notice things are getting added when they shouldn't, check the session state before committing/flushing. 
+
+The places to look are: 
+```python
+session.identity_map.keys()
+session.new
+session.dirty
+session.deleted
+```
+
+If unwanted objects appear there, try to `session.expunge()` them before committing, or if they are persistent, 
+you may need to `session.delete()` them instead. 
+
+#### Double adding a related object through cascades
+
+Sometimes when a child is merged (or added) into a session, the parent is not automatically added. 
+Then, when the parent is added to the session on its own, it gets added as a new object, that can trigger
+unique violations (or, worse, just add duplicates). 
+
+The root of this problem is that the child object is merged without the parent. 
+Remember that a merged object is a new copy of the original, only connected to the session. 
+If you don't cascade the merge to the parent, you can't just assign the parent to the new object. 
+The parent object still keeps a reference to the old child object, and that one is not on the session. 
+Instead, make sure the merged child is assigned a merged parent, and that the parent is related 
+
+#### Cannot access related children when parent is not in the session
+
+This happens when a parent object is not in the session, but you want to access its children.
+The error message is usually something like this: 
+
+```
+sqlalchemy.orm.exc.DetachedInstanceError: Parent instance <Parent at 0x7f7f7f7f7f7f> is not bound to a Session; 
+lazy load operation of attribute 'children' cannot proceed
+```
+
+This happens under three possible circumstances. 
+1. The relationship is lazy loaded (which we generally try to avoid). 
+   Check the relationship definition has `lazy='selectin'`.
+2. The parent object was loaded as a related object itself, and that loading did not recursively load the children. 
+   Most objects will recursively load related objects of related objects, but in some cases this doesn't work, 
+   in particular when there's a many-to-many relationship via an association table (e.g., Provenance.upstreams). 
+   This is fixed by setting the `join_depth=1` or higher, as documented 
+   [here](https://docs.sqlalchemy.org/en/20/orm/self_referential.html#configuring-self-referential-eager-loading)
+3. The session has rolled back, or committed (this option only if you've changed to expire_on_commit=True). 
+   We usually have expire_on_commit=False, so that objects do not get expired when the session is committed.
+   However, when the session is rolled back, all objects are expired. That means you cannot use related objects, 
+   or even regular attributes, after a rollback. In most cases, a rollback is due to some crash, so having some 
+   errors accessing attributes/relationships while handling exceptions and "gracefully" exiting the program is expected, 
+   and doesn't require too much attention. If, however, you explicitly called a rollback, you should expect to have 
+   expired objects, and should go ahead and `session.refresh()` all the objects you need to use.
+
+#### Parent not in session, update along children is not updated in the database (Warning only)
+
+This is a warning that tells you that even though you added / deleted a child object, 
+the relationship cannot automatically update the object in the database, because the parent 
+is not connected to a session. 
+
+This is sometimes important but a lot of times meaningless. For example, if you deleted Parent, 
+and then go on to remove the children from it, it makes little difference that the relationship 
+is no longer emitting SQL changes, because the parent is going to be deleted anyway.
+
+
+#### `When initializing mapper Mapper[...], expression '...' failed to locate a name `
+
+This happens when a related object class is not imported when the relationship needs to be instantiated. 
+
+When two classes, A and B, are related to each other, we would see a definition like this: 
+
+```python
+class A(Base):
+    __tablename__ = 'a'
+    id = Column(Integer, primary_key=True)
+    b_id = Column(Integer, ForeignKey('b.id'))
+    b = relationship('B')
+
+class B(Base):
+    __tablename__ = 'b'
+    id = Column(Integer, primary_key=True)
+    a_id = Column(Integer, ForeignKey('a.id'))
+    a = relationship('A')
+```
+
+Notice that the `relationship` function is called with a string argument. 
+This is because the class `B` is not defined yet when the class `A` is defined.
+This solves a "chicken and egg" problem, by making a promise to the mapper that 
+when the relationships are instatiated, both classes will have been imported. 
+
+If some of the related objects are on a different file (module) and that file 
+is not imported by any of the code you are running, you will get the error above.
+
+This usually happens on scripts and parallel pipelines that only use a subset of the classes.
+To fix this, simply import the missing class module at the beginning of the script. 
+
+
+#### Changing the primary key of an object causes update instead of new object
+
+For objects that don't have an auto-incrementing primary key (e.g., Provenance),
+the user is in control of the value that goes into the primary key. 
+Sometimes, the user changes this value, e.g., when a Provenance gets new parameters
+and the `update_id()` method is called. 
+
+If the object is already in the session, and the primary key is changed, SQLAlchemy
+will update the object in the database, instead of creating a new one.
+This will remove the old object and may cause problems with objects that relate to 
+that row in the table. 
+
+Make sure to detach your object, or make a brand new one and copy properties over 
+to the new instance before merging it back into the session as a new object. 
+
+
+#### Deadlocks when querying the database
+
+This can occur when an internal session is querying the same objects 
+that an external session is using. 
+In general, you should not be opening an internal session when a different one is open, 
+instead, pass the session as an argument into the lower scope so all functions use the same session.
+
+If the app freezes, check for a deadlock: 
+Go into the DB and do `select * from pg_locks;` to see if there are many locks.
+
+Sometimes using `SELECT pg_cancel_backend(pid) FROM pg_locks; ` will free the lock. 
+Otherwise, try to restart the psql service. 
diff --git a/improc/alignment.py b/improc/alignment.py
index aae18a27..5eca7df4 100644
--- a/improc/alignment.py
+++ b/improc/alignment.py
@@ -3,16 +3,16 @@
 import random
 import time
 import subprocess
+import warnings
 
 import numpy as np
 
 import astropy.table
 import astropy.wcs.utils
-from astropy.io import fits
 
 from util import ldac
 from util.exceptions import SubprocessFailure
-from util.util import read_fits_image
+from util.util import read_fits_image, save_fits_image_file
 from util.logger import SCLogger
 import improc.scamp
 import improc.tools
@@ -21,6 +21,7 @@
 from models.provenance import Provenance
 from models.image import Image
 from models.source_list import SourceList
+from models.background import Background
 from models.enums_and_bitflags import string_to_bitflag, flag_image_bits_inverse
 
 from pipeline.data_store import DataStore
@@ -216,13 +217,15 @@ def _align_swarp( self, image, target, sources, target_sources ):
         tmptargetcat = tmppath / f'{tmpname}_target.sources.fits'
         tmpim = tmppath / f'{tmpname}_image.fits'
         tmpflags = tmppath / f'{tmpname}_flags.fits'
+        tmpbg = tmppath / f'{tmpname}_bg.fits'
 
         outim = tmppath / f'{tmpname}_warped.image.fits'
         outwt = tmppath / f'{tmpname}_warped.weight.fits'
         outfl = tmppath / f'{tmpname}_warped.flags.fits'
+        outbg = tmppath / f'{tmpname}_warped.bg.fits'
         outimhead = tmppath / f'{tmpname}_warped.image.head'
         outflhead = tmppath / f'{tmpname}_warped.flags.head'
-
+        outbghead = tmppath / f'{tmpname}_warped.bg.head'
         swarp_vmem_dir = tmppath /f'{tmpname}_vmem'
 
         # Writing this all out because several times I've looked at code
@@ -325,6 +328,7 @@ def _align_swarp( self, image, target, sources, target_sources ):
             hdr['NAXIS2'] = target.data.shape[0]
             hdr.tofile( outimhead )
             hdr.tofile( outflhead )
+            hdr.tofile( outbghead )
 
             # Warp the image
             # TODO : support single image with image, weight, flags in
@@ -347,14 +351,18 @@ def _align_swarp( self, image, target, sources, target_sources ):
             # putting in a symbolic link for the full FITS, instead of
             # copying the FITS data as here.  Look into that.)
 
-            with fits.open( impaths[imdex], mode='readonly' ) as hdul:
-                improc.tools.strip_wcs_keywords( hdul[0].header )
-                hdul[0].header.update( imagewcs.wcs.to_header() )
-                hdul.writeto( tmpim )
-            with fits.open( impaths[fldex], mode='readonly' ) as hdul:
-                improc.tools.strip_wcs_keywords( hdul[0].header )
-                hdul[0].header.update( imagewcs.wcs.to_header() )
-                hdul.writeto( tmpflags )
+            hdr = image.header.copy()
+            improc.tools.strip_wcs_keywords(hdr)
+            hdr.update(imagewcs.wcs.to_header())
+            if image.bg is None:
+                # to avoid this warning, consider adding a "zero" background object to the image
+                warnings.warn("No background image found. Using original image data.")
+                data = image.data
+            else:
+                data = image.data_bgsub
+
+            save_fits_image_file(tmpim, data, hdr, extname=None, single_file=False)
+            save_fits_image_file(tmpflags, image.flags, hdr, extname=None, single_file=False)
 
             swarp_vmem_dir.mkdir( exist_ok=True, parents=True )
 
@@ -402,8 +410,9 @@ def _align_swarp( self, image, target, sources, target_sources ):
 
             warpedim.data, warpedim.header = read_fits_image( outim, output="both" )
             # TODO: either make this not a hardcoded header value, or verify
-            # that we've constructed these images to have these hardcoded values
-            # (which would probably be a mistake, since it a priori assumes two amps).
+            #  that we've constructed these images to have these hardcoded values
+            #  (which would probably be a mistake, since it a priori assumes two amps).
+            #  Issue #216
             for att in ['SATURATA', 'SATURATB']:
                 if att in image.header:
                     warpedim.header[att] = image.header[att]
@@ -412,6 +421,43 @@ def _align_swarp( self, image, target, sources, target_sources ):
             warpedim.flags = read_fits_image(outfl)
             warpedim.flags = np.rint(warpedim.flags).astype(np.uint16)  # convert back to integers
 
+            # warp the background noise image:
+            if image.bg is not None:
+                bg = Background(
+                    value=0,
+                    noise=image.bg.noise,
+                    format=image.bg.format,
+                    method=image.bg.method,
+                    _bitflag=image.bg._bitflag,
+                    image=warpedim,
+                    provenance=image.bg.provenance,
+                    provenance_id=image.bg.provenance_id,
+                )
+                # TODO: what about polynomial model backgrounds?
+                if image.bg.format == 'map':
+                    save_fits_image_file(tmpbg, image.bg.variance, hdr, extname=None, single_file=False)
+                    command = ['swarp', tmpbg,
+                               '-IMAGEOUT_NAME', outbg,
+                               '-SUBTRACT_BACK', 'N',
+                               '-RESAMPLE_DIR', FileOnDiskMixin.temp_path,
+                               '-VMEM_DIR', swarp_vmem_dir,
+                               # '-VMEM_DIR', '/tmp',
+                               '-VMEM_MAX', '1024',
+                               '-MEM_MAX', '1024',
+                               '-WRITE_XML', 'N']
+
+                    t0 = time.perf_counter()
+                    res = subprocess.run(command, capture_output=True, timeout=self.pars.swarp_timeout)
+                    t1 = time.perf_counter()
+                    SCLogger.debug(f"swarp of background took {t1 - t0:.2f} seconds")
+                    if res.returncode != 0:
+                        raise SubprocessFailure(res)
+
+                    bg.variance = read_fits_image(outbg, output='data')
+                    bg.counts = np.zeros_like(bg.variance)
+
+                warpedim.bg = bg
+
             # re-calculate the source list and PSF for the warped image
             extractor = Detector()
             extractor.pars.override(sources.provenance.parameters['sources'], ignore_addons=True)
@@ -449,11 +495,14 @@ def _align_swarp( self, image, target, sources, target_sources ):
             tmptargetcat.unlink( missing_ok=True )
             tmpim.unlink( missing_ok=True )
             tmpflags.unlink( missing_ok=True )
+            tmpbg.unlink( missing_ok=True )
             outim.unlink( missing_ok=True )
             outwt.unlink( missing_ok=True )
             outfl.unlink( missing_ok=True )
+            outbg.unlink( missing_ok=True )
             outimhead.unlink( missing_ok=True )
             outflhead.unlink( missing_ok=True )
+            outbghead.unlink( missing_ok=True )
             for f in swarp_vmem_dir.iterdir():
                 f.unlink()
             swarp_vmem_dir.rmdir()
@@ -506,10 +555,33 @@ def run( self, source_image, target_image ):
         if target_image == source_image:
             warped_image = Image.copy_image( source_image )
             warped_image.type = 'Warped'
+            if source_image.bg is None:
+                warnings.warn("No background image found. Using original image data.")
+                warped_image.data = source_image.data
+                warped_image.bg = None  # this will be a problem later if you need to coadd the images!
+            else:
+                warped_image.data = source_image.data_bgsub
+                # make a copy of the background object but with zero mean
+                bg = Background(
+                    value=0,
+                    noise=source_image.bg.noise,
+                    format=source_image.bg.format,
+                    method=source_image.bg.method,
+                    _bitflag=source_image.bg._bitflag,
+                    image=warped_image,
+                    provenance=source_image.bg.provenance,
+                    provenance_id=source_image.bg.provenance_id,
+                )
+                if bg.format == 'map':
+                    bg.counts = np.zeros_like(warped_image.data)
+                    bg.variance = source_image.bg.variance
+                warped_image.bg = bg
+
             warped_image.psf = source_image.psf
             warped_image.zp = source_image.zp
             warped_image.wcs = source_image.wcs
             # TODO: what about SourceList?
+            # TODO: should these objects be copies of the products, or references to the same objects?
         else:  # Do the warp
             if self.pars.method == 'swarp':
                 SCLogger.debug( 'Aligning with swarp' )
diff --git a/improc/photometry.py b/improc/photometry.py
index 4ea12415..3c9918e3 100644
--- a/improc/photometry.py
+++ b/improc/photometry.py
@@ -251,6 +251,7 @@ def iterative_cutouts_photometry(
         normalizations=norms,
         background=background,
         variance=variance,
+        n_pix_bg=nandata.size,
         offset_x=cx,
         offset_y=cy,
         moment_xx=cxx,
@@ -272,7 +273,7 @@ def iterative_cutouts_photometry(
 
         # for each radius, do 1-3 rounds of repositioning the centroid
         for i in range(iterations):
-            flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, failure = calc_at_position(
+            flux, area, background, variance, n_pix_bg, norm, cx, cy, cxx, cyy, cxy, failure = calc_at_position(
                 nandata, r, annulus, xgrid, ygrid, cx, cy, local_bg=local_bg, full=False  # reposition only!
             )
 
@@ -296,7 +297,7 @@ def iterative_cutouts_photometry(
 
     # go over each radius again and this time get all outputs (e.g., cxx) using the best centroid
     for j, r in enumerate(radii):
-        flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, failure = calc_at_position(
+        flux, area, background, variance, n_pix_bg, norm, cx, cy, cxx, cyy, cxy, failure = calc_at_position(
             nandata,
             r,
             annulus,
@@ -323,6 +324,7 @@ def iterative_cutouts_photometry(
     photometry['areas'] = areas[::-1]  # return radii and areas in increasing order
     photometry['background'] = background
     photometry['variance'] = variance
+    photometry['n_pix_bg'] = n_pix_bg
     photometry['normalizations'] = norms[::-1]  # return radii and areas in increasing order
     photometry['offset_x'] = best_cx
     photometry['offset_y'] = best_cy
@@ -398,6 +400,8 @@ def calc_at_position(data, radius, annulus, xgrid, ygrid, cx, cy, local_bg=True,
         The background level.
     variance: float
         The variance of the background.
+    n_pix_bg: float
+        Number of pixels in the background annulus.
     norm: float
         The normalization factor for the flux error
         (this is the sqrt of the sum of squares of the aperture mask).
@@ -418,9 +422,9 @@ def calc_at_position(data, radius, annulus, xgrid, ygrid, cx, cy, local_bg=True,
         If True, it flags to the outer scope to stop
         the iterative process.
     """
-    flux = area = background = variance = norm = cxx = cyy = cxy = 0
+    flux = area = background = variance = n_pix_bg = norm = cxx = cyy = cxy = 0
     if np.all(np.isnan(data)):
-        return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
+        return flux, area, background, variance, n_pix_bg, norm, cx, cy, cxx, cyy, cxy, True
 
     # make a circle-mask based on the centroid position
     if not np.isfinite(cx) or not np.isfinite(cy):
@@ -429,14 +433,14 @@ def calc_at_position(data, radius, annulus, xgrid, ygrid, cx, cy, local_bg=True,
     # get a circular mask
     mask = get_circle(radius=radius, imsize=data.shape[0], soft=soft).get_image(cx, cy)
     if np.nansum(mask) == 0:
-        return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
+        return flux, area, background, variance, n_pix_bg, norm, cx, cy, cxx, cyy, cxy, True
 
     masked_data = data * mask
 
     flux = np.nansum(masked_data)  # total flux, not per pixel!
     area = np.nansum(mask)  # save the number of pixels in the aperture
     denominator = flux
-    masked_data_bg = masked_data
+    masked_data_bgsub = masked_data
 
     # get an offset annulus to get a local background estimate
     if full or local_bg:
@@ -446,7 +450,7 @@ def calc_at_position(data, radius, annulus, xgrid, ygrid, cx, cy, local_bg=True,
         annulus_map[annulus_map == 0.] = np.nan  # flag pixels outside annulus as nan
 
         if np.nansum(annulus_map) == 0:  # this can happen if annulus is too large
-            return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
+            return flux, area, background, variance, n_pix_bg, norm, cx, cy, cxx, cyy, cxy, True
 
         annulus_map_sum = np.nansum(annulus_map)
         if annulus_map_sum == 0 or np.all(np.isnan(annulus_map)):
@@ -462,26 +466,27 @@ def calc_at_position(data, radius, annulus, xgrid, ygrid, cx, cy, local_bg=True,
 
         if local_bg:  # update these to use the local background
             denominator = (flux - background * area)
-            masked_data_bg = (data - background) * mask
+            masked_data_bgsub = (data - background) * mask
 
     if denominator == 0:  # this should only happen in pathological cases
-        return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
+        return flux, area, background, variance, n_pix_bg, norm, cx, cy, cxx, cyy, cxy, True
 
     if not fixed:  # update the centroids
-        cx = np.nansum(xgrid * masked_data_bg) / denominator
-        cy = np.nansum(ygrid * masked_data_bg) / denominator
+        cx = np.nansum(xgrid * masked_data_bgsub) / denominator
+        cy = np.nansum(ygrid * masked_data_bgsub) / denominator
 
         # check that we got reasonable values!
         if np.isnan(cx) or abs(cx) > data.shape[1] / 2 or np.isnan(cy) or abs(cy) > data.shape[0] / 2:
-            return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
+            return flux, area, background, variance, n_pix_bg, norm, cx, cy, cxx, cyy, cxy, True
 
     if full:
         # update the second moments
-        cxx = np.nansum((xgrid - cx) ** 2 * masked_data_bg) / denominator
-        cyy = np.nansum((ygrid - cy) ** 2 * masked_data_bg) / denominator
-        cxy = np.nansum((xgrid - cx) * (ygrid - cy) * masked_data_bg) / denominator
+        cxx = np.nansum((xgrid - cx) ** 2 * masked_data_bgsub) / denominator
+        cyy = np.nansum((ygrid - cy) ** 2 * masked_data_bgsub) / denominator
+        cxy = np.nansum((xgrid - cx) * (ygrid - cy) * masked_data_bgsub) / denominator
 
-    return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, False
+    n_pix_bg = annulus_map_sum
+    return flux, area, background, variance, n_pix_bg, norm, cx, cy, cxx, cyy, cxy, False
 
 
 if __name__ == '__main__':
diff --git a/improc/simulator.py b/improc/simulator.py
index 02de229b..206713c2 100644
--- a/improc/simulator.py
+++ b/improc/simulator.py
@@ -7,12 +7,15 @@
 
 from util.logger import SCLogger
 
-
+# this is commented out as there are some problems installing it
+# consider replacing with https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.moyal.html
+# if this turns out to be important enough (it is not a main part of the simulator)
 # import pylandau
 
 from pipeline.parameters import Parameters
 from improc.tools import make_gaussian
 
+
 class SimPars(Parameters):
 
     def __init__(self, **kwargs):
diff --git a/models/background.py b/models/background.py
new file mode 100644
index 00000000..59803aad
--- /dev/null
+++ b/models/background.py
@@ -0,0 +1,433 @@
+import os
+
+import numpy as np
+
+import h5py
+
+import sqlalchemy as sa
+import sqlalchemy.orm as orm
+from sqlalchemy.ext.hybrid import hybrid_property
+from sqlalchemy.schema import UniqueConstraint
+
+from models.base import Base, SeeChangeBase, SmartSession, AutoIDMixin, FileOnDiskMixin, HasBitFlagBadness
+from models.image import Image
+
+from models.enums_and_bitflags import BackgroundFormatConverter, BackgroundMethodConverter, bg_badness_inverse
+
+
+class Background(Base, AutoIDMixin, FileOnDiskMixin, HasBitFlagBadness):
+    __tablename__ = 'backgrounds'
+
+    __table_args__ = (
+        UniqueConstraint('image_id', 'provenance_id', name='_bg_image_provenance_uc'),
+    )
+
+    _format = sa.Column(
+        sa.SMALLINT,
+        nullable=False,
+        default=BackgroundFormatConverter.convert('scalar'),
+        doc='Format of the Background model. Can include scalar, map, or polynomial. '
+    )
+
+    @hybrid_property
+    def format(self):
+        return BackgroundFormatConverter.convert(self._format)
+
+    @format.inplace.expression
+    @classmethod
+    def format(cls):
+        return sa.case(BackgroundFormatConverter.dict, value=cls._format)
+
+    @format.inplace.setter
+    def format(self, value):
+        self._format = BackgroundFormatConverter.convert(value)
+
+    _method = sa.Column(
+        sa.SMALLINT,
+        nullable=False,
+        default=BackgroundMethodConverter.convert('zero'),
+        doc='Method used to calculate the background. '
+            'Can be an algorithm like "sep", or "zero" for an image that was already background subtracted. ',
+    )
+
+    @hybrid_property
+    def method(self):
+        return BackgroundMethodConverter.convert(self._method)
+
+    @method.inplace.expression
+    @classmethod
+    def method(cls):
+        return sa.case(BackgroundMethodConverter.dict, value=cls._method)
+
+    @method.inplace.setter
+    def method(self, value):
+        self._method = BackgroundMethodConverter.convert(value)
+
+    image_id = sa.Column(
+        sa.ForeignKey('images.id', ondelete='CASCADE', name='backgrounds_image_id_fkey'),
+        nullable=False,
+        index=True,
+        doc="ID of the image for which this is the background."
+    )
+
+    image = orm.relationship(
+        'Image',
+        cascade='save-update, merge, refresh-expire, expunge',
+        passive_deletes=True,
+        lazy='selectin',
+        doc="Image for which this is the background."
+    )
+
+    value = sa.Column(
+        sa.Float,
+        index=True,
+        nullable=False,
+        doc="Value of the background level (in units of counts), as a best representative value for the entire image."
+    )
+
+    noise = sa.Column(
+        sa.Float,
+        index=True,
+        nullable=False,
+        doc="Noise RMS of the background (in units of counts), as a best representative value for the entire image."
+    )
+
+    provenance_id = sa.Column(
+        sa.ForeignKey('provenances.id', ondelete="CASCADE", name='backgrounds_provenance_id_fkey'),
+        nullable=False,
+        index=True,
+        doc=(
+            "ID of the provenance of this Background object. "
+            "The provenance will contain a record of the code version"
+            "and the parameters used to produce this Background object."
+        )
+    )
+
+    provenance = orm.relationship(
+        'Provenance',
+        cascade='save-update, merge, refresh-expire, expunge',
+        lazy='selectin',
+        doc=(
+            "Provenance of this Background object. "
+            "The provenance will contain a record of the code version"
+            "and the parameters used to produce this Background object."
+        )
+    )
+
+    __table_args__ = (
+        sa.Index( 'backgrounds_image_id_provenance_index', 'image_id', 'provenance_id', unique=True ),
+    )
+
+    @property
+    def image_shape(self):
+        if self._image_shape is None and self.filepath is not None:
+            self.load()
+        return self._image_shape
+
+    @image_shape.setter
+    def image_shape(self, value):
+        self._image_shape = value
+
+    @property
+    def counts(self):
+        """The background counts data for this object.
+
+        This will either be a map that is loaded directly from file,
+        or an interpolated map based on the polynomial or scalar value
+        mapped onto the image shape.
+
+        This is a best-estimate of the sky counts, ignoring as best as
+        possible the sources in the sky, and looking only at the smoothed
+        background level.
+        """
+        if self._counts_data is None and self.filepath is not None:
+            self.load()
+        return self._counts_data
+
+    @counts.setter
+    def counts(self, value):
+        self._counts_data = value
+
+    @property
+    def variance(self):
+        """The background variance data for this object.
+
+        This will either be a map that is loaded directly from file,
+        or an interpolated map based on the polynomial or scalar value
+        mapped onto the image shape.
+
+        This is a best-estimate of the sky noise, ignoring as best as
+        possible the sources in the sky, and looking only at the smoothed
+        background variability.
+        """
+        if self._var_data is None and self.filepath is not None:
+            self.load()
+        return self._var_data
+
+    @variance.setter
+    def variance(self, value):
+        self._var_data = value
+
+    @property
+    def rms(self):
+        if self.variance is None:
+            return None
+        return np.sqrt(self.variance)
+
+    @rms.setter
+    def rms(self, value):
+        self.variance = value ** 2
+
+    def _get_inverse_badness(self):
+        """Get a dict with the allowed values of badness that can be assigned to this object"""
+        return bg_badness_inverse
+
+    def __init__( self, *args, **kwargs ):
+        FileOnDiskMixin.__init__( self, **kwargs )
+        HasBitFlagBadness.__init__(self)
+        SeeChangeBase.__init__( self )
+        self._image_shape = None
+        self._counts_data = None
+        self._var_data = None
+
+        # Manually set all properties ( columns or not )
+        for key, value in kwargs.items():
+            if hasattr( self, key ):
+                setattr( self, key, value )
+
+    @sa.orm.reconstructor
+    def init_on_load( self ):
+        Base.init_on_load( self )
+        FileOnDiskMixin.init_on_load( self )
+        self._image_shape = None
+        self._counts_data = None
+        self._var_data = None
+
+    def __setattr__(self, key, value):
+        if key == 'image':
+            if value is not None and not isinstance(value, Image):
+                raise ValueError(f'Background.image must be an Image object. Got {type(value)} instead. ')
+            self._image_shape = value.data.shape
+
+        super().__setattr__(key, value)
+
+    def save( self, filename=None, **kwargs ):
+        """Write the Background to disk.
+
+        May or may not upload to the archive and update the
+        FileOnDiskMixin-included fields of this object based on the
+        additional arguments that are forwarded to FileOnDiskMixin.save.
+
+        This saves an HDF5 file that contains a single group called "/background".
+        It will have a few attributes, notably: "format", "value", "noise" and "image_shape".
+
+        If the format is "map", there are two datasets under this group: "background/counts" and "background/variance".
+        Counts represents the background counts at each location in the image, while the variance represents the noise
+        variability that comes from the sky, ignoring the sources (as much as possible).
+
+        If the format is "polynomial", there are three datasets:
+        "background/coeffs" and "background/x_degree" and "background/y_degree".
+        These will include the coefficients of the polynomial, and the degree of the polynomial in x and y as such:
+        Constant term, x term, y term, x^2 term, xy term, y^2 term, x^3 term, x^2y term, xy^2 term, y^3 term, etc.
+        Which corresponds to a list of degrees:
+        x_degree: [0, 1, 0, 2, 1, 0, 3, 2, 1, 0, ...]
+        y_degree: [0, 0, 1, 0, 1, 2, 0, 1, 2, 3, ...]
+
+        Finally, if the format is "scalar", there would not be any datasets.
+
+        Parameters
+        ----------
+          filename: str or path
+             The path to the file to write, relative to the local store
+             root.  Do not include the extension (e.g. '.h5') at the
+             end of the name; that will be added automatically for all
+             extensions.  If None, will call image.invent_filepath() to get a
+             filestore-standard filename and directory.
+
+          Additional arguments are passed on to FileOnDiskMixin.save()
+
+        """
+        if self.format not in ['scalar', 'map', 'polynomial']:
+            raise ValueError(f'Unknown background format "{self.format}".')
+
+        if self.value is None or self.noise is None:
+            raise RuntimeError( "Both value and noise must be non-None" )
+
+        if self.format == 'map' and (self.counts is None or self.variance is None):
+            raise RuntimeError( "Both counts and variance must be non-None" )
+
+        # TODO: add some checks for the polynomial format
+
+        if filename is not None:
+            if not filename.endswith('.h5'):
+                filename += '.h5'
+            self.filepath = filename
+        else:
+            if self.image.filepath is not None:
+                self.filepath = self.image.filepath
+            else:
+                self.filepath = self.image.invent_filepath()
+
+            if self.provenance is None:
+                raise RuntimeError("Can't invent a filepath for the Background without a provenance")
+            self.filepath += f'.bg_{self.provenance.id[:6]}.h5'
+
+        h5path = os.path.join( self.local_path, f'{self.filepath}')
+
+        with h5py.File(h5path, 'w') as h5f:
+            bggrp = h5f.create_group('background')
+            bggrp.attrs['format'] = self.format
+            bggrp.attrs['method'] = self.method
+            bggrp.attrs['value'] = self.value
+            bggrp.attrs['noise'] = self.noise
+            bggrp.attrs['image_shape'] = self.image_shape
+
+            if self.format == 'map':
+                if self.counts is None or self.variance is None:
+                    raise RuntimeError("Both counts and variance must be non-None")
+                if self.counts.shape != self.image_shape:
+                    raise RuntimeError(
+                        f"Counts shape {self.counts.shape} does not match image shape {self.image_shape}"
+                    )
+                if self.variance.shape != self.image_shape:
+                    raise RuntimeError(
+                        f"Variance shape {self.variance.shape} does not match image shape {self.image_shape}"
+                    )
+
+                bggrp.create_dataset( 'counts', data=self.counts )
+                bggrp.create_dataset( 'variance', data=self.variance )
+            elif self.format == 'polynomial':
+                raise NotImplementedError('Currently we do not support a polynomial background model. ')
+                bggrp.create_dataset( 'coeffs', data=self.counts )
+                bggrp.create_dataset( 'x_degree', data=self.x_degree )
+                bggrp.create_dataset( 'y_degree', data=self.y_degree )
+            elif self.format == 'scalar':
+                pass  # no datasets to create
+            else:
+                raise ValueError( f'Unknown background format "{self.format}".' )
+
+        # Save the file to the archive and update the database record
+        # (From what we did above, the files are already in the right place in the local filestore.)
+        FileOnDiskMixin.save( self, h5path, extension=None, **kwargs )
+
+    def load(self, download=True, always_verify_md5=False, filepath=None):
+        """Load the data from the files into the _counts_data, _var_data and _image_shape fields.
+
+        Parameters
+        ----------
+          download : Bool, default True
+            If True, download the files from the archive if they're not
+            found in local storage.  Ignored if filepath is not None.
+
+          always_verify_md5 : Bool, default False
+            If the file is found locally, verify the md5 of the file; if
+            it doesn't match, re-get the file from the archive.  Ignored
+            if filepath is not None.
+
+        """
+        if filepath is None:
+            filepath = self.get_fullpath(download=download, always_verify_md5=always_verify_md5)
+
+        with h5py.File(filepath, 'r') as h5f:
+            if 'background' not in h5f:
+                raise ValueError('No background group found in the file. ')
+            loaded_format = h5f['background'].attrs['format']
+
+            if self.format != loaded_format:
+                raise ValueError(
+                    f'Loaded background format "{loaded_format}" does not match the expected format "{self.format}".'
+                )
+
+            self.value = float(h5f['background'].attrs['value'])
+            self.noise = float(h5f['background'].attrs['noise'])
+            self.image_shape = tuple(h5f['background'].attrs['image_shape'])
+
+            if loaded_format == 'map':
+                self._counts_data = h5f['background/counts'][:]
+                self._var_data = h5f['background/variance'][:]
+            elif loaded_format == 'polynomial':
+                raise NotImplementedError('Currently we do not support a polynomial background model. ')
+                self._counts_data = h5f['background/coeffs'][:]
+                self._x_degree = h5f['background/x_degree'][:]
+                self._y_degree = h5f['background/y_degree'][:]
+            elif loaded_format == 'scalar':
+                pass
+            else:
+                raise ValueError( f'Unknown background format "{loaded_format}".' )
+
+    def free( self ):
+        """Free loaded world coordinates memory.
+
+        Wipe out the _counts_data and _var_data fields, freeing memory.
+        Depends on python garbage collection, so if there are other
+        references to those objects, the memory won't actually be freed.
+
+        """
+        self._counts_data = None
+        self._var_data = None
+
+    def get_upstreams(self, session=None):
+        """Get the image that was used to make this Background object. """
+        with SmartSession(session) as session:
+            return session.scalars(sa.select(Image).where(Image.id == self.image_id)).all()
+
+    def get_downstreams(self, session=None, siblings=False):
+        """Get the downstreams of this Background object.
+
+        If siblings=True then also include the SourceList, PSF, WCS, and ZP
+        that were created at the same time as this PSF.
+        """
+        from models.source_list import SourceList
+        from models.psf import PSF
+        from models.world_coordinates import WorldCoordinates
+        from models.zero_point import ZeroPoint
+        from models.provenance import Provenance
+
+        with SmartSession(session) as session:
+            subs = session.scalars(
+                sa.select(Image).where(
+                    Image.provenance.has(Provenance.upstreams.any(Provenance.id == self.provenance.id)),
+                    Image.upstream_images.any(Image.id == self.image_id),
+                )
+            ).all()
+            output = subs
+
+            if siblings:
+                # There should be exactly one source list, wcs, and zp per PSF, with the same provenance
+                # as they are created at the same time.
+                sources = session.scalars(
+                    sa.select(SourceList).where(
+                        SourceList.image_id == self.image_id, SourceList.provenance_id == self.provenance_id
+                    )
+                ).all()
+                if len(sources) != 1:
+                    raise ValueError(
+                        f"Expected exactly one source list for Background {self.id}, but found {len(sources)}"
+                    )
+
+                output.append(sources[0])
+
+                psfs = session.scalars(
+                    sa.select(PSF).where(PSF.image_id == self.image_id, PSF.provenance_id == self.provenance_id)
+                ).all()
+                if len(psfs) != 1:
+                    raise ValueError(f"Expected exactly one PSF for Background {self.id}, but found {len(psfs)}")
+
+                output.append(psfs[0])
+
+                wcs = session.scalars(
+                    sa.select(WorldCoordinates).where(WorldCoordinates.sources_id == sources.id)
+                ).all()
+                if len(wcs) != 1:
+                    raise ValueError(f"Expected exactly one wcs for Background {self.id}, but found {len(wcs)}")
+
+                output.append(wcs[0])
+
+                zp = session.scalars(sa.select(ZeroPoint).where(ZeroPoint.sources_id == sources.id)).all()
+
+                if len(zp) != 1:
+                    raise ValueError(f"Expected exactly one zp for Background {self.id}, but found {len(zp)}")
+
+                output.append(zp[0])
+
+        return output
diff --git a/models/base.py b/models/base.py
index f1a96801..f7cc1d70 100644
--- a/models/base.py
+++ b/models/base.py
@@ -1697,6 +1697,10 @@ def append_badness(self, value):
         doc='Free text comment about this data product, e.g., why it is bad. '
     )
 
+    def __init__(self):
+        self._bitflag = 0
+        self._upstream_bitflag = 0
+
     def update_downstream_badness(self, session=None, commit=True, siblings=True):
         """Send a recursive command to update all downstream objects that have bitflags.
 
diff --git a/models/cutouts.py b/models/cutouts.py
index b35e153e..a4df997c 100644
--- a/models/cutouts.py
+++ b/models/cutouts.py
@@ -126,6 +126,7 @@ def ref_image(self):
 
     def __init__(self, *args, **kwargs):
         FileOnDiskMixin.__init__(self, *args, **kwargs)
+        HasBitFlagBadness.__init__(self)
         SeeChangeBase.__init__(self)  # don't pass kwargs as they could contain non-column key-values
 
         self.format = 'hdf5'  # the default should match the column-defined default above!
@@ -291,7 +292,6 @@ def invent_filepath(self):
             filename = os.path.splitext(filename)[0]
 
         filename += '.cutouts_'
-        self.provenance.update_id()
         filename += self.provenance.id[:6]
         if self.format == 'hdf5':
             filename += '.h5'
diff --git a/models/enums_and_bitflags.py b/models/enums_and_bitflags.py
index 85f62131..b3673d57 100644
--- a/models/enums_and_bitflags.py
+++ b/models/enums_and_bitflags.py
@@ -239,6 +239,27 @@ class PSFFormatConverter( EnumConverter ):
     _dict_inverse = None
 
 
+class BackgroundFormatConverter( EnumConverter ):
+    _dict = {
+        0: 'scalar',
+        1: 'map',
+        2: 'polynomial',
+    }
+    _allowed_values = None
+    _dict_filtered = None
+    _dict_inverse = None
+
+
+class BackgroundMethodConverter( EnumConverter ):
+    _dict = {
+        0: 'zero',
+        1: 'sep',
+    }
+    _allowed_values = None
+    _dict_filtered = None
+    _dict_inverse = None
+
+
 def bitflag_to_string(value, dictionary):
 
     """
@@ -364,6 +385,12 @@ def string_to_bitflag(value, dictionary):
 source_list_badness_inverse = {EnumConverter.c(v): k for k, v in source_list_badness_dict.items()}
 
 
+# these are the ways a Background object is allowed to be bad
+background_badness_dict = {
+
+}
+
+
 # these are the ways a WorldCoordinates/ZeroPoint object is allowed to be bad
 # mostly due to bad matches to the catalog
 catalog_match_badness_dict = {
@@ -374,6 +401,15 @@ def string_to_bitflag(value, dictionary):
 catalog_match_badness_inverse = {EnumConverter.c(v): k for k, v in catalog_match_badness_dict.items()}
 
 
+# TODO: need to consider what kinds of bad backgrounds we really might have
+# TODO: make sure we are not repeating the same keywords in other badness dictionaries
+bg_badness_dict = {
+    31: 'too dense',
+    32: 'bad fit',
+}
+bg_badness_inverse = {EnumConverter.c(v): k for k, v in bg_badness_dict.items()}
+
+
 # these are the ways a Cutouts object is allowed to be bad
 cutouts_badness_dict = {
     41: 'cosmic ray',
@@ -391,6 +427,10 @@ def string_to_bitflag(value, dictionary):
 data_badness_dict.update(image_badness_dict)
 data_badness_dict.update(cutouts_badness_dict)
 data_badness_dict.update(source_list_badness_dict)
+data_badness_dict.update(psf_badness_dict)
+data_badness_dict.update(bg_badness_dict)
+data_badness_dict.update(catalog_match_badness_dict)
+data_badness_dict.update(bg_badness_dict)
 data_badness_inverse = {EnumConverter.c(v): k for k, v in data_badness_dict.items()}
 if 0 in data_badness_inverse:
     raise ValueError('Cannot have a badness bitflag of zero. This is reserved for good data.')
@@ -402,6 +442,20 @@ class BadnessConverter( EnumConverter ):
     _dict_filtered = None
     _dict_inverse = None
 
+
+# bitflag for image preprocessing steps that have been done
+image_preprocessing_dict = {
+    0: 'overscan',
+    1: 'zero',
+    2: 'dark',
+    3: 'linearity',
+    4: 'flat',
+    5: 'fringe',
+    6: 'illumination'
+}
+image_preprocessing_inverse = {EnumConverter.c(v):k for k, v in image_preprocessing_dict.items()}
+
+
 # bitflag used in flag images
 flag_image_bits = {
     0: 'bad pixel',        # Bad pixel flagged by the instrument
@@ -422,7 +476,7 @@ class BitFlagConverter( EnumConverter ):
 # the list of possible processing steps from a section of an exposure up to measurements, r/b scores, and report
 process_steps_dict = {
     1: 'preprocessing',  # creates an Image from a section of the Exposure
-    2: 'extraction',     # creates a SourceList from an Image, and a PSF
+    2: 'extraction',     # creates a SourceList, PSF, Background, WorldCoordinates, and ZeroPoint from an Image
     5: 'subtraction',    # creates a subtraction Image
     6: 'detection',      # creates a SourceList from a subtraction Image
     7: 'cutting',        # creates Cutouts from a subtraction Image
@@ -437,7 +491,7 @@ class BitFlagConverter( EnumConverter ):
     1: 'image',
     2: 'sources',
     3: 'psf',
-    # 4: 'background',  # not yet implemented
+    4: 'bg',
     5: 'wcs',
     6: 'zp',
     7: 'sub_image',
diff --git a/models/exposure.py b/models/exposure.py
index 052d3269..fa4d2fe3 100644
--- a/models/exposure.py
+++ b/models/exposure.py
@@ -340,6 +340,7 @@ def __init__(self, current_file=None, invent_filepath=True, **kwargs):
 
         """
         FileOnDiskMixin.__init__(self, **kwargs)
+        HasBitFlagBadness.__init__(self)
         SeeChangeBase.__init__(self)  # don't pass kwargs as they could contain non-column key-values
 
         self._data = None  # the underlying image data for each section
diff --git a/models/image.py b/models/image.py
index d1efd3f1..515c0e29 100644
--- a/models/image.py
+++ b/models/image.py
@@ -113,6 +113,7 @@ def format(self, value):
         cascade='save-update, merge, refresh-expire, expunge',
         passive_deletes=True,
         lazy='selectin',
+        join_depth=1,  # this enables the eager load of one generation of upstreams
         order_by='images.c.mjd',  # in chronological order of exposure start time
         doc='Images used to produce a multi-image object, like a coadd or a subtraction. '
     )
@@ -460,7 +461,6 @@ def _get_inverse_badness(self):
         'data',
         'flags',
         'weight',
-        'background',  # TODO: remove this when adding the Background object (issue #186)
         'score',  # the matched-filter score of the image (e.g., from ZOGY)
         'psfflux',  # the PSF-fitted equivalent flux of the image (e.g., from ZOGY)
         'psffluxerr', # the error in the PSF-fitted equivalent flux of the image (e.g., from ZOGY)
@@ -468,6 +468,7 @@ def _get_inverse_badness(self):
 
     def __init__(self, *args, **kwargs):
         FileOnDiskMixin.__init__(self, *args, **kwargs)
+        HasBitFlagBadness.__init__(self)
         SeeChangeBase.__init__(self)  # don't pass kwargs as they could contain non-column key-values
 
         self.raw_data = None  # the raw exposure pixels (2D float or uint16 or whatever) not saved to disk!
@@ -477,7 +478,6 @@ def __init__(self, *args, **kwargs):
         self._data = None  # the underlying pixel data array (2D float array)
         self._flags = None  # the bit-flag array (2D int array)
         self._weight = None  # the inverse-variance array (2D float array)
-        self._background = None  # an estimate for the background flux (2D float array)
         self._score = None  # the image after filtering with the PSF and normalizing to S/N units (2D float array)
         self._psfflux = None  # the PSF-fitted equivalent flux of the image (2D float array)
         self._psffluxerr = None  # the error in the PSF-fitted equivalent flux of the image (2D float array)
@@ -485,6 +485,7 @@ def __init__(self, *args, **kwargs):
         # additional data products that could go with the Image
         self.sources = None  # the sources extracted from this Image (optionally loaded)
         self.psf = None  # the point-spread-function object (optionally loaded)
+        self.bg = None  # the background object (optionally loaded)
         self.wcs = None  # the WorldCoordinates object (optionally loaded)
         self.zp = None  # the zero-point object (optionally loaded)
 
@@ -526,6 +527,7 @@ def init_on_load(self):
 
         self.sources = None
         self.psf = None
+        self.bg = None
         self.wcs = None
         self.zp = None
 
@@ -594,16 +596,14 @@ def merge_all(self, session):
             self.psf.image_id = new_image.id
             self.psf.provenance_id = self.psf.provenance.id if self.psf.provenance is not None else None
             new_image.psf = self.psf.safe_merge(session=session)
-            if new_image.psf._bitflag is None:  # I don't know why this isn't set to 0 using the default
-                new_image.psf._bitflag = 0
-            if new_image.psf._upstream_bitflag is None:  # I don't know why this isn't set to 0 using the default
-                new_image.psf._upstream_bitflag = 0
+
+        if self.bg is not None:
+            self.bg.image = new_image
+            self.bg.image_id = new_image.id
+            self.bg.provenance_id = self.bg.provenance.id if self.bg.provenance is not None else None
+            new_image.bg = self.bg.safe_merge(session=session)
 
         # take care of the upstream images and their products
-        # if sa.inspect(self).detached:  # self can't load the images, but new_image has them
-        #     upstream_list = new_image.upstream_images
-        # else:
-        #     upstream_list = self.upstream_images  # can use the original images, before merging into new_image
         try:
             upstream_list = self.upstream_images  # can use the original images, before merging into new_image
         except DetachedInstanceError as e:
@@ -1095,7 +1095,7 @@ def _make_aligned_images(self):
 
         # verify all products are loaded
         for im in self.upstream_images:
-            if im.sources is None or im.wcs is None or im.zp is None:
+            if im.sources is None or im.bg is None or im.wcs is None or im.zp is None:
                 raise RuntimeError('Some images are missing data products. Try running load_upstream_products().')
 
         aligned = []
@@ -1529,8 +1529,8 @@ def free( self, free_derived_products=True, free_aligned=True, only_free=None ):
         Parameters
         ----------
           free_derived_products: bool, default True
-             If True, will also call free on self.sources, self.psf, and
-             self.wcs
+             If True, will also call free on self.sources, self.psf,
+             self.bg and self.wcs.
 
           free_aligned: bool, default True
              Will call free() on each of the aligned images referenced
@@ -1560,11 +1560,11 @@ def free( self, free_derived_products=True, free_aligned=True, only_free=None ):
                 self.sources.free()
             if self.psf is not None:
                 self.psf.free()
-            # This implementation in WCS should be done after PR167 is done.
-            # Not a big deal if it's not done, because WCSes will not use
-            # very much memory
-            # if self.wcs is not None:
-            #     self.wcs.free()
+            if self.bg is not None:
+                self.bg.free()
+            if self.wcs is not None:
+                self.wcs.free()
+
         if free_aligned:
             if self._aligned_images is not None:
                 for alim in self._aligned_images:
@@ -1678,6 +1678,9 @@ def load_upstream_products(self, session=None):
             if im.psf is None or im.psf.provenance_id not in prov_ids:
                 need_to_load = True
                 break
+            if im.bg is None or im.bg.provenance_id not in prov_ids:
+                need_to_load = True
+                break
             if im.wcs is None or im.wcs.provenance_id not in prov_ids:
                 need_to_load = True
                 break
@@ -1690,6 +1693,7 @@ def load_upstream_products(self, session=None):
 
         from models.source_list import SourceList
         from models.psf import PSF
+        from models.background import Background
         from models.world_coordinates import WorldCoordinates
         from models.zero_point import ZeroPoint
 
@@ -1717,6 +1721,13 @@ def load_upstream_products(self, session=None):
                     )
                 ).all()
 
+                bg_results = session.scalars(
+                    sa.select(Background).where(
+                        Background.image_id.in_(im_ids),
+                        Background.provenance_id.in_(prov_ids),
+                    )
+                ).all()
+
                 wcs_results = session.scalars(
                     sa.select(WorldCoordinates).where(
                         WorldCoordinates.sources_id.in_(sources_ids),
@@ -1748,6 +1759,14 @@ def load_upstream_products(self, session=None):
                     elif len(psfs) == 1:
                         im.psf = psfs[0]
 
+                    bgs = [b for b in bg_results if b.image_id == im.id]  # only get the bgs for this image
+                    if len(bgs) > 1:
+                        raise ValueError(
+                            f"Image {im.id} has more than one Background matching upstream provenance."
+                        )
+                    elif len(bgs) == 1:
+                        im.bg = bgs[0]
+
                     if im.sources is not None:
                         wcses = [w for w in wcs_results if w.sources_id == im.sources.id]  # the wcses for this image
                         if len(wcses) > 1:
@@ -1804,6 +1823,8 @@ def get_upstreams(self, session=None):
                     upstreams.append(im.sources)
                 if im.psf is not None:
                     upstreams.append(im.psf)
+                if im.bg is not None:
+                    upstreams.append(im.bg)
                 if im.wcs is not None:
                     upstreams.append(im.wcs)
                 if im.zp is not None:
@@ -1816,17 +1837,12 @@ def get_downstreams(self, session=None, siblings=False):
         # avoids circular import
         from models.source_list import SourceList
         from models.psf import PSF
+        from models.background import Background
         from models.world_coordinates import WorldCoordinates
         from models.zero_point import ZeroPoint
 
         downstreams = []
         with SmartSession(session) as session:
-            # get all psfs that are related to this image (regardless of provenance)
-            psfs = session.scalars(sa.select(PSF).where(PSF.image_id == self.id)).all()
-            downstreams += psfs
-            if self.psf is not None and self.psf not in psfs:  # if not in the session, could be duplicate!
-                downstreams.append(self.psf)
-
             # get all source lists that are related to this image (regardless of provenance)
             sources = session.scalars(
                 sa.select(SourceList).where(SourceList.image_id == self.id)
@@ -1835,6 +1851,17 @@ def get_downstreams(self, session=None, siblings=False):
             if self.sources is not None and self.sources not in sources:  # if not in the session, could be duplicate!
                 downstreams.append(self.sources)
 
+            # get all psfs that are related to this image (regardless of provenance)
+            psfs = session.scalars(sa.select(PSF).where(PSF.image_id == self.id)).all()
+            downstreams += psfs
+            if self.psf is not None and self.psf not in psfs:  # if not in the session, could be duplicate!
+                downstreams.append(self.psf)
+
+            bgs = session.scalars(sa.select(Background).where(Background.image_id == self.id)).all()
+            downstreams += bgs
+            if self.bg is not None and self.bg not in bgs:  # if not in the session, could be duplicate!
+                downstreams.append(self.bg)
+
             wcses = []
             zps = []
             for s in sources:
@@ -1928,17 +1955,6 @@ def weight(self):
     def weight(self, value):
         self._weight = value
 
-    @property
-    def background(self):
-        """An estimate for the background flux (2D float array). """
-        if self._data is None and self.filepath is not None:
-            self.load()
-        return self._background
-
-    @background.setter
-    def background(self, value):
-        self._background = value
-
     @property
     def score(self):
         """The image after filtering with the PSF and normalizing to S/N units (2D float array). """
@@ -1999,6 +2015,26 @@ def nanscore(self):
     def nanscore(self, value):
         self._nanscore = value
 
+    @property
+    def data_bgsub(self):
+        """The image data, after subtracting the background. If no Background object is loaded, will raise. """
+        if self.bg is None:
+            raise ValueError("No background is loaded for this image.")
+        if self.bg.format == 'scalar':
+            return self.data - self.bg.value
+        else:
+            return self.data - self.bg.counts
+
+    @property
+    def nandata_bgsub(self):
+        """The image data, after subtracting the background and masking with NaNs wherever the flag is not zero. """
+        if self.bg is None:
+            raise ValueError("No background is loaded for this image.")
+        if self.bg.format == 'scalar':
+            return self.nandata - self.bg.value
+        else:
+            return self.nandata - self.bg.counts
+
     def show(self, **kwargs):
         """
         Display the image using the matplotlib imshow function.
diff --git a/models/instrument.py b/models/instrument.py
index dea62d94..a8afb278 100644
--- a/models/instrument.py
+++ b/models/instrument.py
@@ -1172,6 +1172,7 @@ def standard_apertures( cls ):
            list of float
 
         """
+        return RuntimeError('We should no longer depend on instruments to give the standard apertures')
         return [ 0.6732, 1., 2., 3., 4., 5., 7., 10. ]
 
     @classmethod
@@ -1200,6 +1201,7 @@ def fiducial_aperture( cls ):
         # though, for diagnostic purposes.
 
         # Note that this 5 is an index, not the value... it's coincidence that the index number is 5.
+        return RuntimeError('We should no longer depend on instruments to give the fiducial aperture')
         return 5
 
     # Gaia specific methods
diff --git a/models/measurements.py b/models/measurements.py
index 547660f6..6b42fda8 100644
--- a/models/measurements.py
+++ b/models/measurements.py
@@ -117,18 +117,19 @@ class Measurements(Base, AutoIDMixin, SpatiallyIndexed, HasBitFlagBadness):
     def flux(self):
         """The background subtracted aperture flux in the "best" aperture. """
         if self.best_aperture == -1:
-            return self.flux_psf - self.background * self.area_psf
+            return self.flux_psf - self.bkg_mean * self.area_psf
         else:
-            return self.flux_apertures[self.best_aperture] - self.background * self.area_apertures[self.best_aperture]
+            return self.flux_apertures[self.best_aperture] - self.bkg_mean * self.area_apertures[self.best_aperture]
 
     @property
     def flux_err(self):
         """The error on the background subtracted aperture flux in the "best" aperture. """
+        # we divide by the number of pixels of the background as that is how well we can estimate the b/g mean
         if self.best_aperture == -1:
-            return np.sqrt(self.flux_psf_err ** 2 + self.background_err ** 2 * self.area_psf)
+            return np.sqrt(self.flux_psf_err ** 2 + self.bkg_std ** 2 / self.bkg_pix * self.area_psf)
         else:
             err = self.flux_apertures_err[self.best_aperture]
-            err += self.background_err ** 2 * self.area_apertures[self.best_aperture]
+            err += self.bkg_std ** 2 / self.bkg_pix * self.area_apertures[self.best_aperture]
             return np.sqrt(err)
 
     @property
@@ -165,15 +166,15 @@ def mag_apertures_err(self):
 
     @property
     def magnitude(self):
+        mag = -2.5 * np.log10(self.flux) + self.zp.zp
         if self.best_aperture == -1:
-            return self.mag_psf
-        return self.mag_apertures[self.best_aperture]
+            return mag
+        else:
+            return mag + self.zp.aper_cors[self.best_aperture]
 
     @property
     def magnitude_err(self):
-        if self.best_aperture == -1:
-            return self.mag_psf_err
-        return self.mag_apertures_err[self.best_aperture]
+        return np.sqrt((2.5 / np.log(10) * self.flux_err / self.flux) ** 2 + self.zp.dzp ** 2)
 
     @property
     def lim_mag(self):
@@ -213,18 +214,25 @@ def instrument_object(self):
             return None
         return self.cutouts.sources.image.instrument_object
 
-    background = sa.Column(
+    bkg_mean = sa.Column(
         sa.REAL,
         nullable=False,
         doc="Background of the measurement, from a local annulus. Given as counts per pixel. "
     )
 
-    background_err = sa.Column(
+    bkg_std = sa.Column(
         sa.REAL,
         nullable=False,
         doc="RMS error of the background measurement, from a local annulus. Given as counts per pixel. "
     )
 
+    bkg_pix = sa.Column(
+        sa.REAL,
+        nullable=False,
+        doc="Annulus area (in pixels) used to calculate the mean/std of the background. "
+            "An estimate of the error on the mean would be bkg_std / sqrt(bkg_pix)."
+    )
+
     area_psf = sa.Column(
         sa.REAL,
         nullable=False,
@@ -291,6 +299,7 @@ def instrument_object(self):
 
     def __init__(self, **kwargs):
         SeeChangeBase.__init__(self)  # don't pass kwargs as they could contain non-column key-values
+        HasBitFlagBadness.__init__(self)
         self._cutouts_list_index = None  # helper (transient) attribute that helps find the right cutouts in a list
 
         # manually set all properties (columns or not)
@@ -474,7 +483,7 @@ def get_flux_at_point(self, ra, dec, aperture=None):
             mask[start_y:end_y, start_x:end_x] = psf_clip[start_y + dy:end_y + dy, start_x + dx:end_x + dx]
             mask[np.isnan(im)] = 0  # exclude bad pixels from the mask
             flux = np.nansum(im * mask) / np.nansum(mask ** 2)
-            fluxerr = self.background_err / np.sqrt(np.nansum(mask ** 2))
+            fluxerr = self.bkg_std / np.sqrt(np.nansum(mask ** 2))
             area = np.nansum(mask) / (np.nansum(mask ** 2))
         else:
             radius = self.aper_radii[aperture]
@@ -482,7 +491,7 @@ def get_flux_at_point(self, ra, dec, aperture=None):
             mask = get_circle(radius=radius, imsize=im.shape[0], soft=True).get_image(offset_x, offset_y)
             # for aperture photometry we don't normalize, just assume the PSF is in the aperture
             flux = np.nansum(im * mask)
-            fluxerr = self.background_err * np.sqrt(np.nansum(mask ** 2))
+            fluxerr = self.bkg_std * np.sqrt(np.nansum(mask ** 2))
             area = np.nansum(mask)
 
         return flux, fluxerr, area
diff --git a/models/provenance.py b/models/provenance.py
index 2b9ced8a..75ce1d8f 100644
--- a/models/provenance.py
+++ b/models/provenance.py
@@ -125,6 +125,7 @@ class Provenance(Base):
         passive_deletes=True,
         cascade="save-update, merge, expunge, refresh-expire",
         lazy='selectin',  # should be able to get upstream_hashes without a session!
+        join_depth=3,  # how many generations up the upstream chain to load
     )
 
     downstreams = relationship(
diff --git a/models/psf.py b/models/psf.py
index e8272a1c..36dde2fe 100644
--- a/models/psf.py
+++ b/models/psf.py
@@ -66,6 +66,7 @@ def format( self, value ):
         'Image',
         cascade='save-update, merge, refresh-expire, expunge',
         passive_deletes=True,
+        lazy='selectin',
         doc="Image for which this is the PSF."
     )
 
@@ -167,6 +168,7 @@ def _get_inverse_badness(self):
 
     def __init__( self, *args, **kwargs ):
         FileOnDiskMixin.__init__( self, **kwargs )
+        HasBitFlagBadness.__init__(self)
         SeeChangeBase.__init__( self )
         self._header = None
         self._data = None
@@ -274,7 +276,7 @@ def load( self, download=True, always_verify_md5=False, psfpath=None, psfxmlpath
           psfpath : str or Path, default None
             If None, files will be read using the get_fullpath() method
             to get the right files form the local store and/or archive
-            given the databse fields.  If not None, read _header and
+            given the database fields.  If not None, read _header and
             _data from this file.  (This exists so that this method may
             be used to load the data with a psf that's not yet in the
             database, without having to play games with the filepath
@@ -305,7 +307,7 @@ def load( self, download=True, always_verify_md5=False, psfpath=None, psfxmlpath
             self._info = ifp.read()
 
     def free( self ):
-        """Free loaded world coordinates memory.
+        """Free loaded PSF memory.
 
         Wipe out the data, info, and header fields, freeing memory.
         Depends on python garbage collection, so if there are other
@@ -522,17 +524,18 @@ def add_psf_to_image( self, image, x, y, flux, norm=True, noisy=False, weight=No
                                               )
 
     def get_upstreams(self, session=None):
-        """Get the image that was used to make this source list. """
+        """Get the image that was used to make this PSF. """
         with SmartSession(session) as session:
             return session.scalars(sa.select(Image).where(Image.id == self.image_id)).all()
         
     def get_downstreams(self, session=None, siblings=False):
         """Get the downstreams of this PSF.
 
-        If siblings=True then also include the SourceLists, WCSes, ZPs and background objects
+        If siblings=True then also include the SourceList, WCS, ZP and background object
         that were created at the same time as this PSF.
         """
         from models.source_list import SourceList
+        from models.background import Background
         from models.world_coordinates import WorldCoordinates
         from models.zero_point import ZeroPoint
         from models.provenance import Provenance
@@ -559,7 +562,16 @@ def get_downstreams(self, session=None, siblings=False):
 
                 output.append(sources[0])
 
-                # TODO: add background object
+                bgs = session.scalars(
+                    sa.select(Background).where(
+                        Background.image_id == self.image_id,
+                        Background.provenance_id == self.provenance_id
+                    )
+                ).all()
+                if len(bgs) != 1:
+                    raise ValueError(f"Expected exactly one Background for SourceList {self.id}, but found {len(bgs)}")
+
+                output.append(bgs[0])
 
                 wcs = session.scalars(
                     sa.select(WorldCoordinates).where(WorldCoordinates.sources_id == sources.id)
diff --git a/models/reference.py b/models/reference.py
index 780411c2..e35bf6ee 100644
--- a/models/reference.py
+++ b/models/reference.py
@@ -6,6 +6,7 @@
 from models.provenance import Provenance
 from models.source_list import SourceList
 from models.psf import PSF
+from models.background import Background
 from models.world_coordinates import WorldCoordinates
 from models.zero_point import ZeroPoint
 
@@ -138,6 +139,7 @@ class Reference(Base, AutoIDMixin):
     def __init__(self, **kwargs):
         self.sources = None
         self.psf = None
+        self.bg = None
         self.wcs = None
         self.zp = None
         super().__init__(**kwargs)
@@ -150,6 +152,7 @@ def __setattr__(self, key, value):
             self.section_id = value.section_id
             self.sources = value.sources
             self.psf = value.psf
+            self.bg = value.bg
             self.wcs = value.wcs
             self.zp = value.zp
 
@@ -160,6 +163,7 @@ def init_on_load(self):
         Base.init_on_load(self)
         self.sources = None
         self.psf = None
+        self.bg = None
         self.wcs = None
         self.zp = None
         this_object_session = orm.Session.object_session(self)
@@ -169,7 +173,7 @@ def init_on_load(self):
     def make_provenance(self):
         """Make a provenance for this reference image. """
         upstreams = [self.image.provenance]
-        for att in ['image', 'sources', 'psf', 'wcs', 'zp']:
+        for att in ['image', 'sources', 'psf', 'bg', 'wcs', 'zp']:
             if getattr(self, att) is not None:
                 upstreams.append(getattr(self, att).provenance)
             else:
@@ -202,6 +206,8 @@ def get_upstream_provenances(self):
             prov.append(self.sources.provenance)
         if self.psf is not None and self.psf.provenance is not None and self.psf.provenance.id is not None:
             prov.append(self.psf.provenance)
+        if self.bg is not None and self.bg.provenance is not None and self.bg.provenance.id is not None:
+            prov.append(self.bg.provenance)
         if self.wcs is not None and self.wcs.provenance is not None and self.wcs.provenance.id is not None:
             prov.append(self.wcs.provenance)
         if self.zp is not None and self.zp.provenance is not None and self.zp.provenance.id is not None:
@@ -245,6 +251,20 @@ def load_upstream_products(self, session=None):
                 self.image.psf = psfs[0]
                 self.psf = psfs[0]
 
+            bgs = session.scalars(
+                sa.select(Background).where(
+                    Background.image_id == self.image.id,
+                    Background.provenance_id.in_(prov_ids),
+                )
+            ).all()
+            if len(bgs) > 1:
+                raise ValueError(
+                    f"Image {self.image_id} has more than one Background matching upstream provenance."
+                )
+            elif len(bgs) == 1:
+                self.image.bg = bgs[0]
+                self.bg = bgs[0]
+
             if self.sources is not None:
                 wcses = session.scalars(
                     sa.select(WorldCoordinates).where(
diff --git a/models/report.py b/models/report.py
index 6869b114..ead59d29 100644
--- a/models/report.py
+++ b/models/report.py
@@ -291,17 +291,32 @@ def __init__(self, **kwargs):
     def init_on_load(self):
         SeeChangeBase.init_on_load(self)
 
-    def scan_datastore(self, ds, process_step, session=None):
+    def scan_datastore(self, ds, process_step=None, session=None):
         """Go over all the data in a datastore and update the report accordingly.
-        Will commit the changes to the database.
+        Will commit the Report object to the database.
         If there are any exceptions pending on the datastore it will re-raise them.
+
+        Parameters
+        ----------
+        ds : DataStore
+            The datastore to scan for information.
+        process_step : str, optional
+            The name of the process step that was just completed.
+            This will be added to the progress bitflag.
+            If not given, will skip adding the progress flag,
+            but will also skip checking warnings and errors...
+            Use without a process_step just to update general
+            properties of the datastore like the runtime and memory usage,
+            or for updating the products_exist and products_committed bitflags
+            (e.g., after saving the datastore).
+        session : sqlalchemy.orm.Session, optional
+            The session to use for committing the changes to the database.
+            If not given, will open a session and close it at the end
+            of the function.
         """
         # parse the error, if it exists, so we can get to other data products without raising
         exception = ds.read_exception()
         
-        # append the newest step to the progress bitflag
-        self.append_progress(process_step)
-        
         # check which objects exist on the datastore, and which have been committed
         for prod in pipeline_products_dict.values():
             if getattr(ds, prod) is not None:
@@ -313,18 +328,23 @@ def scan_datastore(self, ds, process_step, session=None):
         self.process_runtime = ds.runtimes  # update with new dictionary
         self.process_memory = ds.memory_usages  # update with new dictionary
 
-        # parse the warnings, if they exist
-        if isinstance(ds.warnings_list, list):
-            new_string = self.read_warnings(process_step, ds.warnings_list)
-            if self.warnings is None or self.warnings == '':
-                self.warnings = new_string
-            else:
-                self.warnings += '\n***|***|***\n' + new_string
-        
-        if exception is not None:
-            self.error_type = exception.__class__.__name__
-            self.error_message = str(exception)
-            self.error_step = process_step
+        if process_step is not None:
+            # append the newest step to the progress bitflag
+            if process_step in process_steps_inverse:  # skip steps not in the dict
+                self.append_progress(process_step)
+
+            # parse the warnings, if they exist
+            if isinstance(ds.warnings_list, list):
+                new_string = self.read_warnings(process_step, ds.warnings_list)
+                if self.warnings is None or self.warnings == '':
+                    self.warnings = new_string
+                else:
+                    self.warnings += '\n***|***|***\n' + new_string
+
+            if exception is not None:
+                self.error_type = exception.__class__.__name__
+                self.error_message = str(exception)
+                self.error_step = process_step
 
         with SmartSession(session) as session:
             new_report = self.commit_to_database(session=session)
diff --git a/models/source_list.py b/models/source_list.py
index 2daf1bbd..8bc19bbc 100644
--- a/models/source_list.py
+++ b/models/source_list.py
@@ -91,23 +91,21 @@ def format(self, value):
         doc="Radius of apertures used for aperture photometry in pixels."
     )
 
-    _inf_aper_num = sa.Column(
+    inf_aper_num = sa.Column(
         sa.SMALLINT,
         nullable=True,
         default=None,
         index=False,
-        doc="Which element of aper_rads to use as the 'infinite' aperture; null = last one"
+        doc="Which element of aper_rads to use as the 'infinite' aperture; -1 = last one. "
     )
 
-    @property
-    def inf_aper_num( self ):
-        if self._inf_aper_num is None:
-            if self.aper_rads is None:
-                return None
-            else:
-                return len(self.aper_rads) - 1
-        else:
-            return self._inf_aper_num
+    best_aper_num = sa.Column(
+        sa.SMALLINT,
+        nullable=True,
+        default=None,
+        index=False,
+        doc="Which element of aper_rads to use as the 'best' aperture; -1 = use PSF photometry. "
+    )
 
     num_sources = sa.Column(
         sa.Integer,
@@ -144,6 +142,7 @@ def _get_inverse_badness(self):
 
     def __init__(self, *args, **kwargs):
         FileOnDiskMixin.__init__(self, *args, **kwargs)
+        HasBitFlagBadness.__init__(self)
         SeeChangeBase.__init__(self)  # don't pass kwargs as they could contain non-column key-values
 
         self._data = None
@@ -409,7 +408,7 @@ def apfluxadu( self, apnum=0, ap=None ):
           ap: float, default None
             If not None, look for an aperture that's within 0.01 pixels
             of this and return flux in apertures of that radius.  Raises
-            an exception if such an aperture doesn't apear in aper_rads
+            an exception if such an aperture doesn't appear in aper_rads
 
         Returns
         -------
@@ -420,7 +419,7 @@ def apfluxadu( self, apnum=0, ap=None ):
             raise NotImplementedError( f"Not currently implemented for format {self.format}" )
 
         if ap is None:
-            if ( self.aper_rads is None ) or ( apnum < 0 ) or ( apnum >= len(self.aper_rads) ):
+            if ( self.aper_rads is None ) or ( apnum >= len(self.aper_rads) ):
                 raise ValueError( f"Aperture radius number {apnum} doesn't exist." )
         else:
             w = np.where( np.abs( np.array( self.aper_rads) - ap ) < 0.01 )[0]
@@ -485,7 +484,7 @@ def calc_aper_cor( self, aper_num=0, inf_aper_num=None, min_stars=20 ):
             inf_aper_num = self.inf_aper_num
         if inf_aper_num is None:
             raise RuntimeError( f"Can't determine which aperture to use as the \"infinite\" aperture" )
-        if ( inf_aper_num < 0 ) or ( inf_aper_num >= len(self.aper_rads) ):
+        if inf_aper_num >= len(self.aper_rads):
             raise ValueError( f"inf_aper_num {inf_aper_num} is outside available list of {len(self.aper_rads)}" )
 
         bigflux, bigfluxerr = self.apfluxadu( apnum=inf_aper_num )
@@ -752,10 +751,11 @@ def get_upstreams(self, session=None):
     def get_downstreams(self, session=None, siblings=False):
         """Get all the data products that are made using this source list.
 
-        If siblings=True then also include the PSFs, WCSes, ZPs and background objects
+        If siblings=True then also include the PSF, Background, WCS, and ZP
         that were created at the same time as this SourceList.
         """
         from models.psf import PSF
+        from models.background import Background
         from models.world_coordinates import WorldCoordinates
         from models.zero_point import ZeroPoint
         from models.cutouts import Cutouts
@@ -780,7 +780,14 @@ def get_downstreams(self, session=None, siblings=False):
                 if len(psfs) != 1:
                     raise ValueError(f"Expected exactly one PSF for SourceList {self.id}, but found {len(psfs)}")
 
-                # TODO: add background object
+                bgs = session.scalars(
+                    sa.select(Background).where(
+                        Background.image_id == self.image_id,
+                        Background.provenance_id == self.provenance_id
+                    )
+                ).all()
+                if len(bgs) != 1:
+                    raise ValueError(f"Expected exactly one Background for SourceList {self.id}, but found {len(bgs)}")
 
                 wcs = session.scalars(sa.select(WorldCoordinates).where(WorldCoordinates.sources_id == self.id)).all()
                 if len(wcs) != 1:
@@ -792,7 +799,8 @@ def get_downstreams(self, session=None, siblings=False):
                     raise ValueError(
                         f"Expected exactly one ZeroPoint for SourceList {self.id}, but found {len(zps)}"
                     )
-                output += psfs + wcs + zps
+
+                output += psfs + bgs + wcs + zps
 
         return output
 
diff --git a/models/world_coordinates.py b/models/world_coordinates.py
index d4676115..7720a41e 100644
--- a/models/world_coordinates.py
+++ b/models/world_coordinates.py
@@ -75,6 +75,7 @@ def wcs( self, value ):
 
     def __init__(self, *args, **kwargs):
         FileOnDiskMixin.__init__( self, **kwargs )
+        HasBitFlagBadness.__init__(self)
         SeeChangeBase.__init__( self )
         self._wcs = None
 
@@ -106,11 +107,12 @@ def get_upstreams(self, session=None):
     def get_downstreams(self, session=None, siblings=False):
         """Get the downstreams of this WorldCoordinates.
 
-        If siblings=True  then also include the SourceLists, PSFs, ZPs and background objects
+        If siblings=True  then also include the SourceList, PSF, background object and ZP
         that were created at the same time as this WorldCoordinates.
         """
         from models.source_list import SourceList
         from models.psf import PSF
+        from models.background import Background
         from models.zero_point import ZeroPoint
         from models.provenance import Provenance
 
@@ -143,7 +145,18 @@ def get_downstreams(self, session=None, siblings=False):
 
                 output.append(psf[0])
 
-                # TODO: add background object
+                bgs = session.scalars(
+                    sa.select(Background).where(
+                        Background.image_id == sources.image_id, Background.provenance_id == self.provenance_id
+                    )
+                ).all()
+
+                if len(bgs) > 1:
+                    raise ValueError(
+                        f"Expected exactly one Background for WorldCoordinates {self.id}, but found {len(bgs)}."
+                    )
+
+                output.append(bgs[0])
 
                 zp = session.scalars(sa.select(ZeroPoint).where(ZeroPoint.sources_id == sources.id)).all()
 
@@ -223,4 +236,12 @@ def load( self, download=True, always_verify_md5=False, txtpath=None ):
         with open( txtpath ) as ifp:
             headertxt = ifp.read()
             self.wcs = WCS( fits.Header.fromstring( headertxt , sep='\\n' ))
-    
+
+    def free(self):
+        """Free loaded world coordinates memory.
+
+        Wipe out the _wcs text field, freeing a small amount of memory.
+        Depends on python garbage collection, so if there are other
+        references to those objects, the memory won't actually be freed.
+        """
+        self._wcs = None
diff --git a/models/zero_point.py b/models/zero_point.py
index 257a6dc8..0e8bdbcc 100644
--- a/models/zero_point.py
+++ b/models/zero_point.py
@@ -94,6 +94,7 @@ class ZeroPoint(Base, AutoIDMixin, HasBitFlagBadness):
     )
 
     def __init__(self, *args, **kwargs):
+        HasBitFlagBadness.__init__(self)
         SeeChangeBase.__init__(self)  # don't pass kwargs as they could contain non-column key-values
 
         # manually set all properties (columns or not)
@@ -145,19 +146,19 @@ def get_upstreams(self, session=None):
     def get_downstreams(self, session=None, siblings=False):
         """Get the downstreams of this ZeroPoint.
 
-        If siblings=True then also include the SourceLists, PSFs, WCSes, and background objects
+        If siblings=True then also include the SourceList, PSF, background object and WCS
         that were created at the same time as this ZeroPoint.
         """
         from models.source_list import SourceList
         from models.psf import PSF
+        from models.background import Background
         from models.world_coordinates import WorldCoordinates
         from models.provenance import Provenance
 
         with SmartSession(session) as session:
             subs = session.scalars(
                 sa.select(Image).where(
-                    Image.provenance.has(Provenance.upstreams.any(Provenance.id == self.provenance.id)),
-                    Image.upstream_images.any(Image.id == self.sources.image_id),
+                    Image.provenance.has(Provenance.upstreams.any(Provenance.id == self.provenance.id))
                 )
             ).all()
             output = subs
@@ -180,7 +181,18 @@ def get_downstreams(self, session=None, siblings=False):
 
                 output.append(psf[0])
 
-                # TODO: add background object
+                bgs = session.scalars(
+                    sa.select(Background).where(
+                        Background.image_id == sources.image_id, Background.provenance_id == self.provenance_id
+                    )
+                ).all()
+
+                if len(bgs) > 1:
+                    raise ValueError(
+                        f"Expected exactly one Background for WorldCoordinates {self.id}, but found {len(bgs)}."
+                    )
+
+                output.append(bgs[0])
 
                 wcs = session.scalars(
                     sa.select(WorldCoordinates).where(WorldCoordinates.sources_id == sources.id)
diff --git a/pipeline/backgrounding.py b/pipeline/backgrounding.py
new file mode 100644
index 00000000..590dcef3
--- /dev/null
+++ b/pipeline/backgrounding.py
@@ -0,0 +1,169 @@
+import os
+import time
+
+import numpy as np
+
+import sep
+
+from pipeline.parameters import Parameters
+from pipeline.data_store import DataStore
+
+from models.background import Background
+
+from util.logger import SCLogger
+from util.util import parse_bool
+
+
+class ParsBackgrounder(Parameters):
+    def __init__(self, **kwargs):
+        super().__init__()
+
+        self.format = self.add_par(
+            'format',
+            'map',
+            str,
+            'Format of the background image. Choose: "map", "scalar", or "polynomial". ',
+            critical=True
+        )
+
+        self.method = self.add_par(
+            'method',
+            'sep',
+            str,
+            'Method to use to estimate the background. Choose: "sep" or "zero". ',
+            critical=True
+        )
+
+        self.poly_order = self.add_par(
+            'poly_order',
+            1,
+            int,
+            'Order of the polynomial to fit to the background. ',
+            critical=True
+        )
+
+        self.sep_box_size = self.add_par(
+            'sep_box_size',
+            128,
+            int,
+            'Size of the box in pixels to use for the background estimation using sep. ',
+            critical=True
+        )
+
+        self.sep_filt_size = self.add_par(
+            'sep_filt_size',
+            3,
+            int,
+            'Size of the filter to use for the background estimation using sep. ',
+            critical=True
+        )
+
+        self._enforce_no_new_attrs = True
+
+        self.override(kwargs)
+
+    def get_process_name(self):
+        return 'backgrounding'
+
+
+class Backgrounder:
+    def __init__(self, **kwargs):
+        self.pars = ParsBackgrounder(**kwargs)
+
+        # this is useful for tests, where we can know if
+        # the object did any work or just loaded from DB or datastore
+        self.has_recalculated = False
+
+    def run(self, *args, **kwargs):
+        """Calculate the background for the given image.
+
+        Arguments are parsed by the DataStore.parse_args() method.
+        Returns a DataStore object with the products of the processing.
+        """
+        self.has_recalculated = False
+
+        try:  # first make sure we get back a datastore, even an empty one
+            ds, session = DataStore.from_args(*args, **kwargs)
+        except Exception as e:
+            return DataStore.catch_failure_to_parse(e, *args)
+
+        try:
+            t_start = time.perf_counter()
+            if parse_bool(os.getenv('SEECHANGE_TRACEMALLOC')):
+                import tracemalloc
+                tracemalloc.reset_peak()  # start accounting for the peak memory usage from here
+
+            self.pars.do_warning_exception_hangup_injection_here()
+
+            # get the provenance for this step:
+            prov = ds.get_provenance('extraction', self.pars.get_critical_pars(), session=session)
+
+            # try to find the background object in memory or in the database:
+            bg = ds.get_background(prov, session=session)
+
+            if bg is None:  # need to produce a background object
+                self.has_recalculated = True
+                image = ds.get_image(session=session)
+
+                if self.pars.method == 'sep':
+                    # Estimate the background mean and RMS with sep
+                    boxsize = self.pars.sep_box_size
+                    filtsize = self.pars.sep_filt_size
+                    SCLogger.debug("Subtracting sky and estimating sky RMS")
+                    # Dysfunctionality alert: sep requires a *float* image for the mask
+                    # IEEE 32-bit floats have 23 bits in the mantissa, so they should
+                    # be able to precisely represent a 16-bit integer mask image
+                    # In any event, sep.Background uses >0 as "bad"
+                    fmask = np.array(image._flags, dtype=np.float32)
+                    sep_bg_obj = sep.Background(image.data.copy(), mask=fmask,
+                                                  bw=boxsize, bh=boxsize, fw=filtsize, fh=filtsize)
+                    fmask = None
+                    bg = Background(
+                        value=float(np.nanmedian(sep_bg_obj.back())),
+                        noise=float(np.nanmedian(sep_bg_obj.rms())),
+                        counts=sep_bg_obj.back(),
+                        rms=sep_bg_obj.rms(),
+                        format='map',
+                        method='sep'
+                    )
+                elif self.pars.method == 'zero':  # don't measure the b/g
+                    bg = Background(value=0, noise=0, format='scalar', method='zero')
+                else:
+                    raise ValueError(f'Unknown background method "{self.pars.method}"')
+
+                bg.image_id = image.id
+                bg.image = image
+
+            if bg.provenance is None:
+                bg.provenance = prov
+            else:
+                if bg.provenance.id != prov.id:
+                    raise ValueError('Provenance mismatch for background and extraction provenance!')
+
+            # since these are "first look estimates" we don't update them if they are already set
+            if ds.image.bkg_mean_estimate is None and ds.image.bkg_rms_estimate is None:
+                ds.image.bkg_mean_estimate = float( bg.value )
+                ds.image.bkg_rms_estimate = float( bg.noise )
+
+            bg._upstream_bitflag = 0
+            bg._upstream_bitflag |= ds.image.bitflag
+
+            sources = ds.get_sources(session=session)
+            if sources is not None:
+                bg._upstream_bitflag |= sources.bitflag
+
+            psf = ds.get_psf(session=session)
+            if psf is not None:
+                bg._upstream_bitflag |= psf.bitflag
+
+            ds.bg = bg
+
+            ds.runtimes['backgrounding'] = time.perf_counter() - t_start
+            if parse_bool(os.getenv('SEECHANGE_TRACEMALLOC')):
+                import tracemalloc
+                ds.memory_usages['backgrounding'] = tracemalloc.get_traced_memory()[1] / 1024 ** 2  # in MB
+
+        except Exception as e:
+            ds.catch_exception(e)
+        finally:  # make sure datastore is returned to be used in the next step
+            return ds
diff --git a/pipeline/coaddition.py b/pipeline/coaddition.py
index 4082a119..aec26ed7 100644
--- a/pipeline/coaddition.py
+++ b/pipeline/coaddition.py
@@ -1,4 +1,3 @@
-
 import numpy as np
 from numpy.fft import fft2, ifft2, fftshift
 
@@ -15,6 +14,7 @@
 from pipeline.parameters import Parameters
 from pipeline.data_store import DataStore
 from pipeline.detection import Detector
+from pipeline.backgrounding import Backgrounder
 from pipeline.astro_cal import AstroCalibrator
 from pipeline.photo_cal import PhotCalibrator
 from util.util import get_latest_provenance, parse_session
@@ -272,7 +272,7 @@ def _coadd_zogy(
         ----------
         images: list of Image or list of 2D ndarrays
             Images that have been aligned to each other.
-            Each image must also have a PSF object attached.
+            Each image must also have a PSF and a background object attached.
         weights: list of 2D ndarrays
             The weights to use for each image.
             If images is given as Image objects, can be left as None.
@@ -291,12 +291,10 @@ def _coadd_zogy(
         bkg_means: list of floats
             The mean background for each image.
             If images is given as Image objects, can be left as None.
-            This variable can be used to override the background estimation.
             If images are already background subtracted, set these to zeros.
         bkg_sigmas: list of floats
             The RMS of the background for each image.
             If images is given as Image objects, can be left as None.
-            This variable can be used to override the background estimation.
 
         Returns
         -------
@@ -348,12 +346,10 @@ def _coadd_zogy(
 
         # estimate the background if not given
         if bkg_means is None or bkg_sigmas is None:
-            bkg_means = []
-            bkg_sigmas = []
-            for array in data:
-                bkg, sigma = self._estimate_background(array)
-                bkg_means.append(bkg)
-                bkg_sigmas.append(sigma)
+            if not isinstance(images[0], Image):
+                raise ValueError('Background must be given if images are not Image objects. ')
+            bkg_means = [im.bg.value for im in images]
+            bkg_sigmas = [im.bg.noise for im in images]
 
         imcube = np.array(data)
         flcube = np.array(flags)
@@ -496,6 +492,13 @@ def __init__(self, **kwargs):
         self.pars.add_defaults_to_dict(extraction_config)
         self.extractor = Detector(**extraction_config)
 
+        # background estimation
+        backgrounder_config = self.config.value('extraction.bg', {})
+        backgrounder_config.update(self.config.value('coaddition.extraction.bg', {}))  # override coadd specific pars
+        backgrounder_config.update(kwargs.get('extraction', {}).get('bg', {}))
+        self.pars.add_defaults_to_dict(backgrounder_config)
+        self.backgrounder = Backgrounder(**backgrounder_config)
+
         # astrometric fit using a first pass of sextractor and then astrometric fit to Gaia
         astrometor_config = self.config.value('extraction.wcs', {})
         astrometor_config.update(self.config.value('coaddition.extraction.wcs', {}))  # override coadd specific pars
@@ -511,8 +514,14 @@ def __init__(self, **kwargs):
         self.photometor = PhotCalibrator(**photometor_config)
 
         # make sure when calling get_critical_pars() these objects will produce the full, nested dictionary
-        siblings = {'sources': self.extractor.pars, 'wcs': self.astrometor.pars, 'zp': self.photometor.pars}
+        siblings = {
+            'sources': self.extractor.pars,
+            'bg': self.backgrounder.pars,
+            'wcs': self.astrometor.pars,
+            'zp': self.photometor.pars
+        }
         self.extractor.pars.add_siblings(siblings)
+        self.backgrounder.pars.add_siblings(siblings)
         self.astrometor.pars.add_siblings(siblings)
         self.photometor.pars.add_siblings(siblings)
 
@@ -640,6 +649,7 @@ def run(self, *args, **kwargs):
 
         # TODO: add the warnings/exception capturing, runtime/memory tracking (and Report making) as in top_level.py
         self.datastore = self.extractor.run(self.datastore)
+        self.datastore = self.backgrounder.run(self.datastore)
         self.datastore = self.astrometor.run(self.datastore)
         self.datastore = self.photometor.run(self.datastore)
 
diff --git a/pipeline/cutting.py b/pipeline/cutting.py
index a533abbc..704c28d1 100644
--- a/pipeline/cutting.py
+++ b/pipeline/cutting.py
@@ -84,7 +84,9 @@ def run(self, *args, **kwargs):
                 detections = ds.get_detections(session=session)
 
                 if detections is None:
-                    raise ValueError(f'Cannot find a source list corresponding to the datastore inputs: {ds.get_inputs()}')
+                    raise ValueError(
+                        f'Cannot find a source list corresponding to the datastore inputs: {ds.get_inputs()}'
+                    )
 
                 cutout_list = []
                 x = detections.x
diff --git a/pipeline/data_store.py b/pipeline/data_store.py
index 6f185526..addb4cb1 100644
--- a/pipeline/data_store.py
+++ b/pipeline/data_store.py
@@ -12,6 +12,7 @@
 from models.image import Image, image_upstreams_association_table
 from models.source_list import SourceList
 from models.psf import PSF
+from models.background import Background
 from models.world_coordinates import WorldCoordinates
 from models.zero_point import ZeroPoint
 from models.reference import Reference
@@ -56,6 +57,7 @@ class DataStore:
         'image',
         'sources',
         'psf',
+        'bg',
         'wcs',
         'zp',
         'sub_image',
@@ -271,6 +273,7 @@ def __init__(self, *args, **kwargs):
         self.image = None  # single image from one sensor section
         self.sources = None  # extracted sources (a SourceList object, basically a catalog)
         self.psf = None  # psf determined from the extracted sources
+        self.bg = None  # background from the extraction phase
         self.wcs = None  # astrometric solution
         self.zp = None  # photometric calibration
         self.reference = None  # the Reference object needed to make subtractions
@@ -365,6 +368,12 @@ def __setattr__(self, key, value):
             if key == 'sources' and not isinstance(value, SourceList):
                 raise ValueError(f'sources must be a SourceList object, got {type(value)}')
 
+            if key == 'psf' and not isinstance(value, PSF):
+                raise ValueError(f'psf must be a PSF object, got {type(value)}')
+
+            if key == 'bg' and not isinstance(value, Background):
+                raise ValueError(f'bg must be a Background object, got {type(value)}')
+
             if key == 'wcs' and not isinstance(value, WorldCoordinates):
                 raise ValueError(f'WCS must be a WorldCoordinates object, got {type(value)}')
 
@@ -661,7 +670,7 @@ def get_image(self, provenance=None, session=None):
                             sa.select(Image).where(
                                 Image.exposure_id == self.exposure_id,
                                 Image.section_id == str(self.section_id),
-                                Image.provenance.has(id=provenance.id)
+                                Image.provenance_id == provenance.id,
                             )
                         ).first()
 
@@ -673,7 +682,7 @@ def append_image_products(self, image):
         pipeline applications, to make sure the image
         object has all the data products it needs.
         """
-        for att in ['sources', 'psf', 'wcs', 'zp', 'detections', 'cutouts', 'measurements']:
+        for att in ['sources', 'psf', 'bg', 'wcs', 'zp', 'detections', 'cutouts', 'measurements']:
             if getattr(self, att, None) is not None:
                 setattr(image, att, getattr(self, att))
         if image.sources is not None:
@@ -732,7 +741,7 @@ def get_sources(self, provenance=None, session=None):
                         sa.select(SourceList).where(
                             SourceList.image_id == image.id,
                             SourceList.is_sub.is_(False),
-                            SourceList.provenance.has(id=provenance.id),
+                            SourceList.provenance_id == provenance.id,
                         )
                     ).first()
 
@@ -749,8 +758,6 @@ def get_psf(self, provenance=None, session=None):
             the current code version and critical parameters.
             If none is given, uses the appropriate provenance
             from the prov_tree dictionary.
-            If prov_tree is None, will use the latest provenance
-            for the "extraction" process.
             Usually the provenance is not given when the psf is loaded
             in order to be used as an upstream of the current process.
         session: sqlalchemy.orm.session.Session
@@ -786,11 +793,65 @@ def get_psf(self, provenance=None, session=None):
                 image = self.get_image(session=session)
                 if image is not None:
                     self.psf = session.scalars(
-                        sa.select(PSF).where(PSF.image_id == image.id, PSF.provenance.has(id=provenance.id))
+                        sa.select(PSF).where(PSF.image_id == image.id, PSF.provenance_id == provenance.id)
                     ).first()
 
         return self.psf
 
+    def get_background(self, provenance=None, session=None):
+        """Get a Background object, either from memory or from the database.
+
+        Parameters
+        ----------
+        provenance: Provenance object
+            The provenance to use for the background.
+            This provenance should be consistent with
+            the current code version and critical parameters.
+            If none is given, uses the appropriate provenance
+            from the prov_tree dictionary.
+            Usually the provenance is not given when the background is loaded
+            in order to be used as an upstream of the current process.
+        session: sqlalchemy.orm.session.Session
+            An optional session to use for the database query.
+            If not given, will use the session stored inside the
+            DataStore object; if there is none, will open a new session
+            and close it at the end of the function.
+
+        Returns
+        -------
+        bg: Background object
+            The background object for this image,
+            or None if no matching background is found.
+
+        """
+        process_name = 'extraction'
+        if provenance is None:  # try to get the provenance from the prov_tree
+            provenance = self._get_provenance_for_an_upstream(process_name, session)
+
+        # if background exists in memory, check the provenance is ok
+        if self.bg is not None:
+            # make sure the background object has the correct provenance
+            if self.bg.provenance is None:
+                raise ValueError('Background has no provenance!')
+            if provenance is not None and provenance.id != self.bg.provenance.id:
+                self.bg = None
+
+        # TODO: do we need to test the b/g Provenance has upstreams consistent with self.image.provenance?
+
+        # not in memory, look for it on the DB
+        if self.bg is None:
+            with SmartSession(session, self.session) as session:
+                image = self.get_image(session=session)
+                if image is not None:
+                    self.bg = session.scalars(
+                        sa.select(Background).where(
+                            Background.image_id == image.id,
+                            Background.provenance_id == provenance.id,
+                        )
+                    ).first()
+
+        return self.bg
+
     def get_wcs(self, provenance=None, session=None):
         """Get an astrometric solution in the form of a WorldCoordinates object, from memory or from the database.
 
@@ -840,7 +901,7 @@ def get_wcs(self, provenance=None, session=None):
                 if sources is not None and sources.id is not None:
                     self.wcs = session.scalars(
                         sa.select(WorldCoordinates).where(
-                            WorldCoordinates.sources_id == sources.id, WorldCoordinates.provenance.has(id=provenance.id)
+                            WorldCoordinates.sources_id == sources.id, WorldCoordinates.provenance_id == provenance.id
                         )
                     ).first()
 
@@ -895,7 +956,7 @@ def get_zp(self, provenance=None, session=None):
                 if sources is not None and sources.id is not None:
                     self.zp = session.scalars(
                         sa.select(ZeroPoint).where(
-                            ZeroPoint.sources_id == sources.id, ZeroPoint.provenance.has(id=provenance.id)
+                            ZeroPoint.sources_id == sources.id, ZeroPoint.provenance_id == provenance.id
                         )
                     ).first()
 
@@ -1134,7 +1195,7 @@ def get_subtraction(self, provenance=None, session=None):
                             aliased_table.c.upstream_id == image.id,
                             aliased_table.c.downstream_id == Image.id,
                         )
-                    ).where(Image.provenance.has(id=provenance.id))
+                    ).where(Image.provenance_id == provenance.id)
                 ).first()
 
         if self.sub_image is not None:
@@ -1189,7 +1250,7 @@ def get_detections(self, provenance=None, session=None):
                     sa.select(SourceList).where(
                         SourceList.image_id == sub_image.id,
                         SourceList.is_sub.is_(True),
-                        SourceList.provenance.has(id=provenance.id),
+                        SourceList.provenance_id == provenance.id,
                     )
                 ).first()
 
@@ -1234,7 +1295,7 @@ def get_cutouts(self, provenance=None, session=None):
                 if self.cutouts[0].provenance is None:
                     raise ValueError('Cutouts have no provenance!')
                 if provenance is not None and provenance.id != self.cutouts[0].provenance.id:
-                    self.detections = None
+                    self.cutouts = None
 
         # not in memory, look for it on the DB
         if self.cutouts is None:
@@ -1253,7 +1314,7 @@ def get_cutouts(self, provenance=None, session=None):
                 self.cutouts = session.scalars(
                     sa.select(Cutouts).where(
                         Cutouts.sources_id == sub_image.sources.id,
-                        Cutouts.provenance.has(id=provenance.id),
+                        Cutouts.provenance_id == provenance.id,
                     )
                 ).all()
 
@@ -1304,7 +1365,7 @@ def get_measurements(self, provenance=None, session=None):
                 self.measurements = session.scalars(
                     sa.select(Measurements).where(
                         Measurements.cutouts_id.in_(cutout_ids),
-                        Measurements.provenance.has(id=provenance.id),
+                        Measurements.provenance_id == provenance.id,
                     )
                 ).all()
 
@@ -1335,7 +1396,7 @@ def get_all_data_products(self, output='dict', omit_exposure=False):
             no nested). Any None values will be removed.
         """
         attributes = [] if omit_exposure else [ '_exposure' ]
-        attributes.extend( [ 'image', 'wcs', 'sources', 'psf', 'zp', 'sub_image',
+        attributes.extend( [ 'image', 'wcs', 'sources', 'psf', 'bg', 'zp', 'sub_image',
                              'detections', 'cutouts', 'measurements' ] )
         result = {att: getattr(self, att) for att in attributes}
         if output == 'dict':
@@ -1430,7 +1491,7 @@ def save_and_commit(self, exists_ok=False, overwrite=True, no_archive=False,
                 continue
 
             SCLogger.debug( f'save_and_commit considering a {obj.__class__.__name__} with filepath '
-                           f'{obj.filepath if isinstance(obj,FileOnDiskMixin) else "<none>"}' )
+                            f'{obj.filepath if isinstance(obj,FileOnDiskMixin) else "<none>"}' )
 
             if isinstance(obj, FileOnDiskMixin):
                 mustsave = True
@@ -1473,7 +1534,7 @@ def save_and_commit(self, exists_ok=False, overwrite=True, no_archive=False,
         with SmartSession(session, self.session) as session:
             if self.image is not None:
                 self.image = self.image.merge_all(session)
-                for att in ['sources', 'psf', 'wcs', 'zp']:
+                for att in ['sources', 'psf', 'bg', 'wcs', 'zp']:
                     setattr(self, att, None)  # avoid automatically appending to the image self's non-merged products
                 for att in ['exposure', 'sources', 'psf', 'wcs', 'zp']:
                     if getattr(self.image, att, None) is not None:
@@ -1485,13 +1546,14 @@ def save_and_commit(self, exists_ok=False, overwrite=True, no_archive=False,
             if self.image_id is None and self.image is not None:
                 self.image_id = self.image.id
 
-            self.psf = self.image.psf
             self.sources = self.image.sources
+            self.psf = self.image.psf
+            self.bg = self.image.bg
             self.wcs = self.image.wcs
             self.zp = self.image.zp
 
             session.commit()
-            self.products_committed = 'image, sources, psf, wcs, zp'
+            self.products_committed = 'image, sources, psf, wcs, zp, bg'
 
             if self.sub_image is not None:
                 if self.reference is not None:
@@ -1501,17 +1563,20 @@ def save_and_commit(self, exists_ok=False, overwrite=True, no_archive=False,
                 self.sub_image.ref_image.id = self.sub_image.ref_image_id
                 self.detections = self.sub_image.sources
 
-            session.commit()
-            self.products_committed += ', sub_image'
+                session.commit()
+                self.products_committed += ', sub_image'
 
             if self.detections is not None:
+                more_products = 'detections'
                 if self.cutouts is not None:
                     if self.measurements is not None:  # keep track of which cutouts goes to which measurements
                         for m in self.measurements:
-                            m._cutouts_list_index = self.cutouts.index(m.cutouts)
+                            idx = [c.index_in_sources for c in self.cutouts].index(m.cutouts.index_in_sources)
+                            m._cutouts_list_index = idx
                     for cutout in self.cutouts:
                         cutout.sources = self.detections
                     self.cutouts = Cutouts.merge_list(self.cutouts, session)
+                    more_products += ', cutouts'
 
                 if self.measurements is not None:
                     for i, m in enumerate(self.measurements):
@@ -1520,9 +1585,10 @@ def save_and_commit(self, exists_ok=False, overwrite=True, no_archive=False,
                         self.measurements[i].associate_object(session)
                         self.measurements[i] = session.merge(self.measurements[i])
                         self.measurements[i].object.measurements.append(self.measurements[i])
+                    more_products += ', measurements'
 
-            session.commit()
-            self.products_committed += ', detections, cutouts, measurements'
+                session.commit()
+                self.products_committed += ', ' + more_products
 
     def delete_everything(self, session=None, commit=True):
         """Delete everything associated with this sub-image.
diff --git a/pipeline/detection.py b/pipeline/detection.py
index f7f8629f..85307c7e 100644
--- a/pipeline/detection.py
+++ b/pipeline/detection.py
@@ -3,6 +3,7 @@
 import random
 import subprocess
 import time
+import warnings
 
 import numpy as np
 import numpy.lib.recfunctions as rfn
@@ -23,8 +24,9 @@
 
 from models.base import FileOnDiskMixin, CODE_ROOT
 from models.image import Image
-from models.psf import PSF
 from models.source_list import SourceList
+from models.psf import PSF
+from models.background import Background
 
 from improc.tools import sigma_clipping
 
@@ -47,33 +49,72 @@ def __init__(self, **kwargs):
             critical=True
         )
 
-        self.psf = self.add_par(
-            'psf',
-            None,
-            ( PSF, int, None ),
-            'Use this PSF; pass the PSF object, or its integer id. '
-            'If None, will not do PSF photometry.  Ignored if measure_psf is True.' ,
+        self.background_format = self.add_par(
+            'background_format',
+            'map',
+            str,
+            'Format of the background; one of "map", "scalar", or "polynomial".',
+            critical=True
+        )
+
+        self.background_order = self.add_par(
+            'background_order',
+            2,
+            int,
+            'Order of the polynomial background. Ignored unless background is "polynomial".',
+            critical=True
+        )
+
+        self.background_method = self.add_par(
+            'background_method',
+            'sep',
+            str,
+            'Method to use for background subtraction; currently only "sep" is supported.',
+            critical=True
+        )
+
+        self.background_box_size = self.add_par(
+            'background_box_size',
+            128,
+            int,
+            'Size of the box to use for background estimation in sep.',
+            critical=True
+        )
+
+        self.background_filt_size = self.add_par(
+            'background_filt_size',
+            3,
+            int,
+            'Size of the filter to use for background estimation in sep.',
             critical=True
         )
 
         self.apers = self.add_par(
             'apers',
-            None,
-            ( None, list ),
-            'Apertures in which to measure photometry; a list of floats or None',
+            [1.0, 2.0, 3.0, 5.0],
+            list,
+            'Apertures in which to measure photometry; a list of floats. ',
             critical=True
         )
         self.add_alias( 'apertures', 'apers' )
 
         self.inf_aper_num = self.add_par(
             'inf_aper_num',
-            None,
-            ( None, int ),
-            ( 'Which of apers is the one to use as the "infinite" aperture for aperture corrections; '
-              'default is to use the last one.  Ignored if self.apers is None.' ),
+            -1,
+            int,
+            'Which of apers is the one to use as the "infinite" aperture for aperture corrections. '
+            'If -1, will use the last aperture, not the PSF flux! ',
             critical=True
         )
 
+        self.best_aper_num = self.add_par(
+            'best_aper_num',
+            0,
+            int,
+            'Which of apers is the one to use as the "best" aperture, for things like plotting or calculating'
+            'the limiting magnitude. Note that -1 will use the PSF flux, not the last aperture on the list. '
+        )
+
         self.aperunit = self.add_par(
             'aperunit',
             'fwhm',
@@ -307,6 +348,7 @@ def run(self, *args, **kwargs):
                         raise ValueError(f'Cannot find an image corresponding to the datastore inputs: {ds.get_inputs()}')
 
                     sources, psf, bkg, bkgsig = self.extract_sources( image )
+
                     sources.image = image
                     if sources.provenance is None:
                         sources.provenance = prov
@@ -319,14 +361,11 @@ def run(self, *args, **kwargs):
                         psf.provenance = prov
                     else:
                         if psf.provenance.id != prov.id:
-                            raise ValueError('Provenance mismatch for pfs and provenance!')
+                            raise ValueError('Provenance mismatch for PSF and extraction provenance!')
 
                 ds.sources = sources
                 ds.psf = psf
                 ds.image.fwhm_estimate = psf.fwhm_pixels  # TODO: should we only write if the property is None?
-                if self.has_recalculated:
-                    ds.image.bkg_mean_estimate = float( bkg )
-                    ds.image.bkg_rms_estimate = float( bkgsig )
 
                 ds.runtimes['extraction'] = time.perf_counter() - t_start
                 if parse_bool(os.getenv('SEECHANGE_TRACEMALLOC')):
@@ -339,7 +378,25 @@ def run(self, *args, **kwargs):
                 return ds
 
     def extract_sources(self, image):
-        """Calls one of the extraction methods, based on self.pars.method. """
+        """Calls one of the extraction methods, based on self.pars.method.
+
+        Parameters
+        ----------
+        image: Image
+          The Image object from which to extract sources.
+
+        Returns
+        -------
+        sources: SourceList object
+            A list of sources with their positions and fluxes.
+        psf: PSF object
+            An estimate for the point spread function of the image.
+        bkg: float
+            An estimate for the mean value of the background of the image.
+        bkgsig: float
+            An estimate for the standard deviation of the background of the image.
+
+        """
         sources = None
         psf = None
         bkg = None
@@ -348,8 +405,7 @@ def extract_sources(self, image):
             sources = self.extract_sources_sep(image)
         elif self.pars.method == 'sextractor':
             if self.pars.subtraction:
-                psffile = None if self.pars.psf is None else self.pars.psf.get_fullpath()
-                sources, _, _, _ = self.extract_sources_sextractor(image, psffile=psffile)
+                sources, _, _, _ = self.extract_sources_sextractor(image, psffile=None)
             else:
                 sources, psf, bkg, bkgsig = self.extract_sources_sextractor(image)
         elif self.pars.method == 'filter':
@@ -360,14 +416,10 @@ def extract_sources(self, image):
         else:
             raise ValueError(f'Unknown extraction method "{self.pars.method}"')
 
-        if psf is not None:
-            if psf._upstream_bitflag is None:
-                psf._upstream_bitflag = 0
-            psf._upstream_bitflag |= image.bitflag
         if sources is not None:
-            if sources._upstream_bitflag is None:
-                sources._upstream_bitflag = 0
             sources._upstream_bitflag |= image.bitflag
+        if psf is not None:
+            psf._upstream_bitflag |= image.bitflag
 
         return sources, psf, bkg, bkgsig
 
@@ -378,14 +430,7 @@ def extract_sources_sextractor( self, image, psffile=None ):
         psfxmlpath = None
 
         try:  # cleanup at the end
-            if self.pars.apers is None:
-                apers = np.array( image.instrument_object.standard_apertures() )
-                inf_aper_num = image.instrument_object.fiducial_aperture()
-            else:
-                apers = self.pars.apers
-                inf_aper_num = self.pars.inf_aper_num
-                if inf_aper_num is None:
-                    inf_aper_num = len( apers ) - 1
+            apers = np.array(self.pars.apers)
 
             if self.pars.measure_psf:
                 # Run sextractor once without a psf to get objects from
@@ -408,9 +453,6 @@ def extract_sources_sextractor( self, image, psffile=None ):
                 psf = self._run_psfex( tempnamebase, image, do_not_cleanup=True )
                 psfpath = pathlib.Path( FileOnDiskMixin.temp_path ) / f'{tempnamebase}.sources.psf'
                 psfxmlpath = pathlib.Path( FileOnDiskMixin.temp_path ) / f'{tempnamebase}.sources.psf.xml'
-            elif self.pars.psf is not None:
-                psf = self.pars.psf
-                psfpath, psfxmlpath = psf.get_fullpath()
             else:
                 psf = None
 
@@ -424,28 +466,43 @@ def extract_sources_sextractor( self, image, psffile=None ):
             # Now that we have a psf, run sextractor (maybe a second time)
             # to get the actual measurements.
             SCLogger.debug( "detection: running sextractor with psf to get final source list" )
-            sources, bkg, bkgsig = self._run_sextractor_once( image, apers=apers,
-                                                              psffile=psfpath, tempname=tempnamebase )
+
+            if psf is not None:
+                psf_clip = psf.get_clip()
+                psf_norm = 1 / np.sqrt(np.sum(psf_clip ** 2))  # normalization factor for the sextractor thresholds
+            else:  # we don't have a psf for some reason, use the "good enough" approximation
+                psf_norm = 3.0
+
+            sources, bkg, bkgsig = self._run_sextractor_once(
+                image,
+                apers=apers,
+                psffile=psfpath,
+                psfnorm=psf_norm,
+                tempname=tempnamebase,
+            )
             SCLogger.debug( f"detection: sextractor found {len(sources.data)} sources" )
 
             snr = sources.apfluxadu()[0] / sources.apfluxadu()[1]
             if snr.min() > self.pars.threshold:
-                SCLogger.warning( "SExtractor may not have detected everything down to your threshold." )
+                warnings.warn( "SExtractor may not have detected everything down to your threshold." )
             w = np.where( snr >= self.pars.threshold )
             sources.data = sources.data[w]
             sources.num_sources = len( sources.data )
-            sources._inf_aper_num = inf_aper_num
+            sources.inf_aper_num = self.pars.inf_aper_num
+            sources.best_aper_num = self.pars.best_aper_num
 
         finally:
             # Clean up the temporary files created (that weren't already cleaned up by _run_sextractor_once)
             sourcepath.unlink( missing_ok=True )
-            if ( psffile is None ) and ( self.pars.psf is None ):
-                if psfpath is not None: psfpath.unlink( missing_ok=True )
-                if psfxmlpath is not None: psfxmlpath.unlink( missing_ok=True )
+            if psffile is None:
+                if psfpath is not None:
+                    psfpath.unlink( missing_ok=True )
+                if psfxmlpath is not None:
+                    psfxmlpath.unlink( missing_ok=True )
 
         return sources, psf, bkg, bkgsig
 
-    def _run_sextractor_once( self, image, apers=[5, ], psffile=None, tempname=None, do_not_cleanup=False ):
+    def _run_sextractor_once(self, image, apers=[5, ], psffile=None, psfnorm=3.0, tempname=None, do_not_cleanup=False):
         """Extract a SourceList from a FITS image using SExtractor.
 
         This function should not be called from outside this class.
@@ -463,6 +520,12 @@ def _run_sextractor_once( self, image, apers=[5, ], psffile=None, tempname=None,
             File that has the PSF to use for PSF photometry.  If None,
             won't do psf photometry.
 
+          psfnorm: float
+            The normalization of the PSF image (i.e., the sqrt of the
+            sum of squares of the psf values).  This is used to set the
+            threshold for sextractor.  When the PSF is not known, we
+            will use a rough approximation and set this value to 3.0.
+
           tempname: str
             If not None, a filename base for where the catalog will be
             written.  The source file will be written to
@@ -619,8 +682,8 @@ def _run_sextractor_once( self, image, apers=[5, ], psffile=None, tempname=None,
                      "-XML_NAME", tmpxml,
                      "-PARAMETERS_NAME", paramfile,
                      "-THRESH_TYPE", "RELATIVE",
-                     "-DETECT_THRESH", str( self.pars.threshold / 3. ),
-                     "-ANALYSIS_THRESH", str( self.pars.threshold / 3. ),
+                     "-DETECT_THRESH", str( self.pars.threshold / psfnorm ),
+                     "-ANALYSIS_THRESH", str( self.pars.threshold / psfnorm ),
                      "-FILTER", "Y",
                      "-FILTER_NAME", str(conv),
                      "-WEIGHT_TYPE", "MAP_WEIGHT",
@@ -631,7 +694,7 @@ def _run_sextractor_once( self, image, apers=[5, ], psffile=None, tempname=None,
                      "-FLAG_TYPE", "OR",
                      "-PHOT_APERTURES", ",".join( [ str(a*2.) for a in apers ] ),
                      "-SATUR_LEVEL", str( image.instrument_object.average_saturation_limit( image ) ),
-                     "-GAIN", "1.0",
+                     "-GAIN", "1.0",  # TODO: we should probably put the instrument gain here
                      "-STARNNW_NAME", nnw,
                      "-BACK_TYPE", "AUTO",
                      "-BACK_SIZE", str( image.instrument_object.background_box_size ),
diff --git a/pipeline/measuring.py b/pipeline/measuring.py
index 33566609..aefedccf 100644
--- a/pipeline/measuring.py
+++ b/pipeline/measuring.py
@@ -39,15 +39,11 @@ def __init__(self, **kwargs):
             'adjust the annulus size for each image based on the PSF width. '
         )
 
-        # TODO: should we choose the "best aperture" using the config, or should each Image have its own aperture?
-        self.chosen_aperture = self.add_par(
-            'chosen_aperture',
-            0,
-            [str, int],
-            'The aperture radius that is used for photometry. '
-            'Choose either the index in the aperture_radii list, '
-            'the string "psf", or the string "auto" to choose '
-            'the best aperture in each image separately. '
+        self.use_annulus_for_centroids = self.add_par(
+            'use_annulus_for_centroids',
+            True,
+            bool,
+            'Use the local background measurements via an annulus to adjust the centroids and second moments. '
         )
 
         self.analytical_cuts = self.add_par(
@@ -217,6 +213,7 @@ def run(self, *args, **kwargs):
                     # make sure to remember which cutout belongs to this measurement,
                     # before either of them is in the DB and then use the cutouts_id instead
                     m._cutouts_list_index = i
+                    m.best_aperture = c.sources.best_aper_num
 
                     m.aper_radii = c.sources.image.new_image.zp.aper_cor_radii  # zero point corrected aperture radii
 
@@ -239,14 +236,16 @@ def run(self, *args, **kwargs):
                         flags,
                         radii=m.aper_radii,
                         annulus=annulus_radii_pixels,
+                        local_bg=self.pars.use_annulus_for_centroids,
                     )
 
                     m.flux_apertures = output['fluxes']
                     m.flux_apertures_err = [np.sqrt(output['variance']) * norm for norm in output['normalizations']]
                     m.aper_radii = output['radii']
                     m.area_apertures = output['areas']
-                    m.background = output['background']
-                    m.background_err = np.sqrt(output['variance'])
+                    m.bkg_mean = output['background']
+                    m.bkg_std = np.sqrt(output['variance'])
+                    m.bkg_pix = output['n_pix_bg']
                     m.offset_x = output['offset_x']
                     m.offset_y = output['offset_y']
                     m.width = (output['major'] + output['minor']) / 2
@@ -288,29 +287,16 @@ def run(self, *args, **kwargs):
                     m.flux_psf_err = fluxerr
                     m.area_psf = area
 
-                    # decide on the "best" aperture
-                    if self.pars.chosen_aperture == 'auto':
-                        raise NotImplementedError('Automatic aperture selection is not yet implemented.')
-                    if self.pars.chosen_aperture == 'psf':
-                        ap_index = -1
-                    elif isinstance(self.pars.chosen_aperture, int):
-                        ap_index = self.pars.chosen_aperture
-                    else:
-                        raise ValueError(
-                            f'Invalid value "{self.pars.chosen_aperture}" for chosen_aperture in the measuring parameters.'
-                        )
-                    m.best_aperture = ap_index
-
                     # update the provenance
                     m.provenance = prov
                     m.provenance_id = prov.id
 
                     # Apply analytic cuts to each stamp image, to rule out artefacts.
                     m.disqualifier_scores = {}
-                    if m.background != 0 and m.background_err > 0.1:
-                        norm_data = (c.sub_nandata - m.background) / m.background_err  # normalize
+                    if m.bkg_mean != 0 and m.bkg_std > 0.1:
+                        norm_data = (c.sub_nandata - m.bkg_mean) / m.bkg_std  # normalize
                     else:
-                        warnings.warn(f'Background mean= {m.background}, std= {m.background_err}, normalization skipped!')
+                        warnings.warn(f'Background mean= {m.bkg_mean}, std= {m.bkg_std}, normalization skipped!')
                         norm_data = c.sub_nandata  # no good background measurement, do not normalize!
 
                     positives = np.sum(norm_data > self.pars.outlier_sigma)
diff --git a/pipeline/subtraction.py b/pipeline/subtraction.py
index 357efa2c..f66318bc 100644
--- a/pipeline/subtraction.py
+++ b/pipeline/subtraction.py
@@ -155,21 +155,12 @@ def _subtract_zogy(self, new_image, ref_image):
         ref_image_data = ref_image.data
         new_image_psf = new_image.psf.get_clip()
         ref_image_psf = ref_image.psf.get_clip()
-        new_image_noise = new_image.bkg_rms_estimate  # TOOD: improve this by using a Background object?
-        ref_image_noise = 1.0  # proper coaddition images have noise=1.0 by construction
+        new_image_noise = new_image.bkg_rms_estimate
+        ref_image_noise = ref_image.bkg_rms_estimate
         new_image_flux_zp = 10 ** (0.4 * new_image.zp.zp)
         ref_image_flux_zp = 10 ** (0.4 * ref_image.zp.zp)
         # TODO: consider adding an estimate for the astrometric uncertainty dx, dy
 
-        # do some additional processing of the new image
-        mu, sigma = sigma_clipping(new_image_data)
-        new_image_data = (new_image_data - mu) / sigma  # TODO: skip this if we already background subtracted
-        if new_image_noise is not None:
-            new_image_noise = new_image_noise / sigma
-        else:
-            new_image_noise = 1.0  # TODO: this can go away after we verify images always have background estimates!
-        new_image_flux_zp = new_image_flux_zp / sigma
-
         new_image_data = self.inpainter.run(new_image_data, new_image.flags, new_image.weight)
 
         output = zogy_subtract(
diff --git a/pipeline/top_level.py b/pipeline/top_level.py
index aecd0689..9ed8ba2a 100644
--- a/pipeline/top_level.py
+++ b/pipeline/top_level.py
@@ -1,5 +1,6 @@
 import os
 import datetime
+import time
 import warnings
 
 import sqlalchemy as sa
@@ -7,6 +8,7 @@
 from pipeline.parameters import Parameters
 from pipeline.data_store import DataStore, UPSTREAM_STEPS
 from pipeline.preprocessing import Preprocessor
+from pipeline.backgrounding import Backgrounder
 from pipeline.astro_cal import AstroCalibrator
 from pipeline.photo_cal import PhotCalibrator
 from pipeline.subtraction import Subtractor
@@ -33,7 +35,7 @@
     'extraction': {
         'sources': 'extractor',
         'psf': 'extractor',
-        'background': 'extractor',
+        'bg': 'backgrounder',
         'wcs': 'astrometor',
         'zp': 'photometor',
     },
@@ -52,7 +54,18 @@ def __init__(self, **kwargs):
         super().__init__()
 
         self.example_pipeline_parameter = self.add_par(
-            'example_pipeline_parameter', 1, int, 'an example pipeline parameter'
+            'example_pipeline_parameter', 1, int, 'an example pipeline parameter', critical=False
+        )
+
+        self.save_before_subtraction = self.add_par(
+            'save_before_subtraction',
+            True,
+            bool,
+            'Save intermediate images to the database, '
+            'after doing extraction, background, and astro/photo calibration, '
+            'if there is no reference, will not continue to doing subtraction'
+            'but will still save the products up to that point. ',
+            critical=False,
         )
 
         self._enforce_no_new_attrs = True  # lock against new parameters
@@ -81,6 +94,12 @@ def __init__(self, **kwargs):
         self.pars.add_defaults_to_dict(extraction_config)
         self.extractor = Detector(**extraction_config)
 
+        # background estimation using either sep or other methods
+        background_config = self.config.value('extraction.bg', {})
+        background_config.update(kwargs.get('extraction', {}).get('bg', {}))
+        self.pars.add_defaults_to_dict(background_config)
+        self.backgrounder = Backgrounder(**background_config)
+
         # astrometric fit using a first pass of sextractor and then astrometric fit to Gaia
         astrometor_config = self.config.value('extraction.wcs', {})
         astrometor_config.update(kwargs.get('extraction', {}).get('wcs', {}))
@@ -94,8 +113,14 @@ def __init__(self, **kwargs):
         self.photometor = PhotCalibrator(**photometor_config)
 
         # make sure when calling get_critical_pars() these objects will produce the full, nested dictionary
-        siblings = {'sources': self.extractor.pars, 'wcs': self.astrometor.pars, 'zp': self.photometor.pars}
+        siblings = {
+            'sources': self.extractor.pars,
+            'bg': self.backgrounder.pars,
+            'wcs': self.astrometor.pars,
+            'zp': self.photometor.pars,
+        }
         self.extractor.pars.add_siblings(siblings)
+        self.backgrounder.pars.add_siblings(siblings)
         self.astrometor.pars.add_siblings(siblings)
         self.photometor.pars.add_siblings(siblings)
 
@@ -254,7 +279,7 @@ def run(self, *args, **kwargs):
         with warnings.catch_warnings(record=True) as w:
             ds.warnings_list = w  # appends warning to this list as it goes along
             # run dark/flat preprocessing, cut out a specific section of the sensor
-            # TODO: save the results as Image objects to DB and disk? Or save at the end?
+
             SCLogger.info(f"preprocessor")
             ds = self.preprocessor.run(ds, session)
             ds.update_report('preprocessing', session)
@@ -265,6 +290,11 @@ def run(self, *args, **kwargs):
             ds = self.extractor.run(ds, session)
             ds.update_report('extraction', session)
 
+            # find the background for this image
+            SCLogger.info(f"backgrounder for image id {ds.image.id}")
+            ds = self.backgrounder.run(ds, session)
+            ds.update_report('extraction', session)
+
             # find astrometric solution, save WCS into Image object and FITS headers
             SCLogger.info(f"astrometor for image id {ds.image.id}")
             ds = self.astrometor.run(ds, session)
@@ -275,6 +305,19 @@ def run(self, *args, **kwargs):
             ds = self.photometor.run(ds, session)
             ds.update_report('extraction', session)
 
+            if self.pars.save_before_subtraction:
+                t_start = time.perf_counter()
+                try:
+                    SCLogger.info(f"Saving intermediate image for image id {ds.image.id}")
+                    ds.save_and_commit(session=session)
+                except Exception as e:
+                    ds.update_report('save intermediate', session)
+                    SCLogger.error(f"Failed to save intermediate image for image id {ds.image.id}")
+                    SCLogger.error(e)
+                    raise e
+
+                ds.runtimes['save_intermediate'] = time.perf_counter() - t_start
+
             # fetch reference images and subtract them, save subtracted Image objects to DB and disk
             SCLogger.info(f"subtractor for image id {ds.image.id}")
             ds = self.subtractor.run(ds, session)
@@ -298,6 +341,8 @@ def run(self, *args, **kwargs):
             # measure deep learning models on the cutouts/measurements
             # TODO: add this...
 
+            # TODO: add a saving step at the end too?
+
             ds.finalize_report(session)
 
             return ds
diff --git a/requirements.txt b/requirements.txt
index cf8a1cb3..8a548f29 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,7 +15,6 @@ pandas==2.1.3
 photutils==1.9.0
 psutil==5.9.8
 psycopg2==2.9.9
-pylandau==2.2.1
 pytest==7.4.3
 pytest-timestamper==0.0.10
 python-dateutil==2.8.2
@@ -59,6 +58,7 @@ wget==3.2
 # iniconfig==2.0.0
 # kafka-python==2.0.2
 # kiwisolver==1.4.4
+# pylandau==2.2.1
 # matplotlib==3.7.1
 # mpi4py==3.1.4
 # msgpack==1.0.5
diff --git a/tests/conftest.py b/tests/conftest.py
index 9ff71332..2a63d44d 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -4,7 +4,6 @@
 import uuid
 import shutil
 import pathlib
-import logging
 
 import numpy as np
 
diff --git a/tests/fixtures/decam.py b/tests/fixtures/decam.py
index 8e70108f..c9925e73 100644
--- a/tests/fixtures/decam.py
+++ b/tests/fixtures/decam.py
@@ -27,6 +27,7 @@
 from util.logger import SCLogger
 from util.cache import copy_to_cache, copy_list_to_cache, copy_from_cache, copy_list_from_cache
 
+
 @pytest.fixture(scope='session')
 def decam_cache_dir(cache_dir):
     output = os.path.join(cache_dir, 'DECam')
@@ -268,7 +269,7 @@ def decam_datastore(
         decam_exposure,
         'N1',
         cache_dir=decam_cache_dir,
-        cache_base_name='115/c4d_20221104_074232_N1_g_Sci_VCOACQ',
+        cache_base_name='115/c4d_20221104_074232_N1_g_Sci_NBXRIO',
         save_original_image=True
     )
     # This save is redundant, as the datastore_factory calls save_and_commit
@@ -327,6 +328,7 @@ def decam_fits_image_filename(download_url, decam_cache_dir):
         except FileNotFoundError:
             pass
 
+
 @pytest.fixture
 def decam_fits_image_filename2(download_url, decam_cache_dir):
     download_url = os.path.join(download_url, 'DECAM')
@@ -345,6 +347,7 @@ def decam_fits_image_filename2(download_url, decam_cache_dir):
         except FileNotFoundError:
             pass
 
+
 @pytest.fixture
 def decam_ref_datastore( code_version, download_url, decam_cache_dir, data_dir, datastore_factory ):
     filebase = 'DECaPS-West_20220112.g.32'
diff --git a/tests/fixtures/pipeline_objects.py b/tests/fixtures/pipeline_objects.py
index 93be2623..dd48ddae 100644
--- a/tests/fixtures/pipeline_objects.py
+++ b/tests/fixtures/pipeline_objects.py
@@ -15,6 +15,7 @@
 from models.image import Image
 from models.source_list import SourceList
 from models.psf import PSF
+from models.background import Background
 from models.world_coordinates import WorldCoordinates
 from models.zero_point import ZeroPoint
 from models.cutouts import Cutouts
@@ -23,6 +24,7 @@
 from pipeline.data_store import DataStore
 from pipeline.preprocessing import Preprocessor
 from pipeline.detection import Detector
+from pipeline.backgrounding import Backgrounder
 from pipeline.astro_cal import AstroCalibrator
 from pipeline.photo_cal import PhotCalibrator
 from pipeline.coaddition import Coadder, CoaddPipeline
@@ -79,6 +81,27 @@ def extractor(extractor_factory):
     return extractor_factory()
 
 
+@pytest.fixture(scope='session')
+def backgrounder_factory(test_config):
+
+    def make_backgrounder():
+        bg = Backgrounder(**test_config.value('extraction.bg'))
+        bg.pars._enforce_no_new_attrs = False
+        bg.pars.test_parameter = bg.pars.add_par(
+            'test_parameter', 'test_value', str, 'parameter to define unique tests', critical=True
+        )
+        bg.pars._enforce_no_new_attrs = True
+
+        return bg
+
+    return make_backgrounder
+
+
+@pytest.fixture
+def backgrounder(backgrounder_factory):
+    return backgrounder_factory()
+
+
 @pytest.fixture(scope='session')
 def astrometor_factory(test_config):
 
@@ -231,6 +254,7 @@ def measurer(measurer_factory):
 def pipeline_factory(
         preprocessor_factory,
         extractor_factory,
+        backgrounder_factory,
         astrometor_factory,
         photometor_factory,
         subtractor_factory,
@@ -243,12 +267,19 @@ def make_pipeline():
         p = Pipeline(**test_config.value('pipeline'))
         p.preprocessor = preprocessor_factory()
         p.extractor = extractor_factory()
+        p.backgrounder = backgrounder_factory()
         p.astrometor = astrometor_factory()
         p.photometor = photometor_factory()
 
         # make sure when calling get_critical_pars() these objects will produce the full, nested dictionary
-        siblings = {'sources': p.extractor.pars, 'wcs': p.astrometor.pars, 'zp': p.photometor.pars}
+        siblings = {
+            'sources': p.extractor.pars,
+            'bg': p.backgrounder.pars,
+            'wcs': p.astrometor.pars,
+            'zp': p.photometor.pars
+        }
         p.extractor.pars.add_siblings(siblings)
+        p.backgrounder.pars.add_siblings(siblings)
         p.astrometor.pars.add_siblings(siblings)
         p.photometor.pars.add_siblings(siblings)
 
@@ -283,8 +314,14 @@ def make_pipeline():
         p.photometor = photometor_factory()
 
         # make sure when calling get_critical_pars() these objects will produce the full, nested dictionary
-        siblings = {'sources': p.extractor.pars, 'wcs': p.astrometor.pars, 'zp': p.photometor.pars}
+        siblings = {
+            'sources': p.extractor.pars,
+            'bg': p.backgrounder.pars,
+            'wcs': p.astrometor.pars,
+            'zp': p.photometor.pars,
+        }
         p.extractor.pars.add_siblings(siblings)
+        p.backgrounder.pars.add_siblings(siblings)
         p.astrometor.pars.add_siblings(siblings)
         p.photometor.pars.add_siblings(siblings)
 
@@ -483,11 +520,11 @@ def make_datastore(
                 else:
                     raise e  # if any other error comes up, raise it
 
-            ############# extraction to create sources / PSF / WCS / ZP #############
+            ############# extraction to create sources / PSF / BG / WCS / ZP #############
             if (   ( not os.getenv( "LIMIT_CACHE_USAGE" ) ) and
                    ( cache_dir is not None ) and ( cache_base_name is not None )
                 ):
-                # try to get the SourceList, PSF, WCS and ZP from cache
+                # try to get the SourceList, PSF, BG, WCS and ZP from cache
                 prov = Provenance(
                     code_version=code_version,
                     process='extraction',
@@ -553,7 +590,35 @@ def make_datastore(
                     # make sure this is saved to the archive as well
                     ds.psf.save(verify_md5=False, overwrite=True)
 
-                ############## astro_cal to create wcs ################
+                # try to get the background from cache
+                cache_name = f'{cache_base_name}.bg_{prov.id[:6]}.h5.json'
+                bg_cache_path = os.path.join(cache_dir, cache_name)
+                if os.path.isfile(bg_cache_path):
+                    SCLogger.debug('loading background from cache. ')
+                    ds.bg = copy_from_cache(Background, cache_dir, cache_name)
+
+                    # if BG already exists on the database, use that instead of this one
+                    existing = session.scalars(
+                        sa.select(Background).where(Background.filepath == ds.bg.filepath)
+                    ).first()
+                    if existing is not None:
+                        # overwrite the existing row data using the JSON cache file
+                        for key in sa.inspect(ds.bg).mapper.columns.keys():
+                            value = getattr(ds.bg, key)
+                            if (
+                                    key not in ['id', 'image_id', 'created_at', 'modified'] and
+                                    value is not None
+                            ):
+                                setattr(existing, key, value)
+                        ds.bg = existing
+
+                    ds.bg.provenance = prov
+                    ds.bg.image = ds.image
+
+                    # make sure this is saved to the archive as well
+                    ds.bg.save(verify_md5=False, overwrite=True)
+
+                # try to get the WCS from cache
                 cache_name = f'{cache_base_name}.wcs_{prov.id[:6]}.txt.json'
                 wcs_cache_path = os.path.join(cache_dir, cache_name)
                 if os.path.isfile(wcs_cache_path):
@@ -588,8 +653,7 @@ def make_datastore(
                     # make sure this is saved to the archive as well
                     ds.wcs.save(verify_md5=False, overwrite=True)
 
-                ########### photo_cal to create zero point ############
-
+                # try to get the ZP from cache
                 cache_name = cache_base_name + '.zp.json'
                 zp_cache_path = os.path.join(cache_dir, cache_name)
                 if os.path.isfile(zp_cache_path):
@@ -621,11 +685,12 @@ def make_datastore(
                     ds.zp.provenance = prov
                     ds.zp.sources = ds.sources
 
-            if ds.sources is None or ds.psf is None or ds.wcs is None or ds.zp is None:  # redo extraction
+            # if any data product is missing, must redo the extraction step
+            if ds.sources is None or ds.psf is None or ds.bg is None or ds.wcs is None or ds.zp is None:
                 SCLogger.debug('extracting sources. ')
                 ds = p.extractor.run(ds, session)
 
-                ds.sources.save()
+                ds.sources.save(overwrite=True)
                 if cache_dir is not None and cache_base_name is not None:
                     output_path = copy_to_cache(ds.sources, cache_dir)
                     if cache_dir is not None and cache_base_name is not None and output_path != sources_cache_path:
@@ -637,9 +702,18 @@ def make_datastore(
                     if cache_dir is not None and cache_base_name is not None and output_path != psf_cache_path:
                         warnings.warn(f'cache path {psf_cache_path} does not match output path {output_path}')
 
+                SCLogger.debug('Running background estimation')
+                ds = p.backgrounder.run(ds, session)
+
+                ds.bg.save(overwrite=True)
+                if cache_dir is not None and cache_base_name is not None:
+                    output_path = copy_to_cache(ds.bg, cache_dir)
+                    if cache_dir is not None and cache_base_name is not None and output_path != bg_cache_path:
+                        warnings.warn(f'cache path {bg_cache_path} does not match output path {output_path}')
+
                 SCLogger.debug('Running astrometric calibration')
                 ds = p.astrometor.run(ds, session)
-                ds.wcs.save()
+                ds.wcs.save(overwrite=True)
                 if ((cache_dir is not None) and (cache_base_name is not None) and
                         (not os.getenv("LIMIT_CACHE_USAGE"))):
                     output_path = copy_to_cache(ds.wcs, cache_dir)
@@ -726,9 +800,7 @@ def make_datastore(
                         parameters=prov_aligned_ref.parameters,
                         upstreams=[
                             ds.image.provenance,
-                            ds.sources.provenance,  # this also includes the PSF's provenance
-                            ds.wcs.provenance,
-                            ds.zp.provenance,
+                            ds.sources.provenance,  # this also includes provs for PSF, BG, WCS, ZP
                         ],
                         process='alignment',
                         is_testing=True,
diff --git a/tests/fixtures/ptf.py b/tests/fixtures/ptf.py
index 1fb671da..5374c04d 100644
--- a/tests/fixtures/ptf.py
+++ b/tests/fixtures/ptf.py
@@ -20,6 +20,7 @@
 from models.image import Image
 from models.source_list import SourceList
 from models.psf import PSF
+from models.background import Background
 from models.world_coordinates import WorldCoordinates
 from models.zero_point import ZeroPoint
 from models.reference import Reference
@@ -172,7 +173,7 @@ def ptf_datastore(datastore_factory, ptf_exposure, ptf_ref, ptf_cache_dir, ptf_b
         ptf_exposure,
         11,
         cache_dir=ptf_cache_dir,
-        cache_base_name='187/PTF_20110429_040004_11_R_Sci_QTD4UW',
+        cache_base_name='187/PTF_20110429_040004_11_R_Sci_BNKEKA',
         overrides={'extraction': {'threshold': 5}},
         bad_pixel_map=ptf_bad_pixel_map,
     )
@@ -329,9 +330,10 @@ def ptf_aligned_images(request, ptf_cache_dir, data_dir, code_version):
             filenames = f.read().splitlines()
         output_images = []
         for filename in filenames:
-            imfile, psffile = filename.split()
+            imfile, psffile, bgfile = filename.split()
             output_images.append(copy_from_cache(Image, cache_dir, imfile + '.image.fits'))
             output_images[-1].psf = copy_from_cache(PSF, cache_dir, psffile + '.fits')
+            output_images[-1].bg = copy_from_cache(Background, cache_dir, bgfile)
             output_images[-1].zp = copy_from_cache(ZeroPoint, cache_dir, imfile + '.zp')
     else:  # no cache available
         ptf_reference_images = request.getfixturevalue('ptf_reference_images')
@@ -351,22 +353,28 @@ def ptf_aligned_images(request, ptf_cache_dir, data_dir, code_version):
 
         filenames = []
         psf_paths = []
+        bg_paths = []
+        # there's an implicit call to Image._make_aligned_images() here
         for image in coadd_image.aligned_images:
             image.save()
             filepath = copy_to_cache(image, cache_dir)
             if image.psf.filepath is None:  # save only PSF objects that haven't been saved yet
                 image.psf.save(overwrite=True)
+            if image.bg.filepath is None:  # save only Background objects that haven't been saved yet
+                image.bg.save(overwrite=True)
             if not os.getenv( "LIMIT_CACHE_USAGE" ):
-               copy_to_cache(image.psf, cache_dir)
-               copy_to_cache(image.zp, cache_dir, filepath=filepath[:-len('.image.fits.json')]+'.zp.json')
+                copy_to_cache(image.psf, cache_dir)
+                copy_to_cache(image.bg, cache_dir)
+                copy_to_cache(image.zp, cache_dir, filepath=filepath[:-len('.image.fits.json')]+'.zp.json')
             filenames.append(image.filepath)
             psf_paths.append(image.psf.filepath)
+            bg_paths.append(image.bg.filepath)
 
         if not os.getenv( "LIMIT_CACHE_USAGE" ):
             os.makedirs(cache_dir, exist_ok=True)
             with open(os.path.join(cache_dir, 'manifest.txt'), 'w') as f:
-                for filename, psf_path in zip(filenames, psf_paths):
-                    f.write(f'{filename} {psf_path}\n')
+                for filename, psf_path, bg_path in zip(filenames, psf_paths, bg_paths):
+                    f.write(f'{filename} {psf_path} {bg_path}\n')
 
         output_images = coadd_image.aligned_images
 
@@ -422,7 +430,7 @@ def ptf_ref(
             is_testing=True,
         )
 
-        cache_base_name = f'187/PTF_20090405_073932_11_R_ComSci_{im_prov.id[:6]}_u-wswtff'
+        cache_base_name = f'187/PTF_20090405_073932_11_R_ComSci_{im_prov.id[:6]}_u-iqxrjn'
 
         # this provenance is used for sources, psf, wcs, zp
         sources_prov = Provenance(
@@ -434,8 +442,9 @@ def ptf_ref(
         )
     extensions = [
         'image.fits',
-        f'psf_{sources_prov.id[:6]}.fits',
         f'sources_{sources_prov.id[:6]}.fits',
+        f'psf_{sources_prov.id[:6]}.fits',
+        f'bg_{sources_prov.id[:6]}.h5',
         f'wcs_{sources_prov.id[:6]}.txt',
         'zp'
     ]
@@ -452,11 +461,6 @@ def ptf_ref(
         coadd_image.ref_image_id = ptf_reference_images[-1].id  # make sure to replace the ID with the new DB value
         assert coadd_image.provenance_id == coadd_image.provenance.id
 
-        # get the PSF:
-        coadd_image.psf = copy_from_cache(PSF, ptf_cache_dir, cache_base_name + f'.psf_{sources_prov.id[:6]}.fits')
-        coadd_image.psf.provenance = sources_prov
-        assert coadd_image.psf.provenance_id == coadd_image.psf.provenance.id
-
         # get the source list:
         coadd_image.sources = copy_from_cache(
             SourceList, ptf_cache_dir, cache_base_name + f'.sources_{sources_prov.id[:6]}.fits'
@@ -464,6 +468,16 @@ def ptf_ref(
         coadd_image.sources.provenance = sources_prov
         assert coadd_image.sources.provenance_id == coadd_image.sources.provenance.id
 
+        # get the PSF:
+        coadd_image.psf = copy_from_cache(PSF, ptf_cache_dir, cache_base_name + f'.psf_{sources_prov.id[:6]}.fits')
+        coadd_image.psf.provenance = sources_prov
+        assert coadd_image.psf.provenance_id == coadd_image.psf.provenance.id
+
+        # get the background:
+        coadd_image.bg = copy_from_cache(Background, ptf_cache_dir, cache_base_name + f'.bg_{sources_prov.id[:6]}.h5')
+        coadd_image.bg.provenance = sources_prov
+        assert coadd_image.bg.provenance_id == coadd_image.bg.provenance.id
+
         # get the WCS:
         coadd_image.wcs = copy_from_cache(
             WorldCoordinates, ptf_cache_dir, cache_base_name + f'.wcs_{sources_prov.id[:6]}.txt'
@@ -491,6 +505,7 @@ def ptf_ref(
             copy_to_cache(pipe.datastore.image, ptf_cache_dir)
             copy_to_cache(pipe.datastore.sources, ptf_cache_dir)
             copy_to_cache(pipe.datastore.psf, ptf_cache_dir)
+            copy_to_cache(pipe.datastore.bg, ptf_cache_dir)
             copy_to_cache(pipe.datastore.wcs, ptf_cache_dir)
             copy_to_cache(pipe.datastore.zp, ptf_cache_dir, cache_base_name + '.zp.json')
 
diff --git a/tests/fixtures/simulated.py b/tests/fixtures/simulated.py
index 5d62bc94..2ccf3d31 100644
--- a/tests/fixtures/simulated.py
+++ b/tests/fixtures/simulated.py
@@ -409,7 +409,7 @@ def sim_image_list(
             im.zp = ZeroPoint()
             im.zp.zp = np.random.uniform(25, 30)
             im.zp.dzp = np.random.uniform(0.01, 0.1)
-            im.zp.aper_cor_radii = im.instrument_object.standard_apertures()
+            im.zp.aper_cor_radii = [1.0, 2.0, 3.0, 5.0]
             im.zp.aper_cors = np.random.normal(0, 0.1, len(im.zp.aper_cor_radii))
             im.zp.provenance = provenance_extra
             im.wcs = WorldCoordinates()
diff --git a/tests/improc/test_alignment.py b/tests/improc/test_alignment.py
index 8ac5968b..257554a8 100644
--- a/tests/improc/test_alignment.py
+++ b/tests/improc/test_alignment.py
@@ -35,10 +35,17 @@ def test_warp_decam( decam_datastore, decam_reference ):
         # Check a couple of spots on the image
         # First, around a star:
         assert ds.image.data[ 2223:2237, 545:559 ].sum() == pytest.approx( 58014.1, rel=0.01 )
-        assert warped.data[ 2223:2237, 545:559 ].sum() == pytest.approx( 22597.9, rel=0.01 )
-        # And a blank spot
-        assert ds.image.data[ 2243:2257, 575:589 ].sum() == pytest.approx( 35298.6, rel=0.01 )    # sky not subtracted
-        assert warped.data[ 2243:2257, 575:589 ].sum() == pytest.approx( 971.7, rel=0.01 )
+        assert warped.data[ 2223:2237, 545:559 ].sum() == pytest.approx( 21602.75, rel=0.01 )
+
+        # And a blank spot (here we can do some statistics instead of hard coded values)
+        num_pix = ds.image.data[2243:2257, 575:589].size
+        bg_mean = num_pix * ds.image.bg.value
+        bg_noise = np.sqrt(num_pix) * ds.image.bg.noise
+        assert abs(ds.image.data[ 2243:2257, 575:589 ].sum() - bg_mean) < bg_noise
+
+        bg_mean = 0  # assume the warped image is background subtracted
+        bg_noise = np.sqrt(num_pix) * ds.ref_image.bg.noise
+        assert abs(warped.data[ 2243:2257, 575:589 ].sum() - bg_mean) < bg_noise
 
         # Make sure the warped image WCS is about right.  We don't
         # expect it to be exactly identical, but it should be very
@@ -88,7 +95,7 @@ def test_alignment_in_image( ptf_reference_images, code_version ):
         aligned = new_image.aligned_images
         assert new_image.upstream_images == ptf_reference_images
         assert len(aligned) == len(ptf_reference_images)
-        assert np.array_equal(aligned[index].data, ptf_reference_images[index].data)
+        assert np.array_equal(aligned[index].data, ptf_reference_images[index].data_bgsub)
         ref = ptf_reference_images[index]
 
         # check that images are aligned properly
@@ -112,7 +119,7 @@ def test_alignment_in_image( ptf_reference_images, code_version ):
             loaded_image = session.scalars(sa.select(Image).where(Image.id == new_image.id)).first()
             assert loaded_image is not None
             assert len(loaded_image.aligned_images) == len(ptf_reference_images)
-            assert np.array_equal(loaded_image.aligned_images[-1].data, ptf_reference_images[-1].data)
+            assert np.array_equal(loaded_image.aligned_images[-1].data, ptf_reference_images[-1].data_bgsub)
 
             # check that images are aligned properly
             for image in loaded_image.aligned_images:
diff --git a/tests/improc/test_zogy.py b/tests/improc/test_zogy.py
index 69aa0365..29ce4006 100644
--- a/tests/improc/test_zogy.py
+++ b/tests/improc/test_zogy.py
@@ -428,7 +428,8 @@ def test_subtraction_seeing_background():
                             raise ValueError(
                                 f'seeing: ({ref_seeing:.2f}, {new_seeing:.2f}), '
                                 f'background: ({ref_bkg:.2f}, {new_bkg:.2f}), '
-                                f'expected/measured: ({expected:.3f}, {measured:.3f})'
+                                f'expected/measured: ({expected:.3f}, {measured:.3f}), '
+                                f'loss: {(expected - measured) / expected:.3f}'
                             )
 
 
diff --git a/tests/models/test_background.py b/tests/models/test_background.py
new file mode 100644
index 00000000..fc58d3b6
--- /dev/null
+++ b/tests/models/test_background.py
@@ -0,0 +1,102 @@
+import pytest
+import numpy as np
+import h5py
+
+from models.provenance import Provenance
+from models.background import Background
+
+
+def test_save_load_backgrounds(decam_raw_image, code_version):
+    image = decam_raw_image
+    bg_mean = 3.14
+    bg_var = 6.28
+
+    try:  # cleanup at the end
+        # Create a background object with a scalar model:
+        b1 = Background(
+            format='scalar',
+            method='sep',
+            image=image,
+            value=bg_mean,
+            noise=np.sqrt(bg_var)
+        )
+
+        prov = Provenance(
+            code_version=code_version,
+            process='extraction',
+            parameters={'method': 'sep', 'format': 'scalar'},
+            upstreams=[image.provenance],
+            is_testing=True,
+        )
+
+        b1.provenance = prov
+
+        b1.save()
+
+        # check the filename contains the provenance hash
+        assert prov.id[:6] in b1.get_fullpath()
+
+        # check that the file contains what we expect:
+        with h5py.File(b1.get_fullpath(), 'r') as f:
+            # check there's a "background" group:
+            assert 'background' in f
+            bg = f['background']
+
+            # check the attributes:
+            assert bg.attrs['format'] == 'scalar'
+            assert bg.attrs['method'] == 'sep'
+            assert bg.attrs['value'] == bg_mean
+            assert bg.attrs['noise'] == np.sqrt(bg_var)
+
+        # make a new background with some data:
+        b2 = Background(
+            format='map',
+            method='sep',
+            image=image,
+            value=bg_mean,
+            noise=np.sqrt(bg_var),
+            counts=np.random.normal(bg_mean, 1, size=(10, 10)),
+            variance=np.random.normal(bg_var, 1, size=(10, 10)),
+        )
+
+        prov = Provenance(
+            code_version=code_version,
+            process='extraction',
+            parameters={'method': 'sep', 'format': 'map'},
+            upstreams=[image.provenance],
+            is_testing=True,
+        )
+
+        b2.provenance = prov
+
+        with pytest.raises(RuntimeError, match='Counts shape .* does not match image shape .*'):
+            b2.save()
+
+        b2.counts = np.random.normal(bg_mean, 1, size=image.data.shape)
+        b2.variance = np.random.normal(bg_var, 1, size=image.data.shape)
+        b2.save()
+
+        # check the filename contains the provenance hash
+        assert prov.id[:6] in b2.get_fullpath()
+
+        # check that the file contains what we expect:
+        with h5py.File(b2.get_fullpath(), 'r') as f:
+            # check there's a "background" group:
+            assert 'background' in f
+            bg = f['background']
+
+            # check the attributes:
+            assert bg.attrs['format'] == 'map'
+            assert bg.attrs['method'] == 'sep'
+            assert bg.attrs['value'] == bg_mean
+            assert bg.attrs['noise'] == np.sqrt(bg_var)
+
+            # check the data:
+            assert np.allclose(bg['counts'], b2.counts)
+            assert np.allclose(bg['variance'], b2.variance)
+
+    finally:
+        if 'b1' in locals():
+            b1.delete_from_disk_and_database()
+        if 'b2' in locals():
+            b2.delete_from_disk_and_database()
diff --git a/tests/models/test_measurements.py b/tests/models/test_measurements.py
index 430598ab..116962a7 100644
--- a/tests/models/test_measurements.py
+++ b/tests/models/test_measurements.py
@@ -13,8 +13,9 @@
 from models.measurements import Measurements
 
 
-def test_measurements_attributes(measurer, ptf_datastore):
+def test_measurements_attributes(measurer, ptf_datastore, test_config):
 
+    aper_radii = test_config.value('extraction.sources.apertures')
     ds = measurer.run(ptf_datastore.cutouts)
     # check that the measurer actually loaded the measurements from db, and not recalculated
     assert len(ds.measurements) <= len(ds.cutouts)  # not all cutouts have saved measurements
@@ -28,7 +29,7 @@ def test_measurements_attributes(measurer, ptf_datastore):
     assert np.allclose(m.aper_radii, new_im.zp.aper_cor_radii)
     assert np.allclose(
         new_im.zp.aper_cor_radii,
-        new_im.psf.fwhm_pixels * np.array(new_im.instrument_object.standard_apertures()),
+        new_im.psf.fwhm_pixels * np.array(aper_radii),
     )
     assert m.mjd == new_im.mjd
     assert m.exp_time == new_im.exp_time
@@ -37,12 +38,24 @@ def test_measurements_attributes(measurer, ptf_datastore):
     original_flux = m.flux_apertures[m.best_aperture]
 
     # set the flux temporarily to something positive
-    m.flux_apertures[m.best_aperture] = 1000
-    assert m.magnitude == -2.5 * np.log10(1000) + new_im.zp.zp + new_im.zp.aper_cors[m.best_aperture]
+    m.flux_apertures[0] = 1000
+    assert m.mag_apertures[0] == -2.5 * np.log10(1000) + new_im.zp.zp + new_im.zp.aper_cors[0]
+
+    m.flux_psf = 1000
+    expected_mag = -2.5 * np.log10(1000) + new_im.zp.zp
+    assert m.mag_psf == expected_mag
 
     # set the flux temporarily to something negative
-    m.flux_apertures[m.best_aperture] = -1000
-    assert np.isnan(m.magnitude)
+    m.flux_apertures[0] = -1000
+    assert np.isnan(m.mag_apertures[0])
+
+    # check that background is subtracted from the "flux" and "magnitude" properties
+    if m.best_aperture == -1:
+        assert m.flux == m.flux_psf - m.bkg_mean * m.area_psf
+        assert m.magnitude > m.mag_psf  # the magnitude has background subtracted from it
+        assert m.magnitude_err > m.mag_psf_err  # the magnitude error is larger because of the error in background
+    else:
+        assert m.flux == m.flux_apertures[m.best_aperture] - m.bkg_mean * m.area_apertures[m.best_aperture]
 
     # set the flux and zero point to some randomly chosen values and test the distribution of the magnitude:
     fiducial_zp = new_im.zp.zp
@@ -134,7 +147,7 @@ def test_filtering_measurements(ptf_datastore):
             )).all()
         assert len(ms) == len(measurements)  # all measurements have the same filter
 
-        ms = session.scalars(sa.select(Measurements).where(Measurements.background > 0)).all()
+        ms = session.scalars(sa.select(Measurements).where(Measurements.bkg_mean > 0)).all()
         assert len(ms) <= len(measurements)  # only some of the measurements have positive background
 
         ms = session.scalars(sa.select(Measurements).where(
diff --git a/tests/models/test_objects.py b/tests/models/test_objects.py
index f52807d7..192deae6 100644
--- a/tests/models/test_objects.py
+++ b/tests/models/test_objects.py
@@ -37,7 +37,7 @@ def test_lightcurves_from_measurements(sim_lightcurves):
         measured_flux = []
 
         for m in lc:
-            measured_flux.append(m.flux_apertures[3] - m.background * m.area_apertures[3])
+            measured_flux.append(m.flux_apertures[3] - m.bkg_mean * m.area_apertures[3])
             expected_flux.append(m.sources.data['flux'][m.cutouts.index_in_sources])
             expected_error.append(m.sources.data['flux_err'][m.cutouts.index_in_sources])
 
diff --git a/tests/models/test_provenance.py b/tests/models/test_provenance.py
index 615c3318..f9c82042 100644
--- a/tests/models/test_provenance.py
+++ b/tests/models/test_provenance.py
@@ -2,6 +2,7 @@
 import uuid
 
 import sqlalchemy as sa
+from sqlalchemy.orm.exc import DetachedInstanceError
 
 from models.base import SmartSession
 from models.provenance import CodeHash, CodeVersion, Provenance
@@ -314,3 +315,68 @@ def check_in_session( sess, obj ):
         if 'p4' in locals():
             session.execute(sa.delete(Provenance).where(Provenance.id == p4.id))
         session.commit()
+
+
+def test_eager_load_upstreams( provenance_base ):
+    try:
+        with SmartSession() as session:
+            provenance_base = session.merge( provenance_base )
+            p1 = Provenance(
+                process="test_process_1",
+                code_version=provenance_base.code_version,
+                parameters={'test_parameter': 'test_value'},
+                upstreams=[ provenance_base ],
+                is_testing=True
+            )
+
+            p2 = Provenance(
+                process="test_process_2",
+                code_version=provenance_base.code_version,
+                parameters={'test_parameter': 'test_value'},
+                upstreams=[ p1 ],
+                is_testing=True
+            )
+
+            p3 = Provenance(
+                process="test_process_3",
+                code_version=provenance_base.code_version,
+                parameters={'test_parameter': 'test_value'},
+                upstreams=[ p2 ],
+                is_testing=True
+            )
+
+            p4 = Provenance(
+                process="test_process_4",
+                code_version=provenance_base.code_version,
+                parameters={'test_parameter': 'test_value'},
+                upstreams=[ p3 ],
+                is_testing=True
+            )
+
+            session.add_all( [ p1, p2, p3, p4 ] )
+            session.commit()
+
+            # Now, in another session....
+            with SmartSession() as session2:
+                p4 = session2.scalars(sa.select(Provenance).where(Provenance.id == p4.id)).first()
+
+            # we are out of the session, so loading of upstream relationships is only for those eager loaded ones
+            assert len(p4.upstreams) == 1  # should be ok
+            assert len(p4.upstreams[0].upstreams) == 1  # this should also be ok
+            assert len(p4.upstreams[0].upstreams[0].upstreams) == 1  # this should also be ok, assuming join_depth=3
+
+            with pytest.raises(DetachedInstanceError):
+                p4.upstreams[0].upstreams[0].upstreams[0].upstreams  # this should fail, as the join_depth is not enough
+
+    finally:
+        with SmartSession() as session:
+            if 'p1' in locals():
+                session.execute(sa.delete(Provenance).where(Provenance.id == p1.id))
+            if 'p2' in locals():
+                session.execute(sa.delete(Provenance).where(Provenance.id == p2.id))
+            if 'p3' in locals():
+                session.execute(sa.delete(Provenance).where(Provenance.id == p3.id))
+            if 'p4' in locals():
+                session.execute(sa.delete(Provenance).where(Provenance.id == p4.id))
+
+            session.commit()
\ No newline at end of file
diff --git a/tests/models/test_reports.py b/tests/models/test_reports.py
index d395ec80..e1e33ead 100644
--- a/tests/models/test_reports.py
+++ b/tests/models/test_reports.py
@@ -93,48 +93,58 @@ def test_measure_runtime_memory(decam_exposure, decam_reference, pipeline_for_te
 
     t0 = time.perf_counter()
 
-    ds = p.run(decam_exposure, 'N1')
-
-    assert p.preprocessor.has_recalculated
-    assert p.extractor.has_recalculated
-    assert p.astrometor.has_recalculated
-    assert p.photometor.has_recalculated
-    assert p.subtractor.has_recalculated
-    assert p.detector.has_recalculated
-    assert p.cutter.has_recalculated
-    assert p.measurer.has_recalculated
-
-    measured_time = 0
-    peak_memory = 0
-    for step in ds.runtimes.keys():  # also make sure all the keys are present in both dictionaries
-        measured_time += ds.runtimes[step]
+    try:
+        ds = p.run(decam_exposure, 'N1')
+
+        total_time = time.perf_counter() - t0
+
+        assert p.preprocessor.has_recalculated
+        assert p.extractor.has_recalculated
+        assert p.backgrounder.has_recalculated
+        assert p.astrometor.has_recalculated
+        assert p.photometor.has_recalculated
+        assert p.subtractor.has_recalculated
+        assert p.detector.has_recalculated
+        assert p.cutter.has_recalculated
+        assert p.measurer.has_recalculated
+
+        measured_time = 0
+        peak_memory = 0
+        for step in ds.runtimes.keys():  # also make sure all the keys are present in both dictionaries
+            measured_time += ds.runtimes[step]
+            if parse_bool(os.getenv('SEECHANGE_TRACEMALLOC')):
+                peak_memory = max(peak_memory, ds.memory_usages[step])
+
+        print(f'total_time: {total_time:.1f}s')
+        print(f'measured_time: {measured_time:.1f}s')
+        pprint(ds.runtimes, sort_dicts=False)
+        assert measured_time > 0.98 * total_time  # at least 99% of the time is accounted for
+
         if parse_bool(os.getenv('SEECHANGE_TRACEMALLOC')):
-            peak_memory = max(peak_memory, ds.memory_usages[step])
-
-    total_time = time.perf_counter() - t0
-
-    print(f'total_time: {total_time:.1f}s')
-    print(f'measured_time: {measured_time:.1f}s')
-    pprint(ds.runtimes, sort_dicts=False)
-    assert measured_time > 0.99 * total_time  # at least 99% of the time is accounted for
-
-    if parse_bool(os.getenv('SEECHANGE_TRACEMALLOC')):
-        print(f'peak_memory: {peak_memory:.1f}MB')
-        pprint(ds.memory_usages, sort_dicts=False)
-        assert 1000.0 < peak_memory < 10000.0  # memory usage is in MB, takes between 1 and 10 GB
-
-    with SmartSession() as session:
-        rep = session.scalars(sa.select(Report).where(Report.exposure_id == decam_exposure.id)).one()
-        assert rep is not None
-        assert rep.success
-        assert rep.process_runtime == ds.runtimes
-        assert rep.process_memory == ds.memory_usages
-        # 'preprocessing, extraction, subtraction, detection, cutting, measuring'
-        assert rep.progress_steps == ', '.join(PROCESS_OBJECTS.keys())
-        assert rep.products_exist == 'image, sources, psf, wcs, zp, sub_image, detections, cutouts, measurements'
-        assert rep.products_committed == ''  # we don't save the data store objects at any point?
-        assert rep.provenance.upstreams[0].id == ds.measurements[0].provenance.id
-        assert rep.num_prev_reports == 0
+            print(f'peak_memory: {peak_memory:.1f}MB')
+            pprint(ds.memory_usages, sort_dicts=False)
+            assert 1000.0 < peak_memory < 10000.0  # memory usage is in MB, takes between 1 and 10 GB
+
+        with SmartSession() as session:
+            rep = session.scalars(sa.select(Report).where(Report.exposure_id == decam_exposure.id)).one()
+            assert rep is not None
+            assert rep.success
+            assert rep.process_runtime == ds.runtimes
+            assert rep.process_memory == ds.memory_usages
+            # should contain: 'preprocessing, extraction, subtraction, detection, cutting, measuring'
+            assert rep.progress_steps == ', '.join(PROCESS_OBJECTS.keys())
+            assert rep.products_exist == ('image, sources, psf, bg, wcs, zp, '
+                                          'sub_image, detections, cutouts, measurements')
+            assert rep.products_committed == 'image, sources, psf, bg, wcs, zp'  # we use intermediate save
+            assert rep.provenance.upstreams[0].id == ds.measurements[0].provenance.id
+            assert rep.num_prev_reports == 0
+            ds.save_and_commit(session=session)
+            rep.scan_datastore(ds, session=session)
+            assert rep.products_committed == ('image, sources, psf, bg, wcs, zp, '
+                                              'sub_image, detections, cutouts, measurements')
+    finally:
+        if 'ds' in locals():
+            ds.delete_everything()
 
 
 def test_inject_warnings(decam_datastore, decam_reference, pipeline_for_tests, decam_default_calibrators):
diff --git a/tests/models/test_source_list.py b/tests/models/test_source_list.py
index 46bafc5c..a355edec 100644
--- a/tests/models/test_source_list.py
+++ b/tests/models/test_source_list.py
@@ -155,8 +155,7 @@ def test_read_sextractor( ztf_filepath_sources ):
     assert sources.num_sources == 112
     assert sources.good.sum() == 105
     assert sources.aper_rads == [ 1.0, 2.5 ]
-    assert sources._inf_aper_num is None
-    assert sources.inf_aper_num == 1
+    assert sources.inf_aper_num is None
     assert sources.x[0] == pytest.approx( 798.24, abs=0.01 )
     assert sources.y[0] == pytest.approx( 17.14, abs=0.01 )
     assert sources.x[50] == pytest.approx( 899.33, abs=0.01 )
@@ -243,12 +242,12 @@ def test_calc_apercor( decam_datastore ):
     sources = decam_datastore.get_sources()
 
     # These numbers are when you don't use is_star at all:
-    assert sources.calc_aper_cor() == pytest.approx(-0.4509, abs=0.01)
-    assert sources.calc_aper_cor(aper_num=1) == pytest.approx(-0.177, abs=0.01)
-    assert sources.calc_aper_cor(inf_aper_num=7) == pytest.approx(-0.4509, abs=0.01)
-    assert sources.calc_aper_cor(inf_aper_num=2) == pytest.approx(-0.428, abs=0.01)
-    assert sources.calc_aper_cor(aper_num=2) == pytest.approx(-0.028, abs=0.01)
-    assert sources.calc_aper_cor(aper_num=2, inf_aper_num=7) == pytest.approx(-0.02356, abs=0.01)
+    assert sources.calc_aper_cor() == pytest.approx(-0.1768, abs=0.01)
+    assert sources.calc_aper_cor(aper_num=1) == pytest.approx(-0.0258, abs=0.01)
+    assert sources.calc_aper_cor(inf_aper_num=3) == pytest.approx(-0.1768, abs=0.01)
+    assert sources.calc_aper_cor(inf_aper_num=1) == pytest.approx(-0.1508, abs=0.01)
+    assert sources.calc_aper_cor(aper_num=2) == pytest.approx(-0.00629, abs=0.01)
+    assert sources.calc_aper_cor(aper_num=2, inf_aper_num=3) == pytest.approx(-0.00629, abs=0.01)
 
     # The numbers below are what you get when you use CLASS_STAR in SourceList.is_star
     # assert sources.calc_aper_cor() == pytest.approx( -0.457, abs=0.01 )
diff --git a/tests/pipeline/test_backgrounding.py b/tests/pipeline/test_backgrounding.py
new file mode 100644
index 00000000..0d995ad5
--- /dev/null
+++ b/tests/pipeline/test_backgrounding.py
@@ -0,0 +1,52 @@
+import pytest
+import uuid
+
+import numpy as np
+
+from improc.tools import sigma_clipping
+
+
+def test_measuring_background(decam_processed_image, backgrounder):
+    backgrounder.pars.test_parameter = uuid.uuid4().hex  # make sure there is no hashed value
+    ds = backgrounder.run(decam_processed_image)
+
+    # check that the background is statistically similar to the image stats
+    mu, sig = sigma_clipping(ds.image.nandata)
+    assert mu == pytest.approx(ds.bg.value, rel=0.01)
+    assert sig == pytest.approx(ds.bg.noise, rel=0.2)  # this is really a very rough estimate
+
+    # is the background subtracted image a good representation?
+    mu, sig = sigma_clipping(ds.image.nandata_bgsub)  # also checks that nandata_bgsub exists
+    assert mu == pytest.approx(0, abs=sig)
+    assert sig < 10
+
+    # most of the pixels are inside a 3 sigma range
+    assert np.sum(np.abs(ds.image.nandata_bgsub) < 3 * sig) > 0.9 * ds.image.nandata.size
+
+    # this is not true of the original image
+    assert np.sum(np.abs(ds.image.nandata) < 3 * sig) < 0.001 * ds.image.nandata.size
+
+    # try to do the background again, but this time using the "zero" method
+    backgrounder.pars.method = 'zero'
+    ds = backgrounder.run(ds)
+    assert ds.bg.method == 'zero'
+    assert ds.bg.value == 0
+    assert ds.bg.noise == 0
+    assert np.array_equal(ds.image.data, ds.image.data_bgsub)
+
+
+def test_warnings_and_exceptions(decam_datastore, backgrounder):
+    backgrounder.pars.inject_warnings = 1
+
+    with pytest.warns(UserWarning) as record:
+        backgrounder.run(decam_datastore)
+    assert len(record) > 0
+    assert any("Warning injected by pipeline parameters in process 'backgrounding'." in str(w.message) for w in record)
+
+    backgrounder.pars.inject_warnings = 0
+    backgrounder.pars.inject_exceptions = 1
+    with pytest.raises(Exception) as excinfo:
+        ds = backgrounder.run(decam_datastore)
+        ds.reraise()
+    assert "Exception injected by pipeline parameters in process 'backgrounding'." in str(excinfo.value)
+    ds.read_exception()
diff --git a/tests/pipeline/test_coaddition.py b/tests/pipeline/test_coaddition.py
index 5cbb194d..700fddeb 100644
--- a/tests/pipeline/test_coaddition.py
+++ b/tests/pipeline/test_coaddition.py
@@ -21,8 +21,8 @@
 from pipeline.photo_cal import PhotCalibrator
 
 
-def estimate_psf_width(data, sz=15, upsampling=25):
-    """Extract a bright star and estimate its FWHM.
+def estimate_psf_width(data, sz=7, upsampling=50, num_stars=20):
+    """Extract a few bright stars and estimate their median FWHM.
 
     This is a very rough-and-dirty method used only for testing.
 
@@ -34,10 +34,13 @@ def estimate_psf_width(data, sz=15, upsampling=25):
         The image data.
     sz: int
         The size of the box to extract around the star.
-        Default is 15.
+        Default is 7.
     upsampling: int
         The factor by which to up-sample the PSF.
-        Default is 25.
+        Default is 50.
+    num_stars: int
+        The number of stars to use to estimate the FWHM.
+        Default is 20.
 
     Returns
     -------
@@ -51,49 +54,58 @@ def estimate_psf_width(data, sz=15, upsampling=25):
     data[:, 0:sz] = np.nan
     data[:, -sz:] = np.nan
 
-    psf = extract_psf_surrogate(data, sz=sz, upsampling=upsampling)
-    flux = []
-    area = []
-    radii = np.array(range(1, psf.shape[0], 2))
-    x, y = np.meshgrid(np.arange(psf.shape[0]), np.arange(psf.shape[1]))
-    rmap = np.sqrt((x - psf.shape[0] // 2) ** 2 + (y - psf.shape[1] // 2) ** 2)
-
-    for r in radii:
-        mask = (rmap <= r + 1) & (rmap > r - 1)
-        area.append(np.sum(mask))
-        flux.append(np.sum(psf[mask]))
-
-    flux = np.array(flux)
-    area = np.array(area, dtype=float)
-    area[area == 0] = np.nan
-    flux_n = flux / area  # normalize by the area of the annulus
-
-    # go over the flux difference curve and find where it drops below half the peak flux:
-    peak = np.nanmax(flux_n)
-    idx = np.where(flux_n <= peak / 2)[0][0]
-
-    fwhm = radii[idx] * 2 / upsampling
+    fwhms = []
+    for i in range(num_stars):
+        psf = extract_psf_surrogate(data, sz=sz, upsampling=upsampling)
+        flux = []
+        area = []
+        radii = np.array(range(1, psf.shape[0] // 2, 2))
+        x, y = np.meshgrid(np.arange(psf.shape[0]), np.arange(psf.shape[1]))
+        rmap = np.sqrt((x - psf.shape[1] // 2) ** 2 + (y - psf.shape[0] // 2) ** 2)
+
+        for r in radii:
+            mask = (rmap <= r + 1) & (rmap > r - 1)
+            area.append(np.sum(mask))
+            flux.append(np.sum(psf[mask]))
+
+        flux = np.array(flux)
+        area = np.array(area, dtype=float)
+        area[area == 0] = np.nan
+        flux_n = flux / area  # normalize by the area of the annulus
+
+        # go over the flux difference curve and find where it drops below half the peak flux:
+        peak = np.nanmax(flux_n)
+        idx = np.where(flux_n <= peak / 2)[0][0]
+
+        fwhm = radii[idx] * 2 / upsampling
+        fwhms.append(fwhm)
+
+    fwhm = np.nanmedian(fwhms)
+    print(f'fwhm median= {fwhm}, fwhm_err= {np.std(fwhms)}')
 
     return fwhm
 
 
-def extract_psf_surrogate(data, sz=15, upsampling=25):
+def extract_psf_surrogate(data, sz=7, upsampling=50):
     """Extract a rough estimate for the PSF from the brightest (non-flagged) star in the image.
 
     This is a very rough-and-dirty method used only for testing.
 
     Assumes the data array has NaNs at all masked pixel locations.
 
+    Will mask the area of the chosen star so that the same array can be
+    re-used to find progressively fainter stars.
+
     Parameters
     ----------
     data: ndarray
         The image data.
     sz: int
         The size of the box to extract around the star.
-        Default is 15.
+        Default is 7.
     upsampling: int
         The factor by which to up-sample the PSF.
-        Default is 25.
+        Default is 50.
 
     Returns
     -------
@@ -110,7 +122,8 @@ def extract_psf_surrogate(data, sz=15, upsampling=25):
     edge_y1 = max(0, y - sz)
     edge_y2 = min(data.shape[0], y + sz)
 
-    psf = data[edge_y1:edge_y2, edge_x1:edge_x2]
+    psf = data[edge_y1:edge_y2, edge_x1:edge_x2].copy()
+    data[edge_y1:edge_y2, edge_x1:edge_x2] = np.nan  # can re-use this array to find other stars
 
     # up-sample the PSF by the given factor:
     psf = ifft2(fftshift(np.pad(fftshift(fft2(psf)), sz*upsampling))).real
@@ -260,20 +273,19 @@ def test_zogy_vs_naive(ptf_aligned_images, coadder):
     # get the FWHM estimate for the regular images and for the coadd
     fwhms = []
     for im in ptf_aligned_images:
-        im_nan = im.data.copy()
-        im_nan[im.flags > 0] = np.nan
-        fwhms.append(estimate_psf_width(im_nan))
+        # choose an area in the middle of the image
+        fwhms.append(estimate_psf_width(im.nandata[1800:2600, 600:1400]))
 
     fwhms = np.array(fwhms)
 
     zogy_im_nans = zogy_im.copy()
     zogy_im_nans[zogy_fl > 0] = np.nan
-    zogy_fwhm = estimate_psf_width(zogy_im_nans)
+    zogy_fwhm = estimate_psf_width(zogy_im_nans[1800:2600, 600:1400])
     naive_im_nans = naive_im.copy()
     naive_im_nans[naive_fl > 0] = np.nan
-    naive_fwhm = estimate_psf_width(naive_im_nans)
+    naive_fwhm = estimate_psf_width(naive_im_nans[1800:2600, 600:1400])
 
-    assert all(zogy_fwhm <= fwhms)  # the ZOGY PSF should be narrower than original PSFs
+    assert zogy_fwhm < np.mean(fwhms)  # the ZOGY PSF should be narrower than original PSFs
     assert zogy_fwhm < naive_fwhm
 
 
@@ -449,8 +461,8 @@ def test_coaddition_pipeline_outputs(ptf_reference_images, ptf_aligned_images):
 
         # check that the ZOGY PSF width is similar to the PSFex result
         assert np.max(coadd_image.zogy_psf) == pytest.approx(np.max(coadd_image.psf.get_clip()), abs=0.01)
-        zogy_fwhm = estimate_psf_width(coadd_image.zogy_psf)
-        psfex_fwhm = estimate_psf_width(np.pad(coadd_image.psf.get_clip(), 20))  # pad so extract_psf_surrogate works
+        zogy_fwhm = estimate_psf_width(coadd_image.zogy_psf, num_stars=1)
+        psfex_fwhm = estimate_psf_width(np.pad(coadd_image.psf.get_clip(), 20), num_stars=1)  # pad so extract_psf_surrogate works
         assert zogy_fwhm == pytest.approx(psfex_fwhm, rel=0.1)
 
         # check that the S/N is consistent with a coadd
@@ -466,8 +478,8 @@ def test_coaddition_pipeline_outputs(ptf_reference_images, ptf_aligned_images):
         # zogy background noise is normalized by construction
         assert bkg_zogy == pytest.approx(1.0, abs=0.1)
 
-        # S/N should be sqrt(N) better # TODO: why is the zogy S/N 15% better than expected??
-        assert snr_zogy == pytest.approx(mean_snr * np.sqrt(len(ptf_reference_images)), rel=0.2)
+        # S/N should be sqrt(N) better # TODO: why is the zogy S/N 20% better than expected??
+        assert snr_zogy == pytest.approx(mean_snr * np.sqrt(len(ptf_reference_images)), rel=0.5)
 
     finally:
         if 'coadd_image' in locals():
diff --git a/tests/pipeline/test_extraction.py b/tests/pipeline/test_extraction.py
index bfd35413..9bd472c0 100644
--- a/tests/pipeline/test_extraction.py
+++ b/tests/pipeline/test_extraction.py
@@ -137,13 +137,28 @@ def test_sextractor_extract_once( decam_datastore, extractor ):
         assert sourcelist.num_sources == 5611
         assert len(sourcelist.data) == sourcelist.num_sources
         assert sourcelist.aper_rads == [ 5. ]
-        assert sourcelist._inf_aper_num is None
-        assert sourcelist.inf_aper_num == 0
 
         assert sourcelist.info['SEXAPED1'] == 10.0
         assert sourcelist.info['SEXAPED2'] == 0.
         assert sourcelist.info['SEXBKGND'] == pytest.approx( 179.8, abs=0.1 )
 
+        snr = sourcelist.apfluxadu()[0] / sourcelist.apfluxadu()[1]
+        # print(
+        #     f'sourcelist.x.min()= {sourcelist.x.min()}',
+        #     f'sourcelist.x.max()= {sourcelist.x.max()}',
+        #     f'sourcelist.y.min()= {sourcelist.y.min()}',
+        #     f'sourcelist.y.max()= {sourcelist.y.max()}',
+        #     f'sourcelist.errx.min()= {sourcelist.errx.min()}',
+        #     f'sourcelist.errx.max()= {sourcelist.errx.max()}',
+        #     f'sourcelist.erry.min()= {sourcelist.erry.min()}',
+        #     f'sourcelist.erry.max()= {sourcelist.erry.max()}',
+        #     f'sourcelist.apfluxadu()[0].min()= {sourcelist.apfluxadu()[0].min()}',
+        #     f'sourcelist.apfluxadu()[0].max()= {sourcelist.apfluxadu()[0].max()}',
+        #     f'snr.min()= {snr.min()}',
+        #     f'snr.max()= {snr.max()}',
+        #     f'snr.mean()= {snr.mean()}',
+        #     f'snr.std()= {snr.std()}'
+        # )
         assert sourcelist.x.min() == pytest.approx( 16.0, abs=0.1 )
         assert sourcelist.x.max() == pytest.approx( 2039.6, abs=0.1 )
         assert sourcelist.y.min() == pytest.approx( 16.264, abs=0.1 )
@@ -156,24 +171,32 @@ def test_sextractor_extract_once( decam_datastore, extractor ):
         assert ( np.sqrt( sourcelist.vary ) == sourcelist.erry ).all()
         assert sourcelist.apfluxadu()[0].min() == pytest.approx( -656.8731, rel=1e-5 )
         assert sourcelist.apfluxadu()[0].max() == pytest.approx( 2850920.0, rel=1e-5 )
-        snr = sourcelist.apfluxadu()[0] / sourcelist.apfluxadu()[1]
         assert snr.min() == pytest.approx( -9.91, abs=0.1 )
         assert snr.max() == pytest.approx( 2348.2166, abs=1. )
         assert snr.mean() == pytest.approx( 146.80, abs=0.1 )
         assert snr.std() == pytest.approx( 285.4, abs=1. )
 
         # Test multiple apertures
-        sourcelist, _, _ = extractor._run_sextractor_once( decam_datastore.image, apers=[2, 5] )
+        sourcelist, _, _ = extractor._run_sextractor_once( decam_datastore.image, apers=[ 2., 5. ])
 
         assert sourcelist.num_sources == 5611    # It *finds* the same things
         assert len(sourcelist.data) == sourcelist.num_sources
         assert sourcelist.aper_rads == [ 2., 5. ]
-        assert sourcelist._inf_aper_num is None
-        assert sourcelist.inf_aper_num == 1
 
         assert sourcelist.info['SEXAPED1'] == 4.0
         assert sourcelist.info['SEXAPED2'] == 10.0
         assert sourcelist.info['SEXBKGND'] == pytest.approx( 179.8, abs=0.1 )
+
+        # print(
+        #     f'sourcelist.x.min()= {sourcelist.x.min()}',
+        #     f'sourcelist.x.max()= {sourcelist.x.max()}',
+        #     f'sourcelist.y.min()= {sourcelist.y.min()}',
+        #     f'sourcelist.y.max()= {sourcelist.y.max()}',
+        #     f'sourcelist.apfluxadu(apnum=1)[0].min()= {sourcelist.apfluxadu(apnum=1)[0].min()}',
+        #     f'sourcelist.apfluxadu(apnum=1)[0].max()= {sourcelist.apfluxadu(apnum=1)[0].max()}',
+        #     f'sourcelist.apfluxadu(apnum=0)[0].min()= {sourcelist.apfluxadu(apnum=0)[0].min()}',
+        #     f'sourcelist.apfluxadu(apnum=0)[0].max()= {sourcelist.apfluxadu(apnum=0)[0].max()}'
+        # )
         assert sourcelist.x.min() == pytest.approx( 16.0, abs=0.1 )
         assert sourcelist.x.max() == pytest.approx( 2039.6, abs=0.1 )
         assert sourcelist.y.min() == pytest.approx( 16.264, abs=0.1 )
@@ -209,7 +232,7 @@ def test_run_psfex( decam_datastore, extractor ):
         assert psf._header['CHI2'] == pytest.approx( 0.9, abs=0.1 )
         bio = io.BytesIO( psf._info.encode( 'utf-8' ) )
         psfstats = votable.parse( bio ).get_table_by_index(1)
-        assert psfstats.array['FWHM_FromFluxRadius_Max'] == pytest.approx( 4.31, abs=0.01 )
+        assert psfstats.array['FWHM_FromFluxRadius_Max'] == pytest.approx( 4.33, abs=0.01 )
         assert not tmppsffile.exists()
         assert not tmppsfxmlfile.exists()
 
@@ -253,29 +276,28 @@ def test_extract_sources_sextractor( decam_datastore, extractor, provenance_base
                 if use:
                     ofp.write( f"image;circle({x+1},{y+1},6) # color=blue width=2\n" )
 
-    assert sources.num_sources == 5500
+    assert sources.num_sources > 5000
     assert sources.num_sources == len(sources.data)
-    assert sources.aper_rads == pytest.approx( [ 2.885, 4.286, 8.572, 12.858,
-                                                 17.145, 21.431, 30.003, 42.862 ], abs=0.01 )
-    assert sources._inf_aper_num == 5
-    assert sources.inf_aper_num == 5
+    expected_radii = np.array([1.0, 2.0, 3.0, 5.0]) * psf.fwhm_pixels
+    assert sources.aper_rads == pytest.approx(expected_radii, abs=0.01 )
+    assert sources.inf_aper_num == -1
     assert psf.fwhm_pixels == pytest.approx( 4.286, abs=0.01 )
     assert psf.fwhm_pixels == pytest.approx( psf.header['PSF_FWHM'], rel=1e-5 )
     assert psf.data.shape == ( 6, 25, 25 )
     assert psf.image_id == ds.image.id
 
-    assert sources.apfluxadu()[0].min() == pytest.approx( 200.34559, rel=1e-5 )
-    assert sources.apfluxadu()[0].max() == pytest.approx( 1105999.625, rel=1e-5 )
-    assert sources.apfluxadu()[0].mean() == pytest.approx( 36779.797 , rel=1e-5 )
-    assert sources.apfluxadu()[0].std() == pytest.approx(  121950.04 , rel=1e-5 )
+    assert sources.apfluxadu()[0].min() == pytest.approx( 275, rel=0.01 )
+    assert sources.apfluxadu()[0].max() == pytest.approx( 2230000, rel=0.01 )
+    assert sources.apfluxadu()[0].mean() == pytest.approx( 54000, rel=0.01 )
+    assert sources.apfluxadu()[0].std() == pytest.approx( 196000, rel=0.01 )
 
-    assert sources.good.sum() == 3638
+    assert sources.good.sum() == pytest.approx(3000, rel=0.01)
     # This value is what you get using the SPREAD_MODEL parameter
     # assert sources.is_star.sum() == 4870
     # assert ( sources.good & sources.is_star ).sum() == 3593
     # This is what you get with CLASS_STAR
-    assert sources.is_star.sum() == 337
-    assert ( sources.good & sources.is_star ).sum() == 61
+    assert sources.is_star.sum() == pytest.approx(325, rel=0.01)
+    assert ( sources.good & sources.is_star ).sum() == pytest.approx(70, abs=5)
 
     try:  # make sure saving the PSF and source list goes as expected, and cleanup at the end
         psf.provenance = provenance_base
@@ -288,95 +310,12 @@ def test_extract_sources_sextractor( decam_datastore, extractor, provenance_base
         assert re.match(r'\d{3}/c4d_\d{8}_\d{6}_N1_g_Sci_.{6}.sources_.{6}.fits', sources.filepath)
         assert os.path.isfile(os.path.join(data_dir, sources.filepath))
 
+        # TODO: add background object here
+
     finally:  # cleanup
         psf.delete_from_disk_and_database()
         sources.delete_from_disk_and_database()
 
-# TODO : add tests that handle different combinations
-#  of measure_psf and psf being passed to the Detector constructor
-
-
-# TODO: is this test really the same as the one above?
-def test_run_detection_sextractor( decam_datastore, extractor ):
-    ds = decam_datastore
-
-    # det = Detector( method='sextractor', measure_psf=True, threshold=5.0 )
-    extractor.pars.method = 'sextractor'
-    extractor.measure_psf = True
-    extractor.pars.threshold = 5.0
-    extractor.pars.test_parameter = uuid.uuid4().hex
-    ds = extractor.run( ds )
-
-    assert extractor.has_recalculated
-    assert ds.sources.num_sources == 5500
-    assert ds.sources.num_sources == len(ds.sources.data)
-    assert ds.sources.aper_rads == pytest.approx( [ 2.88551706,  4.28627014,  8.57254028, 12.85881042, 17.14508057,
-                                                    21.43135071, 30.00389099, 42.86270142], abs=0.01 )
-    assert ds.sources._inf_aper_num == 5
-    assert ds.sources.inf_aper_num == 5
-    assert ds.psf.fwhm_pixels == pytest.approx( 4.286, abs=0.01 )
-    assert ds.psf.fwhm_pixels == pytest.approx( ds.psf.header['PSF_FWHM'], rel=1e-5 )
-    assert ds.psf.data.shape == ( 6, 25, 25 )
-    assert ds.psf.image_id == ds.image.id
-
-    assert ds.sources.apfluxadu()[0].min() == pytest.approx( 200.3456, rel=1e-5 )
-    assert ds.sources.apfluxadu()[0].max() == pytest.approx( 1105999.6, rel=1e-5 )
-    assert ds.sources.apfluxadu()[0].mean() == pytest.approx( 36779.797, rel=1e-5 )
-    assert ds.sources.apfluxadu()[0].std() == pytest.approx(  121950.04 , rel=1e-5 )
-
-    assert ds.sources.good.sum() == 3638
-    # This value is what you get using the SPREAD_MODEL parameter
-    # assert ds.sources.is_star.sum() == 4870
-    # assert ( ds.sources.good & ds.sources.is_star ).sum() == 3593
-    # This value is what you get using the CLASS_STAR parameter
-    assert ds.sources.is_star.sum() == 337
-    assert ( ds.sources.good & ds.sources.is_star ).sum() == 61
-
-    # TODO : actually think about these psf fluxes and how they compare
-    #  to the aperture fluxes (esp. the large-aperture fluxes).  Try to
-    #  understand what SExtractor psf weighted photometry actually
-    #  does....  Preliminary investigations suggest that something may be
-    #  wrong.
-
-    assert ds.sources.psffluxadu()[0].min() == 0.0
-    assert ds.sources.psffluxadu()[0].max() == pytest.approx( 1725000.0, rel=1e-2 )
-    assert ds.sources.psffluxadu()[0].mean() == pytest.approx( 48000.0, rel=1e-2 )
-    assert ds.sources.psffluxadu()[0].std() == pytest.approx( 170000.0, rel=1e-2 )
-
-    assert ds.sources.provenance is not None
-    assert ds.sources.provenance == ds.psf.provenance
-    assert ds.sources.provenance.process == 'extraction'
-
-    assert ds.image.bkg_mean_estimate == pytest.approx( 179.82, abs=0.1 )
-    assert ds.image.bkg_rms_estimate == pytest.approx( 7.533, abs=0.01 )
-
-    from sqlalchemy.exc import IntegrityError
-
-    try:
-        ds.save_and_commit()
-
-        # Make sure all the files exist
-        archive = get_archive_object()
-        imdir = pathlib.Path( FileOnDiskMixin.local_path )
-        relpaths = []
-        relpaths += [ds.image.filepath + ext for ext in ds.image.filepath_extensions]
-        relpaths += [ds.sources.filepath]
-        relpaths += [ds.psf.filepath + ext for ext in ds.psf.filepath_extensions]
-        for relp in relpaths:
-            assert ( imdir / relp ).is_file()
-            assert archive.get_info( relp ) is not None
-
-        # Make sure the bkg fields in the image database table aren't empty
-
-        with SmartSession() as sess:
-            imgs = sess.query( Image ).filter( Image.id == ds.image.id ).all()
-            assert len(imgs) == 1
-            assert imgs[0].bkg_mean_estimate == pytest.approx( 179.82, abs=0.1 )
-            assert imgs[0].bkg_rms_estimate == pytest.approx( 7.533, abs=0.01 )
-
-    finally:
-        ds.delete_everything()
-
 
 def test_warnings_and_exceptions(decam_datastore, extractor):
     extractor.pars.inject_warnings = 1
diff --git a/tests/pipeline/test_measuring.py b/tests/pipeline/test_measuring.py
index 6d529548..24457dc4 100644
--- a/tests/pipeline/test_measuring.py
+++ b/tests/pipeline/test_measuring.py
@@ -103,8 +103,8 @@ def test_measuring(measurer, decam_cutouts, decam_default_calibrators):
 
     assert np.allclose(m.flux_apertures, 100)  # aperture is irrelevant for delta function
     assert m.flux_psf > 150  # flux is more focused than the PSF, so it will bias the flux to be higher than 100
-    assert m.background == 0
-    assert m.background_err == 0
+    assert m.bkg_mean == 0
+    assert m.bkg_std == 0
     for i in range(3):  # check only the last apertures, that are smaller than cutout square
         assert m.area_apertures[i] == pytest.approx(np.pi * (m.aper_radii[i] + 0.5) ** 2, rel=0.1)
 
@@ -117,8 +117,8 @@ def test_measuring(measurer, decam_cutouts, decam_default_calibrators):
 
     assert np.allclose(m.flux_apertures, 200)
     assert m.flux_psf > 300  # flux is more focused than the PSF, so it will bias the flux to be higher than 100
-    assert m.background == 0
-    assert m.background_err == 0
+    assert m.bkg_mean == 0
+    assert m.bkg_std == 0
 
     m = ds.all_measurements[2]  # gaussian
     assert m.disqualifier_scores['negatives'] < 1.0
@@ -127,13 +127,12 @@ def test_measuring(measurer, decam_cutouts, decam_default_calibrators):
     assert m.disqualifier_scores['filter bank'] == 0
     assert m.get_filter_description() == f'PSF match (FWHM= 1.00 x {fwhm:.2f})'
 
-    assert m.flux_apertures[0] < 900
-    assert m.flux_apertures[1] < 1000
-    for i in range(2, len(m.flux_apertures)):
+    assert m.flux_apertures[0] < 1000
+    for i in range(1, len(m.flux_apertures)):
         assert m.flux_apertures[i] == pytest.approx(1000, rel=0.1)
     assert m.flux_psf == pytest.approx(1000, rel=0.1)
-    assert m.background == pytest.approx(0, abs=0.01)
-    assert m.background_err == pytest.approx(0, abs=0.01)
+    assert m.bkg_mean == pytest.approx(0, abs=0.01)
+    assert m.bkg_std == pytest.approx(0, abs=0.01)
 
     # TODO: add test for PSF flux when it is implemented
 
@@ -143,13 +142,12 @@ def test_measuring(measurer, decam_cutouts, decam_default_calibrators):
     assert m.disqualifier_scores['offsets'] == pytest.approx(np.sqrt(2 ** 2 + 3 ** 2), abs=1.0)
     assert m.disqualifier_scores['filter bank'] == 0
 
-    assert m.flux_apertures[0] < 450
-    assert m.flux_apertures[1] < 500
-    for i in range(2, len(m.flux_apertures)):
+    assert m.flux_apertures[0] < 500
+    for i in range(1, len(m.flux_apertures)):
         assert m.flux_apertures[i] == pytest.approx(500, rel=0.1)
     assert m.flux_psf == pytest.approx(500, rel=0.1)
-    assert m.background == pytest.approx(0, abs=0.01)
-    assert m.background_err == pytest.approx(0, abs=0.01)
+    assert m.bkg_mean == pytest.approx(0, abs=0.01)
+    assert m.bkg_std == pytest.approx(0, abs=0.01)
 
     m = ds.all_measurements[4]  # dipole
     assert m.disqualifier_scores['negatives'] == pytest.approx(1.0, abs=0.1)
@@ -160,9 +158,8 @@ def test_measuring(measurer, decam_cutouts, decam_default_calibrators):
     # the dipole's large offsets will short-circuit the iterative repositioning of the aperture (should be flagged!)
     assert all(np.isnan(m.flux_apertures))
     assert all(np.isnan(m.area_apertures))
-    assert m.background == 0
-    assert m.background_err == 0
-    assert m.background_err == 0
+    assert m.bkg_std == 0
+    assert m.bkg_std == 0
 
     m = ds.all_measurements[5]  # shifted gaussian with noise
     assert m.disqualifier_scores['negatives'] < 1.0
@@ -171,15 +168,14 @@ def test_measuring(measurer, decam_cutouts, decam_default_calibrators):
     assert m.disqualifier_scores['filter bank'] == 0
     assert m.get_filter_description() == f'PSF match (FWHM= 1.00 x {fwhm:.2f})'
 
-    assert m.flux_apertures[0] < 450
-    assert m.flux_apertures[1] < 500
-    for i in range(2, len(m.flux_apertures)):
+    assert m.flux_apertures[0] < 500
+    for i in range(1, len(m.flux_apertures)):
         assert m.flux_apertures[i] == pytest.approx(500, rel=0.1)
 
     m = ds.all_measurements[6]  # dipole with noise
     assert m.disqualifier_scores['negatives'] == pytest.approx(1.0, abs=0.2)
     assert m.disqualifier_scores['bad pixels'] == 0
-    assert m.disqualifier_scores['offsets'] > 10
+    assert m.disqualifier_scores['offsets'] > 1
     assert m.disqualifier_scores['filter bank'] > 0
 
     m = ds.all_measurements[7]  # delta function with bad pixel
@@ -209,14 +205,13 @@ def test_measuring(measurer, decam_cutouts, decam_default_calibrators):
     assert m.disqualifier_scores['filter bank'] == 2
     assert m.get_filter_description() == f'PSF mismatch (FWHM= 2.00 x {fwhm:.2f})'
 
-    assert m.flux_apertures[0] < 400
-    assert m.flux_apertures[1] < 600
-    for i in range(2, len(m.flux_apertures)):
+    assert m.flux_apertures[0] < 600
+    for i in range(1, len(m.flux_apertures)):
         assert m.flux_apertures[i] == pytest.approx(1000, rel=1)
     assert m.flux_psf < 500  # flux is more spread out than the PSF, so it will bias the flux to be lower
 
-    assert m.background == pytest.approx(0, abs=0.2)
-    assert m.background_err == pytest.approx(1.0, abs=0.2)
+    assert m.bkg_mean == pytest.approx(0, abs=0.2)
+    assert m.bkg_std == pytest.approx(1.0, abs=0.2)
 
     m = ds.all_measurements[11]  # streak
     assert m.disqualifier_scores['negatives'] < 0.5
@@ -224,8 +219,8 @@ def test_measuring(measurer, decam_cutouts, decam_default_calibrators):
     assert m.disqualifier_scores['offsets'] < 0.7
     assert m.disqualifier_scores['filter bank'] == 28
     assert m.get_filter_description() == 'Streaked (angle= 25.0 deg)'
-    assert m.background < 0.5
-    assert m.background_err < 3.0
+    assert m.bkg_mean < 0.5
+    assert m.bkg_std < 3.0
 
     m = ds.all_measurements[12]  # regular cutout with a bad flag
     assert m.disqualifier_scores['bad_flag'] == 2 ** 41  # this is the bit for 'cosmic ray'
diff --git a/tests/pipeline/test_photo_cal.py b/tests/pipeline/test_photo_cal.py
index ceb5f8eb..ee274a61 100644
--- a/tests/pipeline/test_photo_cal.py
+++ b/tests/pipeline/test_photo_cal.py
@@ -79,4 +79,4 @@ def test_warnings_and_exceptions(decam_datastore, photometor):
         ds = photometor.run(decam_datastore)
         ds.reraise()
     assert "Exception injected by pipeline parameters in process 'photo_cal'." in str(excinfo.value)
-    ds.read_exception()
\ No newline at end of file
+    ds.read_exception()
diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py
index df045e5b..4ab0bfb3 100644
--- a/tests/pipeline/test_pipeline.py
+++ b/tests/pipeline/test_pipeline.py
@@ -170,7 +170,7 @@ def test_parameters( test_config ):
         'subtraction': { 'method': 'override' },
         'detection': { 'threshold': 3.14 },
         'cutting': { 'cutout_size': 666 },
-        'measuring': { 'chosen_aperture': 1 }
+        'measuring': { 'outlier_sigma': 3.5 }
     }
     pipelinemodule = {
         'preprocessing': 'preprocessor',
@@ -209,6 +209,7 @@ def test_data_flow(decam_exposure, decam_reference, decam_default_calibrators, a
     sec_id = ref.section_id
     try:  # cleanup the file at the end
         p = Pipeline()
+        p.pars.save_before_subtraction = False
         assert p.extractor.pars.threshold != 3.14
         assert p.detector.pars.threshold != 3.14
 
@@ -230,8 +231,8 @@ def test_data_flow(decam_exposure, decam_reference, decam_default_calibrators, a
 
             check_datastore_and_database_have_everything(exposure.id, sec_id, ref.image.id, session, ds)
 
-        # feed the pipeline the same data, but missing the upstream data. TODO: add cutouts and measurements
-        attributes = ['image', 'sources', 'wcs', 'zp', 'sub_image', 'detections']
+        # feed the pipeline the same data, but missing the upstream data.
+        attributes = ['image', 'sources', 'sub_image', 'detections', 'cutouts', 'measurements']
 
         for i in range(len(attributes)):
             for j in range(i + 1):
@@ -286,6 +287,7 @@ def test_bitflag_propagation(decam_exposure, decam_reference, decam_default_cali
 
     try:  # cleanup the file at the end
         p = Pipeline()
+        p.pars.save_before_subtraction = False
         exposure.badness = 'banding'  # add a bitflag to check for propagation
 
         # first run the pipeline and check for basic propagation of the single bitflag
@@ -295,6 +297,7 @@ def test_bitflag_propagation(decam_exposure, decam_reference, decam_default_cali
         assert ds.image._upstream_bitflag == 2
         assert ds.sources._upstream_bitflag == 2
         assert ds.psf._upstream_bitflag == 2
+        assert ds.bg._upstream_bitflag == 2
         assert ds.wcs._upstream_bitflag == 2
         assert ds.zp._upstream_bitflag == 2
         assert ds.sub_image._upstream_bitflag == 2
@@ -305,6 +308,7 @@ def test_bitflag_propagation(decam_exposure, decam_reference, decam_default_cali
         # test part 2: Add a second bitflag partway through and check it propagates to downstreams
 
         # delete downstreams of ds.sources
+        ds.bg = None
         ds.wcs = None
         ds.zp = None
         ds.sub_image = None
@@ -406,16 +410,20 @@ def test_get_upstreams_and_downstreams(decam_exposure, decam_reference, decam_de
             assert [upstream.id for upstream in ds.wcs.get_upstreams(session)] == [ds.sources.id]
             assert [upstream.id for upstream in ds.psf.get_upstreams(session)] == [ds.image.id]
             assert [upstream.id for upstream in ds.zp.get_upstreams(session)] == [ds.sources.id]
-            assert [upstream.id for upstream in ds.sub_image.get_upstreams(session)] == [ref.image.id,
-                                                                                  ref.image.sources.id,
-                                                                                  ref.image.psf.id,
-                                                                                  ref.image.wcs.id,
-                                                                                  ref.image.zp.id,
-                                                                                  ds.image.id,
-                                                                                  ds.sources.id,
-                                                                                  ds.psf.id,
-                                                                                  ds.wcs.id,
-                                                                                  ds.zp.id]
+            assert set([upstream.id for upstream in ds.sub_image.get_upstreams(session)]) == set([
+                ref.image.id,
+                ref.image.sources.id,
+                ref.image.psf.id,
+                ref.image.bg.id,
+                ref.image.wcs.id,
+                ref.image.zp.id,
+                ds.image.id,
+                ds.sources.id,
+                ds.psf.id,
+                ds.bg.id,
+                ds.wcs.id,
+                ds.zp.id,
+            ])
             assert [upstream.id for upstream in ds.detections.get_upstreams(session)] == [ds.sub_image.id]
             for cutout in ds.cutouts:
                 assert [upstream.id for upstream in cutout.get_upstreams(session)] == [ds.detections.id]
@@ -428,11 +436,14 @@ def test_get_upstreams_and_downstreams(decam_exposure, decam_reference, decam_de
 
             # test get_downstreams
             assert [downstream.id for downstream in ds.exposure.get_downstreams(session)] == [ds.image.id]
-            assert set([downstream.id for downstream in ds.image.get_downstreams(session)]) == set([ds.psf.id,
-                                                                                    ds.sources.id,
-                                                                                    ds.wcs.id,
-                                                                                    ds.zp.id,
-                                                                                    ds.sub_image.id])
+            assert set([downstream.id for downstream in ds.image.get_downstreams(session)]) == set([
+                ds.sources.id,
+                ds.psf.id,
+                ds.bg.id,
+                ds.wcs.id,
+                ds.zp.id,
+                ds.sub_image.id
+            ])
             assert [downstream.id for downstream in ds.sources.get_downstreams(session)] == [ds.sub_image.id]
             assert [downstream.id for downstream in ds.psf.get_downstreams(session)] == [ds.sub_image.id]
             assert [downstream.id for downstream in ds.wcs.get_downstreams(session)] == [ds.sub_image.id]
@@ -539,6 +550,7 @@ def test_inject_warnings_errors(decam_datastore, decam_reference, pipeline_for_t
     obj_to_process_name = {
         'preprocessor': 'preprocessing',
         'extractor': 'detection',
+        'backgrounder': 'backgrounding',
         'astrometor': 'astro_cal',
         'photometor': 'photo_cal',
         'subtractor': 'subtraction',
diff --git a/tests/pipeline/test_subtraction.py b/tests/pipeline/test_subtraction.py
index 9ff3faf3..5998802c 100644
--- a/tests/pipeline/test_subtraction.py
+++ b/tests/pipeline/test_subtraction.py
@@ -72,10 +72,8 @@ def test_subtraction_ptf_zogy(ptf_ref, ptf_supernova_images, subtractor):
     S[ds.sub_image.flags > 0] = np.nan
 
     mu, sigma = sigma_clipping(S)
-    # assert abs(mu) < 0.01  # the mean should be close to zero
-    assert abs(mu) < 0.2  # this is not working perfectly, we need to improve the background removal!
-    # assert abs(sigma - 1) < 0.1  # the standard deviation should be close to 1
-    assert abs(sigma - 1) < 1  # the standard deviation may be also affected by background...
+    assert abs(mu) < 0.1  # the mean should be close to zero
+    assert abs(sigma - 1) < 0.1  # the standard deviation should be close to 1
 
 
 def test_warnings_and_exceptions(decam_datastore, decam_reference, subtractor, decam_default_calibrators):
diff --git a/util/util.py b/util/util.py
index 8c68ccac..37051f0a 100644
--- a/util/util.py
+++ b/util/util.py
@@ -18,6 +18,7 @@
 
 from models.base import SmartSession, safe_mkdir
 
+
 def ensure_file_does_not_exist( filepath, delete=False ):
     """Check if a file exists.  Delete it, or raise an exception, if it does.
 
@@ -348,7 +349,7 @@ def save_fits_image_file(filename, data, header, extname=None, overwrite=True, s
     The path to the file saved (or written to)
 
     """
-
+    filename = str(filename)  # handle pathlib.Path objects
     hdu = fits.ImageHDU( data, name=extname ) if single_file else fits.PrimaryHDU( data )
 
     if isinstance( header, fits.Header ):