Debug measurements test

.github/workflows/run-pipeline-tests.yml → .github/workflows/run-model-tests-X.yml

@@ -1,4 +1,4 @@
-name: Run Pipeline Tests
+name: Run Model Tests X
 
 on:
   push:
@@ -59,4 +59,5 @@ jobs:
 
       - name: run test
         run: |
-          TEST_SUBFOLDER=tests/pipeline docker compose run runtests
+          shopt -s nullglob
+          TEST_SUBFOLDER=$(ls tests/models/test_{x..z}*.py) docker compose run runtests

default_config.yaml

@@ -82,23 +82,24 @@ preprocessing:
   use_sky_subtraction: False
 
 extraction:
-  measure_psf: true
-  threshold: 3.0
-  method: sextractor
-
-astro_cal:
-  cross_match_catalog: gaia_dr3
-  solution_method: scamp
-  max_catalog_mag: [20.0]
-  mag_range_catalog: 4.0
-  min_catalog_stars: 50
-  max_sources_to_use: [2000, 1000, 500, 200]
-
-photo_cal:
-  cross_match_catalog: gaia_dr3
-  max_catalog_mag: [20.0]
-  mag_range_catalog: 4.0
-  min_catalog_stars: 50
+  sources:
+    measure_psf: true
+    threshold: 3.0
+    method: sextractor
+
+  wcs:
+    cross_match_catalog: gaia_dr3
+    solution_method: scamp
+    max_catalog_mag: [20.0]
+    mag_range_catalog: 4.0
+    min_catalog_stars: 50
+    max_sources_to_use: [2000, 1000, 500, 200]
+
+  zp:
+    cross_match_catalog: gaia_dr3
+    max_catalog_mag: [20.0]
+    mag_range_catalog: 4.0
+    min_catalog_stars: 50
 
 subtraction:
   method: zogy
@@ -170,22 +171,23 @@ coaddition:
       ignore_flags: 0
   # The following are used to override the regular "extraction" parameters
   extraction:
-    measure_psf: true
-    threshold: 3.0
-    method: sextractor
-    # The following are used to override the regular "astro_cal" parameters
-  astro_cal:
-    cross_match_catalog: gaia_dr3
-    solution_method: scamp
-    max_catalog_mag: [22.0]
-    mag_range_catalog: 6.0
-    min_catalog_stars: 50
-    # The following are used to override the regular "photo_cal" parameters
-  photo_cal:
-    cross_match_catalog: gaia_dr3
-    max_catalog_mag: [22.0]
-    mag_range_catalog: 6.0
-    min_catalog_stars: 50
+    sources:
+      measure_psf: true
+      threshold: 3.0
+      method: sextractor
+    # The following are used to override the regular astrometric calibration parameters
+    wcs:
+      cross_match_catalog: gaia_dr3
+      solution_method: scamp
+      max_catalog_mag: [22.0]
+      mag_range_catalog: 6.0
+      min_catalog_stars: 50
+    # The following are used to override the regular photometric calibration parameters
+    zp:
+      cross_match_catalog: gaia_dr3
+      max_catalog_mag: [22.0]
+      mag_range_catalog: 6.0
+      min_catalog_stars: 50
 
 
 # DECam

docs/setup.md

@@ -49,7 +49,7 @@ By default, the volumes with archived files and the database files will still be
    docker compose down -v
 ```
 
-If all is well, the `-v` will delete the volumnes that stored the database and archive files.
+If all is well, the `-v` will delete the volumes that stored the database and archive files.
 
 You can see what volumes docker knows about with
 ```

github_temp/run-pipeline-tests-1.yml

@@ -0,0 +1,64 @@
+name: Run Pipeline Tests 1
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  tests:
+    name: run tests in docker image
+    runs-on: ubuntu-latest
+    env:
+      REGISTRY: ghcr.io
+      COMPOSE_FILE: tests/docker-compose.yaml
+
+    steps:
+      - name: Dump docker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+
+      - name: checkout code
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: log into github container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: setup docker buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver: docker-container
+
+      - name: bake
+        uses: docker/[email protected]
+        with:
+          workdir: tests
+          load: true
+          files: docker-compose.yaml
+          set: |
+            seechange_postgres.tags=ghcr.io/${{ github.repository_owner }}/seechange-postgres
+            seechange_postgres.cache-from=type=gha,scope=cached-seechange-postgres
+            seechange_postgres.cache-to=type=gha,scope=cached-seechange-postgres,mode=max
+            setuptables.tags=ghcr.io/${{ github.repository_owner }}/runtests
+            setuptables.cache-from=type=gha,scope=cached-seechange
+            setuptables.cache-to=type=gha,scope=cached-seechange,mode=max
+            runtests.tags=ghcr.io/${{ github.repository_owner }}/runtests
+            runtests.cache-from=type=gha,scope=cached-seechange
+            runtests.cache-to=type=gha,scope=cached-seechange,mode=max
+            shell.tags=ghcr.io/${{ github.repository_owner }}/runtests
+            shell.cache-from=type=gha,scope=cached-seechange
+            shell.cache-to=type=gha,scope=cached-seechange,mode=max
+
+      - name: run test
+        run: |
+          df -h
+          shopt -s nullglob
+          TEST_SUBFOLDER=$(ls tests/pipeline/test_{a..o}*.py) docker compose run runtests

github_temp/run-pipeline-tests-2.yml

@@ -0,0 +1,63 @@
+name: Run Pipeline Tests 2
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  tests:
+    name: run tests in docker image
+    runs-on: ubuntu-latest
+    env:
+      REGISTRY: ghcr.io
+      COMPOSE_FILE: tests/docker-compose.yaml
+
+    steps:
+      - name: Dump docker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+
+      - name: checkout code
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: log into github container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: setup docker buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver: docker-container
+
+      - name: bake
+        uses: docker/[email protected]
+        with:
+          workdir: tests
+          load: true
+          files: docker-compose.yaml
+          set: |
+            seechange_postgres.tags=ghcr.io/${{ github.repository_owner }}/seechange-postgres
+            seechange_postgres.cache-from=type=gha,scope=cached-seechange-postgres
+            seechange_postgres.cache-to=type=gha,scope=cached-seechange-postgres,mode=max
+            setuptables.tags=ghcr.io/${{ github.repository_owner }}/runtests
+            setuptables.cache-from=type=gha,scope=cached-seechange
+            setuptables.cache-to=type=gha,scope=cached-seechange,mode=max
+            runtests.tags=ghcr.io/${{ github.repository_owner }}/runtests
+            runtests.cache-from=type=gha,scope=cached-seechange
+            runtests.cache-to=type=gha,scope=cached-seechange,mode=max
+            shell.tags=ghcr.io/${{ github.repository_owner }}/runtests
+            shell.cache-from=type=gha,scope=cached-seechange
+            shell.cache-to=type=gha,scope=cached-seechange,mode=max
+
+      - name: run test
+        run: |
+          shopt -s nullglob
+          TEST_SUBFOLDER=$(ls tests/pipeline/test_{p..z}*.py) docker compose run runtests

improc/alignment.py

@@ -414,7 +414,7 @@ def _align_swarp( self, image, target, sources, target_sources ):
 
             # re-calculate the source list and PSF for the warped image
             extractor = Detector()
-            extractor.pars.override(sources.provenance.parameters, ignore_addons=True)
+            extractor.pars.override(sources.provenance.parameters['sources'], ignore_addons=True)
             warpedsrc, warpedpsf, _, _ = extractor.extract_sources(warpedim)
             warpedim.sources = warpedsrc
             warpedim.psf = warpedpsf

improc/photometry.py

@@ -419,6 +419,8 @@ def calc_at_position(data, radius, annulus, xgrid, ygrid, cx, cy, local_bg=True,
         the iterative process.
     """
     flux = area = background = variance = norm = cxx = cyy = cxy = 0
+    if np.all(np.isnan(data)):
+        return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
 
     # make a circle-mask based on the centroid position
     if not np.isfinite(cx) or not np.isfinite(cy):
@@ -447,7 +449,8 @@ def calc_at_position(data, radius, annulus, xgrid, ygrid, cx, cy, local_bg=True,
             return flux, area, background, variance, norm, cx, cy, cxx, cyy, cxy, True
 
         annulus_map_sum = np.nansum(annulus_map)
-        if annulus_map_sum == 0:  # this should only happen in tests or if the annulus is way too large
+        if annulus_map_sum == 0 or np.all(np.isnan(annulus_map)):
+            # this should only happen in tests or if the annulus is way too large or if all pixels are NaN
             background = 0
             variance = 0
             norm = 0

improc/sextrsky.py

@@ -9,6 +9,7 @@
 
 from util.logger import SCLogger
 
+
 def single_sextrsky( imagedata, maskdata=None, sigcut=3 ):
     """Estimate sky and sky sigma of imagedata (ignoreing nonzero maskdata pixels)
 
@@ -66,6 +67,7 @@ def single_sextrsky( imagedata, maskdata=None, sigcut=3 ):
     skysig = 1.4826 * ( np.median( np.abs( imagedata[w] - sky ) ) )
     return sky, skysig
 
+
 def sextrsky( imagedata, maskdata=None, sigcut=3, boxsize=200, filtsize=3 ):
     """Estimate sky using an approximation of the SExtractor algorithm.
 
@@ -178,6 +180,7 @@ def sextrsky( imagedata, maskdata=None, sigcut=3, boxsize=200, filtsize=3 ):
 
 # ======================================================================
 
+
 def main():
     parser = argparse.ArgumentParser( description="Estimate image sky using sextractor algorithm" )
     parser.add_argument( "image", help="Image filename" )

improc/tools.py

@@ -52,13 +52,15 @@ def sigma_clipping(values, nsigma=3.0, iterations=5, axis=None, median=False):
             raise ValueError("values must be a vector, image, or cube")
 
     values = values.copy()
-
-    # first iteration:
-    mean = np.nanmedian(values, axis=axis)
-    rms = np.nanstd(values, axis=axis)
 
     # how many nan values?
     nans = np.isnan(values).sum()
+    if nans == values.size:
+        return np.nan, np.nan
+
+    # first iteration:
+    mean = np.nanmedian(values, axis=axis)
+    rms = np.nanstd(values, axis=axis)
 
     for i in range(iterations):
         # remove pixels that are more than nsigma from the median