Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DM-47010: adapt test expectations to reflect reprocessVisitImage feeding DIA and forced phot #123

Merged
merged 2 commits into from
Nov 7, 2024
Merged

DM-47010: adapt test expectations to reflect reprocessVisitImage feeding DIA and forced phot #123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
73a8ca3
Rewrite number-count checks to use set differences.
TallJimbo Oct 22, 2024
497691a
Update expected counts to reflect PSF failures propagating.
TallJimbo Oct 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions python/lsst/ci/hsc/gen3/data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,18 @@
PSF_MODEL_ROBUSTNESS_FAILURE_DATA_IDS = [
{'visit': 903334, 'detector': 22, 'physical_filter': 'HSC-R'},
]
# The following data IDs fail (with NoWorkFound) in subtractImages with
# insufficient template coverage. There are two other data IDs that succeed
# despite also having less coverage than the threshold.
INSUFFICIENT_TEMPLATE_COVERAGE_FAILURE_DATA_IDS = [
{'visit': 903342, 'detector': 100, 'physical_filter': 'HSC-R'},
{'visit': 904010, 'detector': 100, 'physical_filter': 'HSC-I'},
]

# The following data IDs fail with PSF modeling failures in
# finalizeCharacterization. This is not intentional, and would ideally be
# fixed in the future.
FINAL_PSF_MODEL_FAILURE_DATA_IDS = [
{'visit': 903344, 'detector': 0, 'physical_filter': 'HSC-R'},
{'visit': 903346, 'detector': 1, 'physical_filter': 'HSC-R'},
]
115 changes: 64 additions & 51 deletions tests/test_validate_outputs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,24 @@
import os
import unittest

from lsst.ci.hsc.gen3 import DATA_IDS, ASTROMETRY_FAILURE_DATA_IDS
from lsst.ci.hsc.gen3 import (
DATA_IDS,
ASTROMETRY_FAILURE_DATA_IDS,
INSUFFICIENT_TEMPLATE_COVERAGE_FAILURE_DATA_IDS,
FINAL_PSF_MODEL_FAILURE_DATA_IDS,
)
from lsst.ci.hsc.gen3.tests import MockCheckMixin
from lsst.daf.butler import Butler
from lsst.utils import getPackageDir


def to_set_of_tuples(list_of_dicts):
"""Convert a list of dictionary {visit, detector} data IDs into a set of
(visit, detector) tuples
"""
return {(d["visit"], d["detector"]) for d in list_of_dicts}


class TestValidateOutputs(unittest.TestCase, MockCheckMixin):
"""Check that ci_hsc_gen3 outputs are as expected."""

Expand All @@ -35,12 +47,16 @@ def setUp(self):
instrument="HSC", skymap="discrete/ci_hsc",
writeable=False, collections=["HSC/runs/ci_hsc"])

self._num_exposures = len(DATA_IDS)
self._num_forced_astrom_failures = len(ASTROMETRY_FAILURE_DATA_IDS)
# Four detectors have template coverage < 0.2 soft limit
# Exposures with template cover > 0.2 (expected successes)
self._num_exposures_good_templates = 29
self._num_surprise_diffim_successes = 2
self._raws = to_set_of_tuples(DATA_IDS)
self._forced_astrom_failures = to_set_of_tuples(ASTROMETRY_FAILURE_DATA_IDS)
# Four detectors have template coverage < 0.2 soft limit, but two
# succeed anyway. These are just the failures.
self._insufficient_template_coverage_failures = to_set_of_tuples(
INSUFFICIENT_TEMPLATE_COVERAGE_FAILURE_DATA_IDS
)
# Two detectors fail second-stage PSF modeling, leaving those PSFs None
# in the final visit summary.
self._final_psf_model_failures = to_set_of_tuples(FINAL_PSF_MODEL_FAILURE_DATA_IDS)
self._num_visits = len({data_id["visit"] for data_id in DATA_IDS})
self._num_tracts = 1
self._num_patches = 1
Expand Down Expand Up @@ -131,23 +147,23 @@ def check_sources(self, source_dataset_types, n_expected, min_src, additional_ch

def test_raw(self):
"Test existence of raw exposures."""
self.check_datasets(["raw"], self._num_exposures)
self.check_datasets(["raw"], len(self._raws))

def test_isr_characterize_calibrate(self):
"""Test existence of isr/calibration related files."""
self.check_pipetasks(
["isr", "characterizeImage", "calibrate"],
self._num_exposures,
self._num_exposures
len(self._raws),
len(self._raws)
)
self.check_datasets(
["postISRCCD", "icExp", "icExpBackground", "icSrc", "calexp", "calexpBackground"],
self._num_exposures
len(self._raws)
)
self.check_datasets(["icSrc_schema", "src_schema"], 1)
self.check_sources(
["src"],
self._num_exposures,
len(self._raws),
self._min_sources,
additional_checks=[self.check_aperture_corrections,
self.check_psf_stars_and_flags]
Expand All @@ -157,11 +173,11 @@ def test_source_tables(self):
"""Test existence of source tables."""
self.check_pipetasks(
["writeRecalibratedSourceTable", "transformSourceTable"],
self._num_exposures,
self._num_exposures
len(self._raws),
len(self._raws)
)
self.check_pipetasks(["consolidateSourceTable"], self._num_visits, self._num_visits)
self.check_sources(["sourceTable"], self._num_exposures, self._min_sources)
self.check_sources(["sourceTable"], len(self._raws), self._min_sources)
self.check_sources(["sourceTable_visit"], self._num_visits, self._min_sources)

def test_visit_summary(self):
Expand All @@ -173,7 +189,7 @@ def test_match_catalogs(self):
"""Test existence of srcMatch and srcMatchFull catalogs."""
self.check_datasets(
["srcMatch", "srcMatchFull"],
self._num_exposures - self._num_forced_astrom_failures
len(self._raws - self._forced_astrom_failures)
)

def test_isolated_star_association(self):
Expand Down Expand Up @@ -382,20 +398,22 @@ def test_object_tables(self):

def test_forced_phot_ccd(self):
"""Test existence of forced photometry tables (sources)."""
self.check_pipetasks(["forcedPhotCcd"], self._num_exposures, self._num_exposures)
self.check_pipetasks(["forcedPhotCcd"], len(self._raws), len(self._raws))
# Despite the two detectors with SFM astrometric failures, the external
# calibration files still exist for them, so the forced_src catalogs
# should indeed exist for all detectors. This is not true for the
# final PSF modeling failures, which cause the PVI not to be created
# and forced photometry to skip with NoWorkFound.
self.check_sources(
["forced_src"],
self._num_exposures,
len(self._raws - self._final_psf_model_failures),
self._min_sources,
additional_checks=[self.check_aperture_corrections],
# We only measure psfFlux in single-detector forced photometry.
aperture_algorithms=("base_PsfFlux", ),
)
self.check_datasets(["forced_src_schema"], 1)
# Despite the two detectors with SFM astrometric failures, the external
# calibration files still exist for them, so the forced_src catalogs
# should indeed exist for all detectors.
self.check_datasets(["forced_src"], self._num_exposures)
self.check_datasets(["forced_src"], len(self._raws - self._final_psf_model_failures))

def test_forced_phot_coadd(self):
"""Test existence of forced photometry tables (objects)."""
Expand All @@ -412,25 +430,23 @@ def test_forced_phot_diffim(self):
"""Test existence of forced photometry tables (diffim)."""
self.check_pipetasks(
["forcedPhotDiffim", "forcedPhotCcdOnDiaObjects", "forcedPhotDiffOnDiaObjects"],
self._num_exposures,
self._num_exposures
len(self._raws),
len(self._raws),
)
# No external calibrations are applied to the diffim forced
# measurements, so no tables should be produced the detectors with
# astrometry failures.
# External calibrations are applied to the diffim forced measurements,
# so the astrometry failures don't reduce the counts, but the PSF model
# failures do.
# forced source counts depend on detector/tract overlap.
self.check_sources(
["forced_diff", "forced_diff_diaObject"],
self._num_exposures_good_templates
+ self._num_surprise_diffim_successes
- self._num_forced_astrom_failures,
len(self._raws - self._insufficient_template_coverage_failures - self._final_psf_model_failures),
self._min_diasources
)
self.check_datasets(["forced_diff_schema", "forced_diff_diaObject_schema"], 1)

def test_templates(self):
"""Test existence of templates."""
self.check_pipetasks(["getTemplate"], self._num_exposures, self._num_exposures)
self.check_pipetasks(["getTemplate"], len(self._raws), len(self._raws))
self.check_pipetasks(
["templateGen", "selectGoodSeeingVisits"],
self._num_patches*self._num_bands,
Expand All @@ -439,25 +455,23 @@ def test_templates(self):
# No templates get produced for the detectors with astrometry failures
self.check_datasets(
["goodSeeingDiff_templateExp"],
self._num_exposures - self._num_forced_astrom_failures
len(self._raws - self._forced_astrom_failures),
)
self.check_datasets(["goodSeeingDiff_diaSrc_schema"], 1)

def test_image_difference(self):
"""Test existence of image differences."""
self.check_pipetasks(
["subtractImages", "detectAndMeasureDiaSources"],
self._num_exposures,
self._num_exposures
len(self._raws),
len(self._raws)
)
# No external calibrations are applied to the diffim forced
# measurements, so no tables should be produced the detectors with
# astrometry failures.
# External calibrations are applied to the diffim forced measurements,
# so the astrometry failures don't reduce the counts, but the PSF model
# failures do.
self.check_datasets(
["goodSeeingDiff_differenceExp"],
self._num_exposures_good_templates
+ self._num_surprise_diffim_successes
- self._num_forced_astrom_failures
len(self._raws - self._insufficient_template_coverage_failures - self._final_psf_model_failures)
)
self.check_datasets(["goodSeeingDiff_diaSrc_schema"], 1)

Expand All @@ -473,8 +487,8 @@ def test_forced_source_tables(self):
self.check_pipetasks(
["writeForcedSourceTable",
"writeForcedSourceOnDiaObjectTable"],
self._num_exposures,
self._num_exposures
len(self._raws),
len(self._raws)
)
self.check_pipetasks(
["transformForcedSourceTable",
Expand All @@ -484,20 +498,19 @@ def test_forced_source_tables(self):
1,
1
)
# No external calibrations are applied to the diffim forced
# measurements, so no tables should be produced the detectors with
# astrometry failures.
# External calibrations are applied to the diffim forced measurements,
# so the astrometry failures don't reduce the counts, but the PSF model
# failures do.
self.check_sources(
["forced_diff_diaObject"],
self._num_exposures_good_templates
+ self._num_surprise_diffim_successes
- self._num_forced_astrom_failures,
len(self._raws - self._insufficient_template_coverage_failures - self._final_psf_model_failures),
self._min_diasources
)
# There are fewer forced sources
self.check_sources(["forced_src_diaObject"], self._num_exposures, self._min_diasources)
self.check_sources(["forced_src_diaObject"], len(self._raws - self._final_psf_model_failures),
self._min_diasources)
self.check_datasets(["forced_diff_diaObject_schema", "forced_src_diaObject_schema"], 1)
self.check_datasets(["forced_src_diaObject"], self._num_exposures)
self.check_datasets(["forced_src_diaObject"], len(self._raws - self._final_psf_model_failures))

def test_skymap(self):
"""Test existence of skymap."""
Expand All @@ -506,7 +519,7 @@ def test_skymap(self):
def test_skycorr(self):
"""Test existence of skycorr."""
self.check_pipetasks(["skyCorr"], self._num_visits, self._num_visits)
self.check_datasets(["skyCorr"], self._num_exposures)
self.check_datasets(["skyCorr"], len(self._raws))

def check_aperture_corrections(self, catalog, aperture_algorithms=("base_PsfFlux", "base_GaussianFlux"),
**kwargs):
Expand Down
Loading