diff --git a/bin/pipeline.sh b/bin/pipeline.sh index 1a92c8d..f185a2d 100755 --- a/bin/pipeline.sh +++ b/bin/pipeline.sh @@ -53,20 +53,19 @@ HIPS_QGRAPH_FILE=$(mktemp)_hips.qgraph trap 'rm -f $QGRAPH_FILE $INJECTION_QGRAPH_FILE $RESOURCE_USAGE_QGRAPH_FILE \ $HIPS_QGRAPH_FILE' EXIT -# DM-46272: Change maxMeanDistanceArcsec to something smaller, so that it fails; -# even better, move those settings into DRP-ci_hsc.yaml! pipetask --long-log --log-level="$loglevel" qgraph \ -d "skymap='discrete/ci_hsc' AND tract=0 AND patch=69" \ -b "$repo"/butler.yaml \ --input "$INPUTCOLL" --output "$COLLECTION" \ -p "$DRP_PIPE_DIR/pipelines/HSC/DRP-ci_hsc.yaml" \ - -c calibrateImage:astrometry.maxMeanDistanceArcsec=0.20 \ + -c calibrateImage:astrometry.maxMeanDistanceArcsec=0.02398 \ -c makeDirectWarp:select.maxPsfTraceRadiusDelta=0.2 \ --save-qgraph "$QGRAPH_FILE" pipetask --long-log --log-level="$loglevel" run \ -j "$jobs" -b "$repo"/butler.yaml \ --input "$INPUTCOLL" --output "$COLLECTION" \ + --no-raise-on-partial-outputs \ --register-dataset-types $mock \ --qgraph "$QGRAPH_FILE" diff --git a/python/lsst/ci/hsc/gen3/data.py b/python/lsst/ci/hsc/gen3/data.py index 2e8b01f..19c2500 100644 --- a/python/lsst/ci/hsc/gen3/data.py +++ b/python/lsst/ci/hsc/gen3/data.py @@ -54,14 +54,12 @@ {'visit': 903988, 'detector': 24, 'physical_filter': 'HSC-I'}, ] # The following lists the dataIds that fail the astrometry check with -# the config override calibrate.astrometry.maxMeanDistanceArcsec=0.020 +# the config override calibrateImage.astrometry.maxMeanDistanceArcsec=0.02398 # set. This list is sensitive to the astrometry algorithms and dataset # under consideration, so may require updating if either of those change # in the context of this repository. -# DM-46272: not forcing these failures until we can handle partial outputs; -# uncomment this line as that ticket is sorted out. ASTROMETRY_FAILURE_DATA_IDS = [ - # {'visit': 903344, 'detector': 0, 'physical_filter': 'HSC-R'}, + {'visit': 903988, 'detector': 23, 'physical_filter': 'HSC-I'}, ] # The following lists the dataIds that fail the PSF Model robustness check # with the config override makeWarp.select.maxPsfTraceRadiusDelta=0.2 set. diff --git a/tests/test_astrometryFail.py b/tests/test_astrometryFail.py index 54f7d0b..7052c0d 100644 --- a/tests/test_astrometryFail.py +++ b/tests/test_astrometryFail.py @@ -27,10 +27,9 @@ from lsst.daf.butler import Butler, DataCoordinate from lsst.utils import getPackageDir +from lsst.ci.hsc.gen3 import ASTROMETRY_FAILURE_DATA_IDS -# DM-46272: not forcing these failures until we can handle partial outputs; -# remove the expectedFailures as that ticket is sorted out. class TestAstrometryFails(lsst.utils.tests.TestCase): """Tests the outputs of the forced astrometry failures. """ @@ -40,14 +39,13 @@ def setUp(self): # The dataId here represents one of the astrometry fit failures # imposed by setting astrometry.maxMeanDistanceArcsec: 0.020 in # the pipeline. - self.detector = 0 - self.visit = 903344 + self.detector = ASTROMETRY_FAILURE_DATA_IDS[0]["detector"] + self.visit = ASTROMETRY_FAILURE_DATA_IDS[0]["visit"] self.calexpMinimalDataId = DataCoordinate.standardize( instrument="HSC", detector=self.detector, visit=self.visit, universe=self.butler.dimensions, ) - @unittest.expectedFailure def testWcsAndPhotoCalibIsNoneForFailedAstrom(self): """Test the WCS and photoCalib objects attached to failed WCS exposure. @@ -64,7 +62,6 @@ def testWcsAndPhotoCalibIsNoneForFailedAstrom(self): calexpPhotoCalib = self.butler.get("calexp.photoCalib", self.calexpMinimalDataId) self.assertTrue(calexpPhotoCalib is None) - @unittest.expectedFailure def testSrcCoordsAreNanForFailedAstrom(self): """Test coord values in all source catalogs. @@ -103,7 +100,6 @@ def testCentroidsAreNotNanForFailedAstrom(self): self.assertFalse(np.all(np.isnan(sourceCat["x"]))) self.assertFalse(np.all(np.isnan(sourceCat["y"]))) - @unittest.expectedFailure def testVisitCoordsAreNanForFailedAstrom(self): """Test coord and astrom values for visitTable and visitSummary. @@ -117,10 +113,11 @@ def testVisitCoordsAreNanForFailedAstrom(self): self.assertTrue(np.all(np.isfinite(visitTable["ra"]))) visitSummary = self.butler.get("visitSummary", self.calexpMinimalDataId) - self.assertTrue(np.isfinite(visitSummary["id" == self.detector]["astromOffsetMean"])) - self.assertTrue(np.isfinite(visitSummary["id" == self.detector]["astromOffsetStd"])) - self.assertTrue(np.all(np.isnan(visitSummary["id" == 1]["raCorners"]))) - self.assertTrue(np.all(np.isnan(visitSummary["id" == 1]["decCorners"]))) + row = visitSummary.find(self.detector) + self.assertTrue(np.isfinite(row["astromOffsetMean"])) + self.assertTrue(np.isfinite(row["astromOffsetStd"])) + self.assertTrue(np.all(np.isnan(row["raCorners"]))) + self.assertTrue(np.all(np.isnan(row["decCorners"]))) def testMetadataForFailedAstrom(self): """Test that the metadata for a failed astrometic fit is set properly. diff --git a/tests/test_validate_outputs.py b/tests/test_validate_outputs.py index c24254b..2a2f6c6 100644 --- a/tests/test_validate_outputs.py +++ b/tests/test_validate_outputs.py @@ -390,24 +390,41 @@ def test_object_tables(self): self._num_tracts ) + def test_reprocess_visit_image(self): + """Test the existence of ReprocessVisitImageTask's outputs.""" + # While the external WCS files might someday let us recover from the + # forced astrometry failures at this stage, at present that failure + # prevents isolated star association for generating matches for this + # detector, and that prevents the second round of PSF determination and + # aperture correction from working. This manifests as an + # UpstreamFailureNoWorkFound, and hence we expect metadata outputs but + # not source catalogs or PVIs for the forced-failure data IDs. + self.check_pipetasks(["reprocessVisitImage"], len(self._raws), len(self._raws)) + self.check_sources( + ["sources_footprints_detector"], + len(self._raws - self._forced_astrom_failures), + self._min_sources, + additional_checks=[self.check_aperture_corrections] + ) + self.check_datasets(["sources_schema"], 1) + self.check_datasets(["pvi", "pvi_background"], len(self._raws - self._forced_astrom_failures)) + def test_forced_phot_ccd(self): """Test existence of forced photometry tables (sources).""" + # Lack of PVI (from reprocessVisitImage) leads to NoWorkFound for the + # forced astrometry failure data IDs, which affects regular-output + # counts, but not logs or metadata. self.check_pipetasks(["forcedPhotCcd"], len(self._raws), len(self._raws)) - # Despite the two detectors with SFM astrometric failures, the external - # calibration files still exist for them, so the forced_src catalogs - # should indeed exist for all detectors. This is not true for the - # final PSF modeling failures, which cause the PVI not to be created - # and forced photometry to skip with NoWorkFound. self.check_sources( ["forced_src"], - len(self._raws), + len(self._raws - self._forced_astrom_failures), self._min_sources, additional_checks=[self.check_aperture_corrections], # We only measure psfFlux in single-detector forced photometry. aperture_algorithms=("base_PsfFlux", ), ) self.check_datasets(["forced_src_schema"], 1) - self.check_datasets(["forced_src"], len(self._raws)) + self.check_datasets(["forced_src"], len(self._raws - self._forced_astrom_failures)) def test_forced_phot_coadd(self): """Test existence of forced photometry tables (objects).""" @@ -422,18 +439,17 @@ def test_forced_phot_coadd(self): def test_forced_phot_diffim(self): """Test existence of forced photometry tables (diffim).""" + # Lack of PVI (from reprocessVisitImage) leads to NoWorkFound for the + # forced astrometry failure data IDs, which affects regular-output + # counts, but not logs or metadata. self.check_pipetasks( ["forcedPhotDiffim", "forcedPhotCcdOnDiaObjects", "forcedPhotDiffOnDiaObjects"], len(self._raws), len(self._raws), ) - # External calibrations are applied to the diffim forced measurements, - # so the astrometry failures don't reduce the counts, but the PSF model - # failures do. - # forced source counts depend on detector/tract overlap. self.check_sources( ["forced_diff", "forced_diff_diaObject"], - len(self._raws - self._insufficient_template_coverage_failures), + len(self._raws - self._insufficient_template_coverage_failures - self._forced_astrom_failures), self._min_diasources ) self.check_datasets(["forced_diff_schema", "forced_diff_diaObject_schema"], 1) @@ -460,12 +476,12 @@ def test_image_difference(self): len(self._raws), len(self._raws) ) - # External calibrations are applied to the diffim forced measurements, - # so the astrometry failures don't reduce the counts, but the PSF model - # failures do. + # Lack of PVI (from reprocessVisitImage) leads to NoWorkFound for the + # forced astrometry failure data IDs, which affects regular-output + # counts, but not logs or metadata. self.check_datasets( ["goodSeeingDiff_differenceExp"], - len(self._raws - self._insufficient_template_coverage_failures) + len(self._raws - self._insufficient_template_coverage_failures - self._forced_astrom_failures) ) self.check_datasets(["goodSeeingDiff_diaSrc_schema"], 1) @@ -492,19 +508,19 @@ def test_forced_source_tables(self): 1, 1 ) - # External calibrations are applied to the diffim forced measurements, - # so the astrometry failures don't reduce the counts, but the PSF model - # failures do. + # Lack of PVI (from reprocessVisitImage) leads to NoWorkFound for the + # forced astrometry failure data IDs, which affects regular-output + # counts, but not logs or metadata. self.check_sources( ["forced_diff_diaObject"], - len(self._raws - self._insufficient_template_coverage_failures), + len(self._raws - self._insufficient_template_coverage_failures - self._forced_astrom_failures), self._min_diasources ) # There are fewer forced sources - self.check_sources(["forced_src_diaObject"], len(self._raws), + self.check_sources(["forced_src_diaObject"], len(self._raws - self._forced_astrom_failures), self._min_diasources) self.check_datasets(["forced_diff_diaObject_schema", "forced_src_diaObject_schema"], 1) - self.check_datasets(["forced_src_diaObject"], len(self._raws)) + self.check_datasets(["forced_src_diaObject"], len(self._raws - self._forced_astrom_failures)) def test_skymap(self): """Test existence of skymap."""