From 0611d7bbee2a21493d1fb70a6eb0176a0c7caf1b Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Tue, 10 Sep 2024 09:50:05 -0700 Subject: [PATCH 1/5] Add coordinate errors to test schemas --- python/lsst/ip/diffim/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/lsst/ip/diffim/utils.py b/python/lsst/ip/diffim/utils.py index 970e0254..1e6bd51d 100644 --- a/python/lsst/ip/diffim/utils.py +++ b/python/lsst/ip/diffim/utils.py @@ -1095,6 +1095,8 @@ def detectTestSources(exposure, addMaskPlanes=None): addMaskPlanes = ["STREAK", "INJECTED", "INJECTED_TEMPLATE"] schema = afwTable.SourceTable.makeMinimalSchema() + # Add coordinate error fields: + afwTable.CoordKey.addErrorFields(schema) selectDetection = measAlg.SourceDetectionTask(schema=schema) selectMeasurement = measBase.SingleFrameMeasurementTask(schema=schema) table = afwTable.SourceTable.make(schema) @@ -1267,6 +1269,8 @@ def _makeTruthSchema(): Calib, Ap, and Psf flux slots all are set to ``truth_instFlux``. """ schema = afwTable.SourceTable.makeMinimalSchema() + # Add coordinate error fields: + afwTable.CoordKey.addErrorFields(schema) keys = {} # Don't use a FluxResultKey so we can manage the flux and err separately. keys["instFlux"] = schema.addField("truth_instFlux", type=np.float64, From ac59a76f31539d32e54b0f129ef78f00cb64f045 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Tue, 10 Sep 2024 10:05:06 -0700 Subject: [PATCH 2/5] Refactor subtraction unit tests to configure Task in common method --- tests/test_subtractTask.py | 206 ++++++++++++++++--------------------- 1 file changed, 88 insertions(+), 118 deletions(-) diff --git a/tests/test_subtractTask.py b/tests/test_subtractTask.py index 399a1284..2166de88 100644 --- a/tests/test_subtractTask.py +++ b/tests/test_subtractTask.py @@ -38,7 +38,29 @@ from lsst.pex.exceptions import InvalidParameterError -class AlardLuptonSubtractTest(lsst.utils.tests.TestCase): +class AlardLuptonSubtractTestBase: + def _setup_subtraction(self, **kwargs): + """Setup and configure the image subtraction PipelineTask. + + Parameters + ---------- + **kwargs + Any additional config parameters to set. + + Returns + ------- + `lsst.pipe.base.PipelineTask` + The configured Task to use for detection and measurement. + """ + config = self.subtractTask.ConfigClass() + config.doSubtractBackground = False + config.update(**kwargs) + + return self.subtractTask(config=config) + + +class AlardLuptonSubtractTest(AlardLuptonSubtractTestBase, lsst.utils.tests.TestCase): + subtractTask = subtractImages.AlardLuptonSubtractTask def test_allowed_config_modes(self): """Verify the allowable modes for convolution. @@ -59,8 +81,7 @@ def test_mismatched_template(self): ySize = 200 science, sources = makeTestImage(psfSize=2.4, xSize=xSize + 20, ySize=ySize + 20) template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction() with self.assertRaises(AssertionError): task.run(template, science, sources) @@ -74,8 +95,7 @@ def test_mismatched_filter(self): band="g", physicalFilter="g noCamera") template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True, band="not-g", physicalFilter="not-g noCamera") - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction() with self.assertRaises(AssertionError): task.run(template, science, sources) @@ -99,16 +119,16 @@ def _run_and_check_coverage(template_coverage, template_height = int(science_height*template_coverage + border) template_cut.image.array[:, template_height:] = 0 template_cut.mask.array[:, template_height:] = template_cut.mask.getPlaneBitMask('NO_DATA') - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.requiredTemplateFraction = requiredTemplateFraction - config.minTemplateFractionForExpectedSuccess = minTemplateFractionForExpectedSuccess + task = self._setup_subtraction( + requiredTemplateFraction=requiredTemplateFraction, + minTemplateFractionForExpectedSuccess=minTemplateFractionForExpectedSuccess + ) if template_coverage < requiredTemplateFraction: doRaise = True elif template_coverage < minTemplateFractionForExpectedSuccess: doRaise = True else: doRaise = False - task = subtractImages.AlardLuptonSubtractTask(config=config) if doRaise: with self.assertRaises(NoWorkFound): task.run(template_cut, science.clone(), sources.copy(deep=True)) @@ -124,9 +144,7 @@ def test_clear_template_mask(self): template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, templateBorderSize=20, doApplyCalibration=True) diffimEmptyMaskPlanes = ["DETECTED", "DETECTED_NEGATIVE"] - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.doSubtractBackground = False - config.mode = "convolveTemplate" + task = self._setup_subtraction(mode="convolveTemplate") # Ensure that each each mask plane is set for some pixels mask = template.mask x0 = 50 @@ -139,13 +157,12 @@ def test_clear_template_mask(self): mask.array[x0: x1, y0: y1] |= mask.getPlaneBitMask(maskPlane) self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) > 0)) - task = subtractImages.AlardLuptonSubtractTask(config=config) output = task.run(template, science, sources) # Verify that the template mask has been modified in place for maskPlane in mask.getMaskPlaneDict().keys(): if maskPlane in diffimEmptyMaskPlanes: self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) == 0)) - elif maskPlane in config.preserveTemplateMask: + elif maskPlane in task.config.preserveTemplateMask: self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) > 0)) else: self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) == 0)) @@ -167,9 +184,7 @@ def test_equal_images(self): science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=6) template, _ = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=7, templateBorderSize=20, doApplyCalibration=True) - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.doSubtractBackground = False - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction() output = task.run(template, science, sources) # There shoud be no NaN values in the difference image self.assertTrue(np.all(np.isfinite(output.difference.image.array))) @@ -193,9 +208,7 @@ def test_equal_images_missing_mask_planes(self): science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=6, addMaskPlanes=[]) template, _ = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=7, templateBorderSize=20, doApplyCalibration=True, addMaskPlanes=[]) - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.doSubtractBackground = False - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction() output = task.run(template, science, sources) # There shoud be no NaN values in the difference image self.assertTrue(np.all(np.isfinite(output.difference.image.array))) @@ -257,9 +270,7 @@ def test_psf_size(self): ) # Test that the image subtraction task runs successfully. - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.doSubtractBackground = False - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction() # Test that the task runs if we take the mean FWHM on a grid. with self.assertLogs(level="INFO") as cm: @@ -281,15 +292,10 @@ def test_auto_convolveTemplate(self): science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6) template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, templateBorderSize=20, doApplyCalibration=True) - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.doSubtractBackground = False - config.mode = "convolveTemplate" - - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction(mode="convolveTemplate") output = task.run(template.clone(), science.clone(), sources) - config.mode = "auto" - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction(mode="auto") outputAuto = task.run(template, science, sources) self.assertMaskedImagesEqual(output.difference.maskedImage, outputAuto.difference.maskedImage) @@ -301,15 +307,10 @@ def test_auto_convolveScience(self): science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=6) template, _ = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=7, templateBorderSize=20, doApplyCalibration=True) - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.doSubtractBackground = False - config.mode = "convolveScience" - - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction(mode="convolveScience") output = task.run(template.clone(), science.clone(), sources) - config.mode = "auto" - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction(mode="auto") outputAuto = task.run(template, science, sources) self.assertMaskedImagesEqual(output.difference.maskedImage, outputAuto.difference.maskedImage) @@ -324,10 +325,7 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat science, sources = makeTestImage(psfSize=2.0, noiseLevel=scienceNoiseLevel, noiseSeed=6) template, _ = makeTestImage(psfSize=3.0, noiseLevel=templateNoiseLevel, noiseSeed=7, templateBorderSize=20, doApplyCalibration=True) - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.doSubtractBackground = False - config.mode = "convolveScience" - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction(mode="convolveScience") output = task.run(template, science, sources) self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05) self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05) @@ -362,9 +360,7 @@ def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templat science, sources = makeTestImage(psfSize=3.0, noiseLevel=scienceNoiseLevel, noiseSeed=6) template, _ = makeTestImage(psfSize=2.0, noiseLevel=templateNoiseLevel, noiseSeed=7, templateBorderSize=20, doApplyCalibration=True) - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.doSubtractBackground = False - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction() output = task.run(template, science, sources) self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05) self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05) @@ -401,10 +397,7 @@ def test_symmetry(self): noiseSeed=6, templateBorderSize=0, doApplyCalibration=True) template, _ = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=7, templateBorderSize=0, doApplyCalibration=True) - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.mode = 'auto' - config.doSubtractBackground = False - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction(mode='auto') # The science image will be modified in place, so use a copy for the second run. science_better = task.run(template.clone(), science.clone(), sources) @@ -432,8 +425,7 @@ def test_few_sources(self): ySize = 256 science, sources = makeTestImage(psfSize=2.4, nSrc=10, xSize=xSize, ySize=ySize) template, _ = makeTestImage(psfSize=2.0, nSrc=10, xSize=xSize, ySize=ySize, doApplyCalibration=True) - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction() sources = sources[0:1] with self.assertRaisesRegex(RuntimeError, "Cannot compute PSF matching kernel: too few sources selected."): @@ -455,12 +447,11 @@ def _run_and_check_sources(sourcesIn, maxKernelSources=1000, minKernelSources=3) sources = sourcesIn.copy(deep=True) # Verify that source flags are not set in the input catalog self.assertEqual(np.sum(sources[badSourceFlag]), 0) - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.badSourceFlags = [badSourceFlag, ] - config.maxKernelSources = maxKernelSources - config.minKernelSources = minKernelSources - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction(badSourceFlags=[badSourceFlag, ], + maxKernelSources=maxKernelSources, + minKernelSources=minKernelSources, + ) nSources = len(sources) # Flag a third of the sources sources[0:: 3][badSourceFlag] = True @@ -497,10 +488,7 @@ def test_order_equal_images(self): template1, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=seed2, templateBorderSize=0, doApplyCalibration=True, clearEdgeMask=True) - config1 = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config1.mode = "convolveTemplate" - config1.doSubtractBackground = False - task1 = subtractImages.AlardLuptonSubtractTask(config=config1) + task1 = self._setup_subtraction(mode="convolveTemplate") results_convolveTemplate = task1.run(template1, science1, sources1) science2, sources2 = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=seed1, @@ -508,10 +496,7 @@ def test_order_equal_images(self): template2, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=seed2, templateBorderSize=0, doApplyCalibration=True, clearEdgeMask=True) - config2 = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config2.mode = "convolveScience" - config2.doSubtractBackground = False - task2 = subtractImages.AlardLuptonSubtractTask(config=config2) + task2 = self._setup_subtraction(mode="convolveScience") results_convolveScience = task2.run(template2, science2, sources2) bbox = results_convolveTemplate.difference.getBBox().clippedTo( results_convolveScience.difference.getBBox()) @@ -550,7 +535,10 @@ def test_background_subtraction(self): science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=6, background=background_model, xSize=xSize, ySize=ySize, x0=x0, y0=y0) + # Don't use ``self._setup_subtraction()`` here. + # Modifying the config of a subtask is messy. config = subtractImages.AlardLuptonSubtractTask.ConfigClass() + config.doSubtractBackground = True config.makeKernel.kernel.name = "AL" @@ -597,11 +585,9 @@ def _run_and_check_images(science, template, sources, statsCtrl, different configurations of ``doDecorrelation`` and ``doScaleVariance``. """ - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.doSubtractBackground = False - config.doDecorrelation = doDecorrelation - config.doScaleVariance = doScaleVariance - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction(doDecorrelation=doDecorrelation, + doScaleVariance=doScaleVariance, + ) output = task.run(template.clone(), science.clone(), sources) if doScaleVariance: self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], @@ -665,12 +651,10 @@ def _run_and_check_images(science, template, sources, statsCtrl, different configurations of ``doDecorrelation`` and ``doScaleVariance``. """ - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.mode = "convolveScience" - config.doSubtractBackground = False - config.doDecorrelation = doDecorrelation - config.doScaleVariance = doScaleVariance - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction(mode="convolveScience", + doDecorrelation=doDecorrelation, + doScaleVariance=doScaleVariance, + ) output = task.run(template.clone(), science.clone(), sources) if doScaleVariance: self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], @@ -741,14 +725,12 @@ def test_exposure_properties_convolve_template(self): apCorrMap.set(name, lsst.afw.math.ChebyshevBoundedField(science.getBBox(), rng.randn(3, 3))) science.info.setApCorrMap(apCorrMap) - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.mode = "convolveTemplate" - def _run_and_check_images(doDecorrelation): """Check that the metadata is correct with or without decorrelation. """ - config.doDecorrelation = doDecorrelation - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction(mode="convolveTemplate", + doDecorrelation=doDecorrelation, + ) output = task.run(template.clone(), science.clone(), sources) psfOut = output.difference.psf psfAvgPos = psfOut.getAveragePosition() @@ -790,14 +772,12 @@ def test_exposure_properties_convolve_science(self): apCorrMap.set(name, lsst.afw.math.ChebyshevBoundedField(science.getBBox(), rng.randn(3, 3))) science.info.setApCorrMap(apCorrMap) - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - config.mode = "convolveScience" - def _run_and_check_images(doDecorrelation): """Check that the metadata is correct with or without decorrelation. """ - config.doDecorrelation = doDecorrelation - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction(mode="convolveScience", + doDecorrelation=doDecorrelation, + ) output = task.run(template.clone(), science.clone(), sources) if doDecorrelation: # Decorrelation requires recalculating the PSF, @@ -879,8 +859,7 @@ def test_fake_mask_plane_propagation(self): science_fake_masked = (science.mask.array & science.mask.getPlaneBitMask("FAKE")) > 0 template_fake_masked = (template.mask.array & template.mask.getPlaneBitMask("FAKE")) > 0 - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - task = subtractImages.AlardLuptonSubtractTask(config=config) + task = self._setup_subtraction() subtraction = task.run(template, science, sources) # check subtraction mask plane is set where we set the previous masks @@ -906,10 +885,10 @@ def test_metadata_metrics(self): # The metadata fields are attached to the subtractTask, so we do # need to run that; run it for both "good" and "bad" seeing templates - config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - subtractTask_good = subtractImages.AlardLuptonSubtractTask(config=config) + + subtractTask_good = self._setup_subtraction() _ = subtractTask_good.run(template_good.clone(), science.clone(), sources) - subtractTask_bad = subtractImages.AlardLuptonSubtractTask(config=config) + subtractTask_bad = self._setup_subtraction() _ = subtractTask_bad.run(template_bad.clone(), science.clone(), sources) # Test that the diffim limiting magnitudes are computed correctly @@ -947,7 +926,7 @@ def test_metadata_metrics(self): template_offimage.setPsf(custom_offimage_psf) # Test that template PSF size edge cases are handled correctly. - subtractTask_offimage = subtractImages.AlardLuptonSubtractTask(config=config) + subtractTask_offimage = self._setup_subtraction() _ = subtractTask_offimage.run(template_offimage.clone(), science.clone(), sources) # Test that providing no fallbackPsfSize results in a nan template # limiting magnitude. @@ -965,7 +944,8 @@ def test_metadata_metrics(self): self.assertIn('templateLimitingMagnitude', subtractTask_good.metadata) -class AlardLuptonPreconvolveSubtractTest(lsst.utils.tests.TestCase): +class AlardLuptonPreconvolveSubtractTest(AlardLuptonSubtractTestBase, lsst.utils.tests.TestCase): + subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask def test_mismatched_template(self): """Test that an error is raised if the template @@ -975,8 +955,7 @@ def test_mismatched_template(self): ySize = 200 science, sources = makeTestImage(psfSize=2.4, xSize=xSize + 20, ySize=ySize + 20) template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) - config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() - task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) + task = self._setup_subtraction() with self.assertRaises(AssertionError): task.run(template, science, sources) @@ -992,9 +971,7 @@ def test_equal_images(self): template, _ = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=7, templateBorderSize=20, doApplyCalibration=True, xSize=xSize, ySize=ySize) - config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() - config.doSubtractBackground = False - task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) + task = self._setup_subtraction() output = task.run(template, science, sources) # There shoud be no NaN values in the Score image self.assertTrue(np.all(np.isfinite(output.scoreExposure.image.array))) @@ -1030,14 +1007,13 @@ def _run_and_check_coverage(template_coverage): template_height = int(science_height*template_coverage + border) template_cut.image.array[:, template_height:] = 0 template_cut.mask.array[:, template_height:] = template_cut.mask.getPlaneBitMask('NO_DATA') - config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() - if template_coverage < config.requiredTemplateFraction: + task = self._setup_subtraction() + if template_coverage < task.config.requiredTemplateFraction: doRaise = True - elif template_coverage < config.minTemplateFractionForExpectedSuccess: + elif template_coverage < task.config.minTemplateFractionForExpectedSuccess: doRaise = True else: doRaise = False - task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) if doRaise: with self.assertRaises(NoWorkFound): task.run(template_cut, science.clone(), sources.copy(deep=True)) @@ -1057,8 +1033,8 @@ def test_clear_template_mask(self): templateBorderSize=20, doApplyCalibration=True, xSize=xSize, ySize=ySize) diffimEmptyMaskPlanes = ["DETECTED", "DETECTED_NEGATIVE"] - config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() - config.doSubtractBackground = False # Ensure that each each mask plane is set for some pixels + task = self._setup_subtraction() + # Ensure that each each mask plane is set for some pixels mask = template.mask x0 = 50 x1 = 75 @@ -1070,13 +1046,12 @@ def test_clear_template_mask(self): mask.array[x0: x1, y0: y1] |= mask.getPlaneBitMask(maskPlane) self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) > 0)) - task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) output = task.run(template, science, sources) # Verify that the template mask has been modified in place for maskPlane in mask.getMaskPlaneDict().keys(): if maskPlane in diffimEmptyMaskPlanes: self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) == 0)) - elif maskPlane in config.preserveTemplateMask: + elif maskPlane in task.config.preserveTemplateMask: self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) > 0)) else: self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) == 0)) @@ -1106,9 +1081,7 @@ def test_agnostic_template_psf(self): template2, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=8, doApplyCalibration=True, xSize=xSize, ySize=ySize) - config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() - config.doSubtractBackground = False - task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) + task = self._setup_subtraction() science_better = task.run(template1, science.clone(), sources) template_better = task.run(template2, science, sources) @@ -1138,8 +1111,7 @@ def test_few_sources(self): ySize = 256 science, sources = makeTestImage(psfSize=2.4, nSrc=10, xSize=xSize, ySize=ySize) template, _ = makeTestImage(psfSize=2.0, nSrc=10, xSize=xSize, ySize=ySize, doApplyCalibration=True) - config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() - task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) + task = self._setup_subtraction() sources = sources[0:1] with self.assertRaisesRegex(RuntimeError, "Cannot compute PSF matching kernel: too few sources selected."): @@ -1165,7 +1137,10 @@ def test_background_subtraction(self): science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=6, background=background_model, xSize=xSize, ySize=ySize, x0=x0, y0=y0) + # Don't use ``self._setup_subtraction()`` here. + # Modifying the config of a subtask is messy. config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() + config.doSubtractBackground = True config.makeKernel.kernel.name = "AL" @@ -1209,11 +1184,9 @@ def _run_and_check_images(science, template, sources, statsCtrl, different configurations of ``doDecorrelation`` and ``doScaleVariance``. """ - config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() - config.doSubtractBackground = False - config.doDecorrelation = doDecorrelation - config.doScaleVariance = doScaleVariance - task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) + task = self._setup_subtraction(doDecorrelation=doDecorrelation, + doScaleVariance=doScaleVariance, + ) output = task.run(template.clone(), science.clone(), sources) if doScaleVariance: self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], @@ -1288,13 +1261,10 @@ def test_exposure_properties(self): templateBorderSize=20, doApplyCalibration=True, xSize=xSize, ySize=ySize) - config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() - def _run_and_check_images(doDecorrelation): """Check that the metadata is correct with or without decorrelation. """ - config.doDecorrelation = doDecorrelation - task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config) + task = self._setup_subtraction(doDecorrelation=doDecorrelation) output = task.run(template.clone(), science.clone(), sources) psfOut = output.scoreExposure.psf psfAvgPos = psfOut.getAveragePosition() From ff1e36463e92abfd1a691a4ff0b0afb9b75ccd25 Mon Sep 17 00:00:00 2001 From: Meredith Rawls Date: Mon, 9 Sep 2024 16:55:03 -0700 Subject: [PATCH 3/5] Switch to ScienceSourceSelectorTask --- python/lsst/ip/diffim/subtractImages.py | 45 +++++++++++++++---------- python/lsst/ip/diffim/utils.py | 10 ++++++ tests/test_detectAndMeasure.py | 2 ++ tests/test_subtractTask.py | 27 +++++++++++---- 4 files changed, 60 insertions(+), 24 deletions(-) diff --git a/python/lsst/ip/diffim/subtractImages.py b/python/lsst/ip/diffim/subtractImages.py index cf6917ac..00e56d9d 100644 --- a/python/lsst/ip/diffim/subtractImages.py +++ b/python/lsst/ip/diffim/subtractImages.py @@ -27,7 +27,7 @@ import lsst.afw.math import lsst.geom from lsst.ip.diffim.utils import evaluateMeanPsfFwhm, getPsfFwhm -from lsst.meas.algorithms import ScaleVarianceTask +from lsst.meas.algorithms import ScaleVarianceTask, ScienceSourceSelectorTask import lsst.pex.config import lsst.pipe.base import lsst.pex.exceptions @@ -173,6 +173,10 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config): dtype=bool, default=False, ) + sourceSelector = lsst.pex.config.ConfigurableField( + target=ScienceSourceSelectorTask, + doc="Task to select sources to be used for PSF matching.", + ) detectionThreshold = lsst.pex.config.Field( dtype=float, default=10, @@ -206,6 +210,8 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config): "base_PixelFlags_flag_saturated", "base_PixelFlags_flag_bad", ), + deprecated="This field is no longer used and will be removed after v28." + "Set the equivalent field for the sourceSelector subtask instead.", ) excludeMaskPlanes = lsst.pex.config.ListField( dtype=str, @@ -240,6 +246,13 @@ def setDefaults(self): self.makeKernel.kernel.active.fitForBackground = self.doSubtractBackground self.makeKernel.kernel.active.spatialKernelOrder = 1 self.makeKernel.kernel.active.spatialBgOrder = 2 + self.sourceSelector.doUnresolve = True # apply star-galaxy separation + self.sourceSelector.doIsolated = True + self.sourceSelector.requirePrimary = True + self.sourceSelector.doSkySources = False # Do not include sky sources + self.sourceSelector.doSignalToNoise = True # apply signal to noise filter + self.sourceSelector.signalToNoise.minimum = 10 + self.sourceSelector.signalToNoise.maximum = 500 class AlardLuptonSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig, @@ -265,6 +278,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.makeSubtask("decorrelate") self.makeSubtask("makeKernel") + self.makeSubtask("sourceSelector") if self.config.doScaleVariance: self.makeSubtask("scaleVariance") @@ -772,26 +786,23 @@ def _sourceSelector(self, sources, mask): If there are too few sources to compute the PSF matching kernel remaining after source selection. """ - flags = np.ones(len(sources), dtype=bool) - for flag in self.config.badSourceFlags: - try: - flags *= ~sources[flag] - except Exception as e: - self.log.warning("Could not apply source flag: %s", e) - signalToNoise = sources.getPsfInstFlux()/sources.getPsfInstFluxErr() - sToNFlag = signalToNoise > self.config.detectionThreshold - flags *= sToNFlag - sToNFlagMax = signalToNoise < self.config.detectionThresholdMax - flags *= sToNFlagMax - flags *= self._checkMask(mask, sources, self.config.excludeMaskPlanes) - selectSources = sources[flags].copy(deep=True) + + selected = self.sourceSelector.selectSources(sources).selected + nInitialSelected = np.count_nonzero(selected) + selected *= self._checkMask(mask, sources, self.config.excludeMaskPlanes) + nSelected = np.count_nonzero(selected) + self.log.info("Rejecting %i candidate sources: an excluded template mask plane is set.", + nInitialSelected - nSelected) + selectSources = sources[selected].copy(deep=True) + # Trim selectSources if they exceed ``maxKernelSources``. + # Keep the highest signal-to-noise sources of those selected. if (len(selectSources) > self.config.maxKernelSources) & (self.config.maxKernelSources > 0): signalToNoise = selectSources.getPsfInstFlux()/selectSources.getPsfInstFluxErr() indices = np.argsort(signalToNoise) indices = indices[-self.config.maxKernelSources:] - flags = np.zeros(len(selectSources), dtype=bool) - flags[indices] = True - selectSources = selectSources[flags].copy(deep=True) + selected = np.zeros(len(selectSources), dtype=bool) + selected[indices] = True + selectSources = selectSources[selected].copy(deep=True) self.log.info("%i/%i=%.1f%% of sources selected for PSF matching from the input catalog", len(selectSources), len(sources), 100*len(selectSources)/len(sources)) diff --git a/python/lsst/ip/diffim/utils.py b/python/lsst/ip/diffim/utils.py index 1e6bd51d..b64cc014 100644 --- a/python/lsst/ip/diffim/utils.py +++ b/python/lsst/ip/diffim/utils.py @@ -1284,6 +1284,11 @@ def _makeTruthSchema(): schema.addField("base_PixelFlags_flag_interpolated", "Flag", "testing flag.") schema.addField("base_PixelFlags_flag_saturated", "Flag", "testing flag.") schema.addField("base_PixelFlags_flag_bad", "Flag", "testing flag.") + schema.addField("base_PixelFlags_flag_edge", "Flag", "testing flag.") + schema.addField("base_PsfFlux_flag", "Flag", "testing flag.") + schema.addField("base_ClassificationSizeExtendedness_value", "Flag", "testing flag.") + schema.addField("deblend_nChild", "Flag", "testing flag.") + schema.addField("detect_isPrimary", "Flag", "testing flag.") schema.getAliasMap().set("slot_Centroid", "truth") schema.getAliasMap().set("slot_CalibFlux", "truth") schema.getAliasMap().set("slot_ApFlux", "truth") @@ -1320,6 +1325,11 @@ def _fillTruthCatalog(injectList): footprint.addPeak(x, y, flux) record.setFootprint(footprint) + # Set source records for isolated stars + record["base_ClassificationSizeExtendedness_value"] = 0 + record["deblend_nChild"] = 0 + record["detect_isPrimary"] = True + return catalog diff --git a/tests/test_detectAndMeasure.py b/tests/test_detectAndMeasure.py index 9ef9bfbf..b329f622 100644 --- a/tests/test_detectAndMeasure.py +++ b/tests/test_detectAndMeasure.py @@ -488,6 +488,8 @@ def test_fake_mask_plane_propagation(self): template_fake_masked = (template.mask.array & template_fake_bitmask) > 0 subtractConfig = subtractImages.AlardLuptonSubtractTask.ConfigClass() + subtractConfig.sourceSelector.signalToNoise.fluxField = "truth_instFlux" + subtractConfig.sourceSelector.signalToNoise.errField = "truth_instFluxErr" subtractTask = subtractImages.AlardLuptonSubtractTask(config=subtractConfig) subtraction = subtractTask.run(template, science, sources) diff --git a/tests/test_subtractTask.py b/tests/test_subtractTask.py index 2166de88..d29a6993 100644 --- a/tests/test_subtractTask.py +++ b/tests/test_subtractTask.py @@ -54,6 +54,14 @@ def _setup_subtraction(self, **kwargs): """ config = self.subtractTask.ConfigClass() config.doSubtractBackground = False + config.sourceSelector.signalToNoise.fluxField = "truth_instFlux" + config.sourceSelector.signalToNoise.errField = "truth_instFluxErr" + config.sourceSelector.doUnresolved = True + config.sourceSelector.doIsolated = True + config.sourceSelector.doRequirePrimary = True + config.sourceSelector.doFlags = True + config.sourceSelector.doSignalToNoise = True + config.sourceSelector.flags.bad = ["base_PsfFlux_flag", ] config.update(**kwargs) return self.subtractTask(config=config) @@ -441,17 +449,18 @@ def test_kernel_source_selector(self): xSize=xSize, ySize=ySize) template, _ = makeTestImage(psfSize=2.0, nSrc=nSourcesSimulated, xSize=xSize, ySize=ySize, doApplyCalibration=True) - badSourceFlag = "slot_Centroid_flag" def _run_and_check_sources(sourcesIn, maxKernelSources=1000, minKernelSources=3): sources = sourcesIn.copy(deep=True) - # Verify that source flags are not set in the input catalog - self.assertEqual(np.sum(sources[badSourceFlag]), 0) - task = self._setup_subtraction(badSourceFlags=[badSourceFlag, ], - maxKernelSources=maxKernelSources, + task = self._setup_subtraction(maxKernelSources=maxKernelSources, minKernelSources=minKernelSources, ) + # Verify that source flags are not set in the input catalog + # Note that this will use the last flag in the list for the rest of + # the test. + for badSourceFlag in task.sourceSelector.config.flags.bad: + self.assertEqual(np.sum(sources[badSourceFlag]), 0) nSources = len(sources) # Flag a third of the sources sources[0:: 3][badSourceFlag] = True @@ -538,7 +547,9 @@ def test_background_subtraction(self): # Don't use ``self._setup_subtraction()`` here. # Modifying the config of a subtask is messy. config = subtractImages.AlardLuptonSubtractTask.ConfigClass() - + + config.sourceSelector.signalToNoise.fluxField = "truth_instFlux" + config.sourceSelector.signalToNoise.errField = "truth_instFluxErr" config.doSubtractBackground = True config.makeKernel.kernel.name = "AL" @@ -1140,7 +1151,9 @@ def test_background_subtraction(self): # Don't use ``self._setup_subtraction()`` here. # Modifying the config of a subtask is messy. config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass() - + + config.sourceSelector.signalToNoise.fluxField = "truth_instFlux" + config.sourceSelector.signalToNoise.errField = "truth_instFluxErr" config.doSubtractBackground = True config.makeKernel.kernel.name = "AL" From eed26226f1e1c5fe97c8ca4c662ee388c4d2e61c Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Mon, 9 Sep 2024 23:33:49 -0700 Subject: [PATCH 4/5] Add constraints for isolated stars to psf match source selector --- python/lsst/ip/diffim/subtractImages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/lsst/ip/diffim/subtractImages.py b/python/lsst/ip/diffim/subtractImages.py index 00e56d9d..c765192a 100644 --- a/python/lsst/ip/diffim/subtractImages.py +++ b/python/lsst/ip/diffim/subtractImages.py @@ -246,9 +246,9 @@ def setDefaults(self): self.makeKernel.kernel.active.fitForBackground = self.doSubtractBackground self.makeKernel.kernel.active.spatialKernelOrder = 1 self.makeKernel.kernel.active.spatialBgOrder = 2 - self.sourceSelector.doUnresolve = True # apply star-galaxy separation - self.sourceSelector.doIsolated = True - self.sourceSelector.requirePrimary = True + self.sourceSelector.doUnresolved = True # apply star-galaxy separation + self.sourceSelector.doIsolated = True # apply isolated star selection + self.sourceSelector.doRequirePrimary = True # apply primary flag selection self.sourceSelector.doSkySources = False # Do not include sky sources self.sourceSelector.doSignalToNoise = True # apply signal to noise filter self.sourceSelector.signalToNoise.minimum = 10 From 0b9d46f047865be56b4dd8dd64e727f3a907ed76 Mon Sep 17 00:00:00 2001 From: Ian Sullivan Date: Mon, 9 Sep 2024 23:35:40 -0700 Subject: [PATCH 5/5] Extend template mask check to neighboring pixels Exclude sources from being used for PSF matching if any of the 3x3 central pixels are masked. --- python/lsst/ip/diffim/subtractImages.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/python/lsst/ip/diffim/subtractImages.py b/python/lsst/ip/diffim/subtractImages.py index c765192a..bc7d5262 100644 --- a/python/lsst/ip/diffim/subtractImages.py +++ b/python/lsst/ip/diffim/subtractImages.py @@ -816,8 +816,8 @@ def _sourceSelector(self, sources, mask): return selectSources @staticmethod - def _checkMask(mask, sources, excludeMaskPlanes): - """Exclude sources that are located on masked pixels. + def _checkMask(mask, sources, excludeMaskPlanes, checkAdjacent=True): + """Exclude sources that are located on or adjacent to masked pixels. Parameters ---------- @@ -842,11 +842,18 @@ def _checkMask(mask, sources, excludeMaskPlanes): excludePixelMask = mask.getPlaneBitMask(setExcludeMaskPlanes) - xv = np.rint(sources.getX() - mask.getX0()) - yv = np.rint(sources.getY() - mask.getY0()) + xv = (np.rint(sources.getX() - mask.getX0())).astype(int) + yv = (np.rint(sources.getY() - mask.getY0())).astype(int) - mv = mask.array[yv.astype(int), xv.astype(int)] - flags = np.bitwise_and(mv, excludePixelMask) == 0 + flags = np.ones(len(sources), dtype=bool) + if checkAdjacent: + pixRange = (0, -1, 1) + else: + pixRange = (0,) + for j in pixRange: + for i in pixRange: + mv = mask.array[yv + j, xv + i] + flags *= np.bitwise_and(mv, excludePixelMask) == 0 return flags def _prepareInputs(self, template, science, visitSummary=None):