diff --git a/pipelines/LATISS/cpBfk.yaml b/pipelines/LATISS/cpBfk.yaml index f7f07c00..f91a2a2d 100644 --- a/pipelines/LATISS/cpBfk.yaml +++ b/pipelines/LATISS/cpBfk.yaml @@ -1,4 +1,17 @@ -description: Latiss Brighter-Fatter kernel calibration construction. +description: LATISS Brighter-Fatter kernel calibration construction. instrument: lsst.obs.lsst.Latiss imports: - - location: $CP_PIPE_DIR/pipelines/_ingredients/cpBfk.yaml + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpBfkLSST.yaml +tasks: + cpBfkPtcExtract: + class: lsst.cp.pipe.ptc.PhotonTransferCurveExtractTask + config: + # Limit the covariance range unless we get a super-dense + # PTC run for LATISS. + maximumRangeCovariancesAstier: 8 + efdSalIndex: 201 + cpBfkPtcSolve: + class: lsst.cp.pipe.ptc.PhotonTransferCurveSolveTask + config: + maximumRangeCovariancesAstier: 8 + maximumRangeCovariancesAstierFullCovFit: 8 diff --git a/pipelines/LATISS/cpBias.yaml b/pipelines/LATISS/cpBias.yaml index 973eb924..6093a369 100644 --- a/pipelines/LATISS/cpBias.yaml +++ b/pipelines/LATISS/cpBias.yaml @@ -1,11 +1,9 @@ -description: Latiss bias calibration construction +description: LATISS bias calibration construction instrument: lsst.obs.lsst.Latiss imports: - - location: $CP_PIPE_DIR/pipelines/_ingredients/cpBias.yaml + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpBiasLSST.yaml tasks: cpBiasIsr: - class: lsst.ip.isr.IsrTask + class: lsst.ip.isr.IsrTaskLSST config: - overscan.fitType: 'MEDIAN_PER_ROW' - doDefect: false - doCrosstalk: true + crosstalk.doQuadraticCrosstalkCorrection: false diff --git a/pipelines/LATISS/cpBiasBootstrap.yaml b/pipelines/LATISS/cpBiasBootstrap.yaml new file mode 100644 index 00000000..a8b3c96e --- /dev/null +++ b/pipelines/LATISS/cpBiasBootstrap.yaml @@ -0,0 +1,4 @@ +description: LATISS bootstrap bias calibration construction +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpBiasBootstrapLSST.yaml diff --git a/pipelines/LATISS/cpDark.yaml b/pipelines/LATISS/cpDark.yaml index 7be9d7f1..e622ef73 100644 --- a/pipelines/LATISS/cpDark.yaml +++ b/pipelines/LATISS/cpDark.yaml @@ -1,15 +1,9 @@ -# Note that if you edit this file you may also need to edit -# cpDarkForDefects.yaml. -description: Latiss dark calibration construction +description: LATISS dark calibration construction instrument: lsst.obs.lsst.Latiss imports: - - location: $CP_PIPE_DIR/pipelines/_ingredients/cpDark.yaml + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpDarkLSST.yaml tasks: cpDarkIsr: - class: lsst.ip.isr.IsrTask + class: lsst.ip.isr.IsrTaskLSST config: - overscan.fitType: 'MEDIAN_PER_ROW' - doLinearize: false - doCrosstalk: true - # This is necessary for cosmic ray rejection. - doDefect: true + crosstalk.doQuadraticCrosstalkCorrection: false diff --git a/pipelines/LATISS/cpDarkBootstrap.yaml b/pipelines/LATISS/cpDarkBootstrap.yaml new file mode 100644 index 00000000..0098a577 --- /dev/null +++ b/pipelines/LATISS/cpDarkBootstrap.yaml @@ -0,0 +1,4 @@ +description: LATISS bootstrap dark calibration construction +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpDarkBootstrapLSST.yaml diff --git a/pipelines/LATISS/cpDefects.yaml b/pipelines/LATISS/cpDefects.yaml index bf681f1c..56733d2c 100644 --- a/pipelines/LATISS/cpDefects.yaml +++ b/pipelines/LATISS/cpDefects.yaml @@ -1,7 +1,7 @@ -description: cp_pipe DEFECT calibration construction from combined images. +description: cp_pipe LATISS defect calibration construction. instrument: lsst.obs.lsst.Latiss imports: - - location: $CP_PIPE_DIR/pipelines/_ingredients/cpDefects.yaml + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpDefectsLSST.yaml tasks: cpMeasureBiasDefects: class: lsst.cp.pipe.MeasureDefectsCombinedTask diff --git a/pipelines/LATISS/cpFilterScan.yaml b/pipelines/LATISS/cpFilterScan.yaml index 8cf6f1d3..59192b28 100644 --- a/pipelines/LATISS/cpFilterScan.yaml +++ b/pipelines/LATISS/cpFilterScan.yaml @@ -1,15 +1,8 @@ -description: Latiss filter scan construction +description: LATISS filter scan construction instrument: lsst.obs.lsst.Latiss imports: - - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFilterScan.yaml + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFilterScanLSST.yaml tasks: - cpFilterScanIsr: - class: lsst.ip.isr.IsrTask - config: - connections.outputExposure: 'cpFilterScanIsrExp' - overscan.fitType: 'MEDIAN_PER_ROW' - doLinearize: false - doCrosstalk: false cpFilterScanMerge: class: lsst.cp.pipe.CpFilterScanTask config: diff --git a/pipelines/LATISS/cpFlat.yaml b/pipelines/LATISS/cpFlat.yaml index 371ff431..1d949d64 100644 --- a/pipelines/LATISS/cpFlat.yaml +++ b/pipelines/LATISS/cpFlat.yaml @@ -1,11 +1,9 @@ -description: Latiss flat calibration construction +description: LATISS flat calibration construction instrument: lsst.obs.lsst.Latiss imports: - - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFlatSingleChip.yaml + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFlatSingleChipLSST.yaml tasks: cpFlatIsr: - class: lsst.ip.isr.IsrTask + class: lsst.ip.isr.IsrTaskLSST config: - overscan.fitType: "MEDIAN_PER_ROW" - doLinearize: false - doCrosstalk: true + crosstalk.doQuadraticCrosstalkCorrection: false diff --git a/pipelines/LATISS/cpFlatBootstrap.yaml b/pipelines/LATISS/cpFlatBootstrap.yaml new file mode 100644 index 00000000..b62f6851 --- /dev/null +++ b/pipelines/LATISS/cpFlatBootstrap.yaml @@ -0,0 +1,4 @@ +description: LATISS bootstrap flat construction +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFlatBootstrapSingleChipLSST.yaml diff --git a/pipelines/LATISS/cpFlatSingleChip.yaml b/pipelines/LATISS/cpFlatSingleChip.yaml index 23dcf75a..1d949d64 100644 --- a/pipelines/LATISS/cpFlatSingleChip.yaml +++ b/pipelines/LATISS/cpFlatSingleChip.yaml @@ -1,11 +1,9 @@ -description: cp_pipe FLAT calibration construction +description: LATISS flat calibration construction instrument: lsst.obs.lsst.Latiss imports: - - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFlatSingleChip.yaml + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFlatSingleChipLSST.yaml tasks: cpFlatIsr: - class: lsst.ip.isr.IsrTask + class: lsst.ip.isr.IsrTaskLSST config: - overscan.fitType: "MEDIAN_PER_ROW" - doLinearize: false - doCrosstalk: true + crosstalk.doQuadraticCrosstalkCorrection: false diff --git a/pipelines/LATISS/cpLinearizer.yaml b/pipelines/LATISS/cpLinearizer.yaml index a64d617d..484594b7 100644 --- a/pipelines/LATISS/cpLinearizer.yaml +++ b/pipelines/LATISS/cpLinearizer.yaml @@ -1,11 +1,14 @@ -description: cp_pipe linearity calibration construction. +description: cp_pipe LATISS linearizer calibration construction. instrument: lsst.obs.lsst.Latiss imports: - - location: $CP_PIPE_DIR/pipelines/_ingredients/cpLinearizer.yaml + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpLinearizerLSST.yaml tasks: + cpLinearizerPtcExtract: + class: lsst.cp.pipe.ptc.PhotonTransferCurveExtractTask + config: + efdSalIndex: 201 cpLinearizerSolve: class: lsst.cp.pipe.LinearitySolveTask config: - linearityType: Spline - splineKnots: 10 maxLinearAdu: 120000 + usePhotodiode: false diff --git a/pipelines/LATISS/cpPlotPtc.yaml b/pipelines/LATISS/cpPlotPtc.yaml index a553fa74..b7e2a8fc 100644 --- a/pipelines/LATISS/cpPlotPtc.yaml +++ b/pipelines/LATISS/cpPlotPtc.yaml @@ -1,4 +1,4 @@ -description: cp_pipe task to plot Photon Transfer Curve dataset +description: cp_pipe LATISS task to plot Photon Transfer Curve dataset instrument: lsst.obs.lsst.Latiss imports: - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPlotPtc.yaml diff --git a/pipelines/LATISS/cpPtc.yaml b/pipelines/LATISS/cpPtc.yaml index 0e975633..bd0ed0f4 100644 --- a/pipelines/LATISS/cpPtc.yaml +++ b/pipelines/LATISS/cpPtc.yaml @@ -1,18 +1,9 @@ -description: Latiss Photon-Transfer Curve calibration construction. +description: LATISS Photon-Transfer Curve calibration construction. instrument: lsst.obs.lsst.Latiss imports: - - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPtc.yaml + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPtcLSST.yaml tasks: - cpPtcIsr: - class: lsst.ip.isr.IsrTask - config: - doCrosstalk: false - doDefect: true cpPtcExtract: class: lsst.cp.pipe.ptc.PhotonTransferCurveExtractTask config: - matchExposuresType: TIME - cpPtcSolve: - class: lsst.cp.pipe.ptc.PhotonTransferCurveSolveTask - config: - ptcFitType: EXPAPPROXIMATION + efdSalIndex: 201 diff --git a/pipelines/LATISS/cpSky.yaml b/pipelines/LATISS/cpSky.yaml index d0834deb..7307a26a 100644 --- a/pipelines/LATISS/cpSky.yaml +++ b/pipelines/LATISS/cpSky.yaml @@ -1,15 +1,12 @@ -description: Sky frame generation pipeline definition. +description: Sky frame generation for LATISS pipeline. instrument: lsst.obs.lsst.Latiss imports: - - location: $CP_PIPE_DIR/pipelines/_ingredients/cpSky.yaml + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpSkyLSST.yaml tasks: cpSkyIsr: - class: lsst.ip.isr.isrTask.IsrTask + class: lsst.ip.isr.IsrTaskLSST config: - overscan.fitType: "MEDIAN_PER_ROW" - doLinearize: false - doCrosstalk: true - doBrighterFatter: false + crosstalk.doQuadraticCrosstalkCorrection: false cpSkyImage: class: lsst.cp.pipe.CpSkyImageTask config: diff --git a/pipelines/LATISS/legacy_202409/cpBfk.yaml b/pipelines/LATISS/legacy_202409/cpBfk.yaml new file mode 100644 index 00000000..f7f07c00 --- /dev/null +++ b/pipelines/LATISS/legacy_202409/cpBfk.yaml @@ -0,0 +1,4 @@ +description: Latiss Brighter-Fatter kernel calibration construction. +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpBfk.yaml diff --git a/pipelines/LATISS/legacy_202409/cpBias.yaml b/pipelines/LATISS/legacy_202409/cpBias.yaml new file mode 100644 index 00000000..973eb924 --- /dev/null +++ b/pipelines/LATISS/legacy_202409/cpBias.yaml @@ -0,0 +1,11 @@ +description: Latiss bias calibration construction +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpBias.yaml +tasks: + cpBiasIsr: + class: lsst.ip.isr.IsrTask + config: + overscan.fitType: 'MEDIAN_PER_ROW' + doDefect: false + doCrosstalk: true diff --git a/pipelines/LATISS/cpCrosstalk.yaml b/pipelines/LATISS/legacy_202409/cpCrosstalk.yaml similarity index 100% rename from pipelines/LATISS/cpCrosstalk.yaml rename to pipelines/LATISS/legacy_202409/cpCrosstalk.yaml diff --git a/pipelines/LATISS/cpCti.yaml b/pipelines/LATISS/legacy_202409/cpCti.yaml similarity index 100% rename from pipelines/LATISS/cpCti.yaml rename to pipelines/LATISS/legacy_202409/cpCti.yaml diff --git a/pipelines/LATISS/legacy_202409/cpDark.yaml b/pipelines/LATISS/legacy_202409/cpDark.yaml new file mode 100644 index 00000000..7be9d7f1 --- /dev/null +++ b/pipelines/LATISS/legacy_202409/cpDark.yaml @@ -0,0 +1,15 @@ +# Note that if you edit this file you may also need to edit +# cpDarkForDefects.yaml. +description: Latiss dark calibration construction +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpDark.yaml +tasks: + cpDarkIsr: + class: lsst.ip.isr.IsrTask + config: + overscan.fitType: 'MEDIAN_PER_ROW' + doLinearize: false + doCrosstalk: true + # This is necessary for cosmic ray rejection. + doDefect: true diff --git a/pipelines/LATISS/cpDarkForDefects.yaml b/pipelines/LATISS/legacy_202409/cpDarkForDefects.yaml similarity index 100% rename from pipelines/LATISS/cpDarkForDefects.yaml rename to pipelines/LATISS/legacy_202409/cpDarkForDefects.yaml diff --git a/pipelines/LATISS/legacy_202409/cpDefects.yaml b/pipelines/LATISS/legacy_202409/cpDefects.yaml new file mode 100644 index 00000000..bf681f1c --- /dev/null +++ b/pipelines/LATISS/legacy_202409/cpDefects.yaml @@ -0,0 +1,29 @@ +description: cp_pipe DEFECT calibration construction from combined images. +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpDefects.yaml +tasks: + cpMeasureBiasDefects: + class: lsst.cp.pipe.MeasureDefectsCombinedTask + config: + badPixelsToFillColumnThreshold: 1000 + saturatedPixelsToFillColumnThreshold: 20 + saturatedColumnDilationRadius: 2 + cpMeasureDarkDefects: + class: lsst.cp.pipe.MeasureDefectsCombinedTask + config: + thresholdType: "VALUE" + darkCurrentThreshold: 3.0 + badPixelsToFillColumnThreshold: 1000 + saturatedPixelsToFillColumnThreshold: 20 + saturatedColumnDilationRadius: 2 + cpMeasureFlatDefects: + class: lsst.cp.pipe.MeasureDefectsCombinedWithFilterTask + config: + thresholdType: "VALUE" + fracThresholdFlat: 0.9 + badPixelsToFillColumnThreshold: 1000 + cpMergeDefects: + class: lsst.cp.pipe.MergeDefectsCombinedTask + config: + edgesAsDefects: true diff --git a/pipelines/LATISS/cpDefectsIndividual.yaml b/pipelines/LATISS/legacy_202409/cpDefectsIndividual.yaml similarity index 100% rename from pipelines/LATISS/cpDefectsIndividual.yaml rename to pipelines/LATISS/legacy_202409/cpDefectsIndividual.yaml diff --git a/pipelines/LATISS/legacy_202409/cpFilterScan.yaml b/pipelines/LATISS/legacy_202409/cpFilterScan.yaml new file mode 100644 index 00000000..8cf6f1d3 --- /dev/null +++ b/pipelines/LATISS/legacy_202409/cpFilterScan.yaml @@ -0,0 +1,16 @@ +description: Latiss filter scan construction +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFilterScan.yaml +tasks: + cpFilterScanIsr: + class: lsst.ip.isr.IsrTask + config: + connections.outputExposure: 'cpFilterScanIsrExp' + overscan.fitType: 'MEDIAN_PER_ROW' + doLinearize: false + doCrosstalk: false + cpFilterScanMerge: + class: lsst.cp.pipe.CpFilterScanTask + config: + referenceFilter: 'empty~empty' diff --git a/pipelines/LATISS/legacy_202409/cpFlat.yaml b/pipelines/LATISS/legacy_202409/cpFlat.yaml new file mode 100644 index 00000000..371ff431 --- /dev/null +++ b/pipelines/LATISS/legacy_202409/cpFlat.yaml @@ -0,0 +1,11 @@ +description: Latiss flat calibration construction +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFlatSingleChip.yaml +tasks: + cpFlatIsr: + class: lsst.ip.isr.IsrTask + config: + overscan.fitType: "MEDIAN_PER_ROW" + doLinearize: false + doCrosstalk: true diff --git a/pipelines/LATISS/legacy_202409/cpFlatSingleChip.yaml b/pipelines/LATISS/legacy_202409/cpFlatSingleChip.yaml new file mode 100644 index 00000000..23dcf75a --- /dev/null +++ b/pipelines/LATISS/legacy_202409/cpFlatSingleChip.yaml @@ -0,0 +1,11 @@ +description: cp_pipe FLAT calibration construction +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpFlatSingleChip.yaml +tasks: + cpFlatIsr: + class: lsst.ip.isr.IsrTask + config: + overscan.fitType: "MEDIAN_PER_ROW" + doLinearize: false + doCrosstalk: true diff --git a/pipelines/LATISS/cpFringe.yaml b/pipelines/LATISS/legacy_202409/cpFringe.yaml similarity index 100% rename from pipelines/LATISS/cpFringe.yaml rename to pipelines/LATISS/legacy_202409/cpFringe.yaml diff --git a/pipelines/LATISS/legacy_202409/cpLinearizer.yaml b/pipelines/LATISS/legacy_202409/cpLinearizer.yaml new file mode 100644 index 00000000..a64d617d --- /dev/null +++ b/pipelines/LATISS/legacy_202409/cpLinearizer.yaml @@ -0,0 +1,11 @@ +description: cp_pipe linearity calibration construction. +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpLinearizer.yaml +tasks: + cpLinearizerSolve: + class: lsst.cp.pipe.LinearitySolveTask + config: + linearityType: Spline + splineKnots: 10 + maxLinearAdu: 120000 diff --git a/pipelines/LATISS/legacy_202409/cpPlotPtc.yaml b/pipelines/LATISS/legacy_202409/cpPlotPtc.yaml new file mode 100644 index 00000000..a553fa74 --- /dev/null +++ b/pipelines/LATISS/legacy_202409/cpPlotPtc.yaml @@ -0,0 +1,4 @@ +description: cp_pipe task to plot Photon Transfer Curve dataset +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPlotPtc.yaml diff --git a/pipelines/LATISS/legacy_202409/cpPtc.yaml b/pipelines/LATISS/legacy_202409/cpPtc.yaml new file mode 100644 index 00000000..0e975633 --- /dev/null +++ b/pipelines/LATISS/legacy_202409/cpPtc.yaml @@ -0,0 +1,18 @@ +description: Latiss Photon-Transfer Curve calibration construction. +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpPtc.yaml +tasks: + cpPtcIsr: + class: lsst.ip.isr.IsrTask + config: + doCrosstalk: false + doDefect: true + cpPtcExtract: + class: lsst.cp.pipe.ptc.PhotonTransferCurveExtractTask + config: + matchExposuresType: TIME + cpPtcSolve: + class: lsst.cp.pipe.ptc.PhotonTransferCurveSolveTask + config: + ptcFitType: EXPAPPROXIMATION diff --git a/pipelines/LATISS/legacy_202409/cpSky.yaml b/pipelines/LATISS/legacy_202409/cpSky.yaml new file mode 100644 index 00000000..d0834deb --- /dev/null +++ b/pipelines/LATISS/legacy_202409/cpSky.yaml @@ -0,0 +1,22 @@ +description: Sky frame generation pipeline definition. +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpSky.yaml +tasks: + cpSkyIsr: + class: lsst.ip.isr.isrTask.IsrTask + config: + overscan.fitType: "MEDIAN_PER_ROW" + doLinearize: false + doCrosstalk: true + doBrighterFatter: false + cpSkyImage: + class: lsst.cp.pipe.CpSkyImageTask + config: + # These values roughly equate to the dimensions of a single detector. + # As noted below, the sizes are in millimeters, and correspond to a + # background image of 4096*4096 pixels (4096*0.01=40.96). + # TODO: Revise these values on DM-43321. + largeScaleBackground.xSize: 40.96 # in mm + largeScaleBackground.ySize: 40.96 # in mm + largeScaleBackground.pixelSize: 0.01 # in mm per pixel diff --git a/pipelines/LATISS/legacy_202409/cpSpectroFlat.yaml b/pipelines/LATISS/legacy_202409/cpSpectroFlat.yaml new file mode 100644 index 00000000..b8e9108b --- /dev/null +++ b/pipelines/LATISS/legacy_202409/cpSpectroFlat.yaml @@ -0,0 +1,4 @@ +description: Sky frame generation pipeline definition. +instrument: lsst.obs.lsst.Latiss +imports: + - location: $CP_PIPE_DIR/pipelines/_ingredients/cpSpectroFlat.yaml diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 4d6515b6..9f9ef5fd 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -134,11 +134,16 @@ def test_cameras(self): @unittest.skipIf(not has_obs_lsst, reason="Cannot test LATISS pipelines without obs_lsst") def test_latiss_pipelines(self): for pipeline in self._get_pipelines(exclude=[ + # The following two tasks are not part of the new pipelines. + "cpDarkForDefects.yaml", + "cpDefectsIndividual.yaml", + # The following task is not defined for LATISS. "cpMonochromatorScan.yaml", - # TODO DM-46356: Remove following from exclusion list. - "cpBiasBootstrap.yaml", - "cpDarkBootstrap.yaml", - "cpFlatBootstrap.yaml", + # The following tasks will be added in the future. + "cpCrosstalk.yaml", + "cpFringe.yaml", + # TODO: DM-46426 + "cpCti.yaml", ]): self._check_pipeline(os.path.join(self.pipeline_path, "LATISS", pipeline))