From 7fd827fdea44ae0c9784e9a960a11f7852478e88 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Fri, 30 Jun 2023 10:39:06 -0400 Subject: [PATCH 1/3] Remove "+fakes" pipelines based on soon-to-be retired insertion code. Pipelines based on the new source_injection package will be added in in the future. --- pipelines/HSC/DRP-RC2+fakes.yaml | 9 -- pipelines/HSC/DRP-ci_hsc+fakes.yaml | 21 ---- pipelines/_ingredients/HSC/DRP+fakes.yaml | 140 ---------------------- 3 files changed, 170 deletions(-) delete mode 100644 pipelines/HSC/DRP-RC2+fakes.yaml delete mode 100644 pipelines/HSC/DRP-ci_hsc+fakes.yaml delete mode 100644 pipelines/_ingredients/HSC/DRP+fakes.yaml diff --git a/pipelines/HSC/DRP-RC2+fakes.yaml b/pipelines/HSC/DRP-RC2+fakes.yaml deleted file mode 100644 index d695d483..00000000 --- a/pipelines/HSC/DRP-RC2+fakes.yaml +++ /dev/null @@ -1,9 +0,0 @@ -description: The DRP Fakes pipeline specalised for the HSC RC2 dataset. -instrument: lsst.obs.subaru.HyperSuprimeCam -imports: - - "$DRP_PIPE_DIR/pipelines/_ingredients/HSC/DRP+fakes.yaml" -tasks: - isolatedStarAssociation: - class: lsst.pipe.tasks.isolatedStarAssociation.IsolatedStarAssociationTask - config: - band_order: ['i', 'r', 'g', 'z', 'y', 'N921'] diff --git a/pipelines/HSC/DRP-ci_hsc+fakes.yaml b/pipelines/HSC/DRP-ci_hsc+fakes.yaml deleted file mode 100644 index a22ec3c9..00000000 --- a/pipelines/HSC/DRP-ci_hsc+fakes.yaml +++ /dev/null @@ -1,21 +0,0 @@ -description: The DRP+fakes pipeline specialized for the ci_hsc test dataset. -instrument: lsst.obs.subaru.HyperSuprimeCam -imports: - location: "$DRP_PIPE_DIR/pipelines/_ingredients/HSC/DRP+fakes.yaml" -tasks: - makeWarp: - class: lsst.pipe.tasks.makeWarp.MakeWarpTask - config: - connections.coaddName: "fakes_deep" - connections.calexpType: "fakes_" - deblend: - class: lsst.pipe.tasks.deblendCoaddSourcesPipeline.DeblendCoaddSourcesMultiTask - config: - connections.inputCoaddName: "fakes_deep" - connections.outputCoaddName: "fakes_deep" - multibandDeblend.useCiLimits: true - multibandDeblend.processSingles: false - forcedPhotCcd: - class: lsst.meas.base.forcedPhotCcd.ForcedPhotCcdTask - config: - connections.inputCoaddName: "fakes_deep" diff --git a/pipelines/_ingredients/HSC/DRP+fakes.yaml b/pipelines/_ingredients/HSC/DRP+fakes.yaml deleted file mode 100644 index bb8b478a..00000000 --- a/pipelines/_ingredients/HSC/DRP+fakes.yaml +++ /dev/null @@ -1,140 +0,0 @@ -description: The DRP Fakes pipeline specalised for the HSC RC2 dataset. -instrument: lsst.obs.subaru.HyperSuprimeCam -imports: - location: "$DRP_PIPE_DIR/pipelines/_ingredients/HSC/DRP.yaml" - exclude: - - skyCorr - - jointcal - - gbdesAstrometricFit - - fgcm - - fgcmBuildFromIsolatedStars - - fgcmFitCycle - - fgcmOutputProducts - - isr - - characterizeImage - - calibrate - - selectGoodSeeingVisits - - templateGen - - subtractImages - - detectAndMeasureDiaSources - - transformDiaSourceCat - - consolidateDiaSourceTable - - drpAssociation - - drpDiaCalculation - - forcedPhotDiffim - - forcedPhotCcdOnDiaObjects - - forcedPhotDiffOnDiaObjects - - transformForcedSourceTable - - consolidateForcedSourceTable - - getTemplate - - writeForcedSourceTable - - consolidateAssocDiaSourceTable - - consolidateFullDiaObjectTable - - writeForcedSourceOnDiaObjectTable - - transformForcedSourceOnDiaObjectTable - - consolidateForcedSourceOnDiaObjectTable - - consolidateVisitSummary - - isolatedStarAssociation - - finalizeCharacterization - - updateVisitSummary - - makeCcdVisitTable - - makeVisitTable - - healSparsePropertyMaps - - consolidateHealSparsePropertyMaps - - analyzeObjectTableCore - - analyzeObjectTableSurveyCore - - catalogMatchTract - - refCatObjectTract - - faro_visit - - faro_matched - - faro_tract -tasks: - singleFrameWithFakes: - class: lsst.pipe.tasks.processCcdWithFakes.ProcessCcdWithFakesTask - config: - insertFakes.doProcessAllDataIds: true - doApplyExternalGlobalPhotoCalib: true - doApplyExternalGlobalSkyWcs: true - writePreSourceTable: - class: lsst.pipe.tasks.postprocess.WriteSourceTableTask - config: - connections.catalogType: "fakes_" - connections.outputCatalog: fakes_preSource - transformPreSourceTable: - class: lsst.pipe.tasks.postprocess.TransformSourceTableTask - config: - connections.inputCatalog: fakes_preSource - connections.outputCatalog: fakes_preSourceTable - consolidatePreSourceTable: - class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask - config: - connections.inputCatalogs: fakes_preSourceTable - connections.outputCatalog: fakes_preSourceTable_visit - writeRecalibratedSourceTable: - class: lsst.pipe.tasks.postprocess.WriteRecalibratedSourceTableTask - config: - connections.outputCatalog: fakes_source - connections.catalogType: "fakes_" - transformSourceTable: - class: lsst.pipe.tasks.postprocess.TransformSourceTableTask - config: - connections.catalogType: "fakes_" - consolidateSourceTable: - class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask - config: - connections.catalogType: "fakes_" - assembleCoadd: - class: lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask - config: - connections.inputCoaddName: "fakes_deep" - connections.outputCoaddName: "fakes_deep" - makeWarp: - class: lsst.pipe.tasks.makeWarp.MakeWarpTask - config: - connections.photoCalibName: "fgcm" - useGlobalExternalPhotoCalib: true - connections.coaddName: "fakes_deep" - connections.calexpType: "fakes_" - detection: - class: lsst.pipe.tasks.multiBand.DetectCoaddSourcesTask - config: - connections.inputCoaddName: "fakes_deep" - connections.outputCoaddName: "fakes_deep" - mergeDetections: - class: lsst.pipe.tasks.mergeDetections.MergeDetectionsTask - config: - connections.inputCoaddName: "fakes_deep" - connections.outputCoaddName: "fakes_deep" - deblend: - class: lsst.pipe.tasks.deblendCoaddSourcesPipeline.DeblendCoaddSourcesMultiTask - config: - connections.inputCoaddName: "fakes_deep" - connections.outputCoaddName: "fakes_deep" - measure: - class: lsst.pipe.tasks.multiBand.MeasureMergedCoaddSourcesTask - config: - connections.inputCoaddName: "fakes_deep" - connections.outputCoaddName: "fakes_deep" - mergeMeasurements: - class: lsst.pipe.tasks.mergeMeasurements.MergeMeasurementsTask - config: - connections.inputCoaddName: "fakes_deep" - connections.outputCoaddName: "fakes_deep" - writeObjectTable: - class: lsst.pipe.tasks.postprocess.WriteObjectTableTask - config: - connections.coaddName: "fakes_deep" - transformObjectTable: - class: lsst.pipe.tasks.postprocess.TransformObjectCatalogTask - config: - connections.coaddName: "fakes_deep" - forcedPhotCcd: - class: lsst.meas.base.forcedPhotCcd.ForcedPhotCcdTask - config: - connections.inputCoaddName: "fakes_deep" - connections.exposure: "fakes_calexp" - forcedPhotCoadd: - class: lsst.drp.tasks.forcedPhotCoadd.ForcedPhotCoaddTask - config: - connections.inputCoaddName: "fakes_deep" - connections.outputCoaddName: "fakes_deep" From da6ed623dbd17a6acff5632b665ccd12210cc088 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Fri, 30 Jun 2023 14:57:46 -0400 Subject: [PATCH 2/3] Update contracts to new approach to MakeWarp connections configuration. --- pipelines/_ingredients/DRP-minimal-calibration.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/_ingredients/DRP-minimal-calibration.yaml b/pipelines/_ingredients/DRP-minimal-calibration.yaml index 2e08940b..2ae7a9d3 100644 --- a/pipelines/_ingredients/DRP-minimal-calibration.yaml +++ b/pipelines/_ingredients/DRP-minimal-calibration.yaml @@ -236,6 +236,6 @@ contracts: - drpAssociation.connections.diaObjectTable == drpDiaCalculation.connections.diaObjectTable - forcedPhotDiffim.connections.refCat == forcedPhotCcd.connections.refCat - consolidateHealSparsePropertyMaps.property_maps == healSparsePropertyMaps.property_maps - - "'calib_psf_candidate' not in measure.propagateFlags.source_flags if makeWarp.doApplyFinalizedPsf else True" - - "'calib_psf_used' not in measure.propagateFlags.source_flags if makeWarp.doApplyFinalizedPsf else True" - - "'calib_psf_reserved' not in measure.propagateFlags.source_flags if makeWarp.doApplyFinalizedPsf else True" + - "'calib_psf_candidate' not in measure.propagateFlags.source_flags if makeWarp.useVisitSummaryPsf else True" + - "'calib_psf_used' not in measure.propagateFlags.source_flags if makeWarp.useVisitSummaryPsf else True" + - "'calib_psf_reserved' not in measure.propagateFlags.source_flags if makeWarp.useVisitSummaryPsf else True" From 39e08a026cd387074ab952fd7aafecf06006700c Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Fri, 30 Jun 2023 14:58:12 -0400 Subject: [PATCH 3/3] Don't set unused and deprecated config template overrides. --- pipelines/_ingredients/DECam/DRP.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pipelines/_ingredients/DECam/DRP.yaml b/pipelines/_ingredients/DECam/DRP.yaml index 714f2355..717fc202 100644 --- a/pipelines/_ingredients/DECam/DRP.yaml +++ b/pipelines/_ingredients/DECam/DRP.yaml @@ -54,7 +54,6 @@ tasks: modelPsf.defaultFwhm: 7.7 warpAndPsfMatch.warp.warpingKernelName: lanczos5 coaddPsf.warpingKernelName: lanczos5 - connections.skyWcsName: jointcal # Legacy DECam config to write out warps, even if they are empty doWriteEmptyWarps: true python: | @@ -112,7 +111,6 @@ tasks: class: lsst.pipe.tasks.postprocess.WriteRecalibratedSourceTableTask config: connections.outputCatalog: source - connections.skyWcsName: jointcal jointcal: class: lsst.jointcal.JointcalTask config: