From 96f2f5e51879fdc988145ef81571df902e9f4003 Mon Sep 17 00:00:00 2001 From: Yusra AlSayyad Date: Wed, 9 Oct 2024 08:51:24 -0700 Subject: [PATCH] Rename outputs of isolatedStarAssociation After the great calibration refactor, PreSources will always be generated in single frame processing and have fewer rows than Sources based on a second round of more complete detection: - isolated_star_cat -> isolated_star_presource_associations - isolated_star_sources -> isolated_star_presources --- pipelines/HSC/DRP-Prod.yaml | 2 ++ pipelines/HSC/DRP-RC2.yaml | 5 +---- pipelines/HSC/DRP-RC2_subset.yaml | 1 + pipelines/HSC/DRP-ci_hsc.yaml | 4 ---- pipelines/LATISS/DRP.yaml | 4 ++++ pipelines/LSSTComCam/DRP.yaml | 3 ++- pipelines/LSSTComCam/nightly-validation.yaml | 1 + pipelines/LSSTComCamSim/DRP.yaml | 1 + .../LSSTComCamSim/nightly-validation.yaml | 1 + pipelines/_ingredients/DECam/DRP.yaml | 4 +++- pipelines/_ingredients/DRP-full.yaml | 19 +++++++++++++++++++ .../_ingredients/DRP-minimal-calibration.yaml | 19 +++++++++++++++++-- pipelines/_ingredients/HSC/DRP.yaml | 4 ---- tests/test_pipelines.py | 15 +++++++++++++-- 14 files changed, 65 insertions(+), 18 deletions(-) diff --git a/pipelines/HSC/DRP-Prod.yaml b/pipelines/HSC/DRP-Prod.yaml index 741fdc49..d9d95522 100644 --- a/pipelines/HSC/DRP-Prod.yaml +++ b/pipelines/HSC/DRP-Prod.yaml @@ -46,6 +46,7 @@ subsets: step2b: subset: - isolatedStarAssociation + - analyzeMatchedPreVisitCore - gbdesAstrometricFit description: | Tract-level tasks @@ -119,6 +120,7 @@ subsets: - catalogMatchTract - refCatObjectTract - validateObjectTableCore + - isolatedStarSourceAssociation - analyzeMatchedVisitCore - photometricCatalogMatch - photometricRefCatObjectTract diff --git a/pipelines/HSC/DRP-RC2.yaml b/pipelines/HSC/DRP-RC2.yaml index ab0546db..8b78cc43 100644 --- a/pipelines/HSC/DRP-RC2.yaml +++ b/pipelines/HSC/DRP-RC2.yaml @@ -65,10 +65,6 @@ tasks: class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask config: connections.cycleNumber: 4 - isolatedStarAssociation: - class: lsst.pipe.tasks.isolatedStarAssociation.IsolatedStarAssociationTask - config: - band_order: ['i', 'r', 'g', 'z', 'y', 'N921'] matchedVisitCoreWholeSkyPlot: class: lsst.analysis.tools.tasks.WholeSkyAnalysisTask config: @@ -179,6 +175,7 @@ subsets: # per-tract Tasks - consolidateObjectTable - analyzeMatchedVisitCore + - isolatedStarSourceAssociation - analyzeMatchedPreVisitCore - analyzeObjectTableCore - catalogMatchTract diff --git a/pipelines/HSC/DRP-RC2_subset.yaml b/pipelines/HSC/DRP-RC2_subset.yaml index 96384db2..1cae120b 100644 --- a/pipelines/HSC/DRP-RC2_subset.yaml +++ b/pipelines/HSC/DRP-RC2_subset.yaml @@ -380,6 +380,7 @@ subsets: - psfPhotRepStar3 - psfPhotRepStar4 - objectEpochTable + - isolatedStarSourceAssociation description: | Tasks that can be run together, but only after the 'step1' and 'step2' subsets. diff --git a/pipelines/HSC/DRP-ci_hsc.yaml b/pipelines/HSC/DRP-ci_hsc.yaml index a70e7813..d7114bd9 100644 --- a/pipelines/HSC/DRP-ci_hsc.yaml +++ b/pipelines/HSC/DRP-ci_hsc.yaml @@ -46,10 +46,6 @@ tasks: class: lsst.pipe.tasks.postprocess.WriteRecalibratedSourceTableTask config: connections.outputCatalog: source - isolatedStarAssociation: - class: lsst.pipe.tasks.isolatedStarAssociation.IsolatedStarAssociationTask - config: - band_order: ['i', 'r'] matchedVisitCoreWholeSkyPlot: class: lsst.analysis.tools.tasks.WholeSkyAnalysisTask config: diff --git a/pipelines/LATISS/DRP.yaml b/pipelines/LATISS/DRP.yaml index d259ee52..7991f0fa 100644 --- a/pipelines/LATISS/DRP.yaml +++ b/pipelines/LATISS/DRP.yaml @@ -84,6 +84,9 @@ tasks: class: lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask config: connections.outputName: matchedVisitCore + connections.sourceCatalogs: sourceTable_visit + connections.associatedSources: isolated_star_sources + connections.associatedSourcesInputName: isolated_star_sources atools.stellarPhotometricRepeatability: StellarPhotometricRepeatability atools.stellarPhotometricResiduals: StellarPhotometricResidualsFocalPlane atools.stellarPhotometricResiduals.produce.plot.nBins: 80 @@ -213,6 +216,7 @@ subsets: - photometricCatalogMatch - photometricRefCatObjectTract - validateObjectTableCore + - isolatedStarSourceAssociation - analyzeMatchedVisitCore - analyzeAmpOffsetMetadata # The following are from analysis_drp diff --git a/pipelines/LSSTComCam/DRP.yaml b/pipelines/LSSTComCam/DRP.yaml index 4b7de193..cb0ed07d 100644 --- a/pipelines/LSSTComCam/DRP.yaml +++ b/pipelines/LSSTComCam/DRP.yaml @@ -24,7 +24,6 @@ subsets: select partial visits that overlap that region. step2a: subset: - - analyzeMatchedPreVisitCore - consolidatePreSourceTable - consolidateVisitSummary description: | @@ -37,6 +36,7 @@ subsets: TODO: Evaluate SkyCorr after intial DRP step2b: subset: + - analyzeMatchedPreVisitCore - isolatedStarAssociation - gbdesAstrometricFit description: | @@ -94,6 +94,7 @@ subsets: - selectGoodSeeingVisits - templateGen # analysis_tools tasks + - isolatedStarSourceAssociation - analyzeMatchedVisitCore - analyzeObjectTableCore - catalogMatchTract diff --git a/pipelines/LSSTComCam/nightly-validation.yaml b/pipelines/LSSTComCam/nightly-validation.yaml index fe5c9b98..960a3b8f 100644 --- a/pipelines/LSSTComCam/nightly-validation.yaml +++ b/pipelines/LSSTComCam/nightly-validation.yaml @@ -119,6 +119,7 @@ subsets: - consolidateObjectTable - healSparsePropertyMaps # analysis_tools tasks + - isolatedStarSourceAssociation - analyzeMatchedVisitCore - analyzeObjectTableCore - analyzeObjectTableExtended diff --git a/pipelines/LSSTComCamSim/DRP.yaml b/pipelines/LSSTComCamSim/DRP.yaml index 0ee4f1fe..7f7a0596 100644 --- a/pipelines/LSSTComCamSim/DRP.yaml +++ b/pipelines/LSSTComCamSim/DRP.yaml @@ -96,6 +96,7 @@ subsets: - templateGen # analysis_tools tasks - analyzeAmpOffsetMetadata + - isolatedStarSourceAssociation - analyzeMatchedVisitCore - analyzeObjectTableCore - catalogMatchTract diff --git a/pipelines/LSSTComCamSim/nightly-validation.yaml b/pipelines/LSSTComCamSim/nightly-validation.yaml index 86841da7..a6109cb4 100644 --- a/pipelines/LSSTComCamSim/nightly-validation.yaml +++ b/pipelines/LSSTComCamSim/nightly-validation.yaml @@ -116,6 +116,7 @@ subsets: - healSparsePropertyMaps # analysis_tools tasks - analyzeAmpOffsetMetadata + - isolatedStarSourceAssociation - analyzeMatchedVisitCore - analyzeObjectTableCore - analyzeObjectTableExtended diff --git a/pipelines/_ingredients/DECam/DRP.yaml b/pipelines/_ingredients/DECam/DRP.yaml index 6a7bd98b..dbcf050c 100644 --- a/pipelines/_ingredients/DECam/DRP.yaml +++ b/pipelines/_ingredients/DECam/DRP.yaml @@ -105,7 +105,9 @@ tasks: isolatedStarAssociation: class: lsst.pipe.tasks.isolatedStarAssociation.IsolatedStarAssociationTask config: - connections.source_table_visit: preSourceTable_visit + # connections.source_table_visit: preSourceTable_visit + # connections.isolated_star_sources: isolated_star_presources + # connections.isolated_star_cat: isolated_star_presource_associations python: config.band_order += ["N708", "N540"] writeRecalibratedSourceTable: diff --git a/pipelines/_ingredients/DRP-full.yaml b/pipelines/_ingredients/DRP-full.yaml index 17180ebe..043aeeeb 100644 --- a/pipelines/_ingredients/DRP-full.yaml +++ b/pipelines/_ingredients/DRP-full.yaml @@ -19,13 +19,32 @@ tasks: skyCorr: class: lsst.pipe.tasks.skyCorrection.SkyCorrectionTask isolatedStarAssociation: + # for global calibration and PSF estimates on preSources class: lsst.pipe.tasks.isolatedStarAssociation.IsolatedStarAssociationTask config: + # TODO DM-43077: delete these three overrides connections.source_table_visit: preSourceTable_visit + connections.isolated_star_sources: isolated_star_presources + connections.isolated_star_cat: isolated_star_presource_associations + isolatedStarSourceAssociation: + # for final repeatability metrics on final source table + class: lsst.pipe.tasks.isolatedStarAssociation.IsolatedStarAssociationTask + config: + connections.source_table_visit: sourceTable_visit + connections.isolated_star_sources: isolated_star_sources + connections.isolated_star_cat: isolated_star_source_associations + finalizeCharacterization: + class: lsst.pipe.tasks.finalizeCharacterization.FinalizeCharacterizationTask + config: + connections.isolated_star_cats: isolated_star_presource_associations + connections.isolated_star_sources: isolated_star_presources gbdesAstrometricFit: class: lsst.drp.tasks.gbdesAstrometricFit.GbdesAstrometricFitTask fgcmBuildFromIsolatedStars: class: lsst.fgcmcal.fgcmBuildFromIsolatedStars.FgcmBuildFromIsolatedStarsTask + config: + connections.isolated_star_cats: isolated_star_presources + connections.isolated_star_sources: isolated_star_presource_associations fgcmFitCycle: class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask fgcmOutputProducts: diff --git a/pipelines/_ingredients/DRP-minimal-calibration.yaml b/pipelines/_ingredients/DRP-minimal-calibration.yaml index 31c78d6e..b59559ca 100644 --- a/pipelines/_ingredients/DRP-minimal-calibration.yaml +++ b/pipelines/_ingredients/DRP-minimal-calibration.yaml @@ -13,8 +13,23 @@ tasks: transformSourceTable: lsst.pipe.tasks.postprocess.TransformSourceTableTask consolidateSourceTable: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask consolidateVisitSummary: lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask - isolatedStarAssociation: lsst.pipe.tasks.isolatedStarAssociation.IsolatedStarAssociationTask - finalizeCharacterization: lsst.pipe.tasks.finalizeCharacterization.FinalizeCharacterizationTask + isolatedStarAssociation: + class: lsst.pipe.tasks.isolatedStarAssociation.IsolatedStarAssociationTask + config: + # TODO DM-43077: Remove all three connections overrides. + # DM-43077 introduces presources to the imSim pipelines + # presource are the default catalog to match on + connections.source_table_visit: sourceTable_visit + connections.isolated_star_sources: isolated_star_sources + connections.isolated_star_cat: isolated_star_source_associations + finalizeCharacterization: + class: lsst.pipe.tasks.finalizeCharacterization.FinalizeCharacterizationTask + config: + # TODO DM-43077: Remove both connections overrides. + # DM-43077 introduces presources to the imSim pipelines + # presources are the default catalog to match on + connections.isolated_star_cats: isolated_star_source_associations + connections.isolated_star_sources: isolated_star_sources updateVisitSummary: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask makeCcdVisitTable: lsst.pipe.tasks.postprocess.MakeCcdVisitTableTask makeVisitTable: lsst.pipe.tasks.postprocess.MakeVisitTableTask diff --git a/pipelines/_ingredients/HSC/DRP.yaml b/pipelines/_ingredients/HSC/DRP.yaml index 4dd4623c..0c9591e2 100644 --- a/pipelines/_ingredients/HSC/DRP.yaml +++ b/pipelines/_ingredients/HSC/DRP.yaml @@ -17,10 +17,6 @@ tasks: config: doAmpOffset: true ampOffset.doApplyAmpOffset: true - isolatedStarAssociation: - class: lsst.pipe.tasks.isolatedStarAssociation.IsolatedStarAssociationTask - config: - band_order: ["i", "r", "z", "y", "g", "N921", "N816", "N1010", "N387", "N515"] fgcmFitCycle: class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask config: diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index b49db427..8f12b11e 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -108,8 +108,6 @@ "icExp", "icExpBackground", "icSrc", - "isolated_star_cat", - "isolated_star_sources", "objectTable", "objectTable_tract", "postISRCCD", @@ -134,6 +132,11 @@ "preSourceTable_visit", "skyCorr", "srcMatchFull", + # DM-43077 move 2 presource associations and friends to COMMON_OUTPUTS + "isolated_star_presource_associations", + "isolated_star_presources", + "isolated_star_source_associations", + "isolated_star_sources", } # LATISS common outputs, in addition to COMMON_OUTPUTS @@ -176,6 +179,11 @@ "mergedForcedSource", "mergedForcedSourceOnDiaObject", "transmission_atmosphere_fgcm", + # DM-43077 move 2 presource associations and friends to COMMON_OUTPUTS + "isolated_star_presource_associations", + "isolated_star_presources", + "isolated_star_source_associations", + "isolated_star_sources", } # LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS @@ -207,6 +215,9 @@ "goodSeeingVisits", "mergedForcedSource", "mergedForcedSourceOnDiaObject", + # TODO DM-43077: remove these two + "isolated_star_source_associations", + "isolated_star_sources", }