From 849b213b7b3476615cc22e816daa0e7f0eef51c4 Mon Sep 17 00:00:00 2001 From: Yusra AlSayyad Date: Fri, 15 Mar 2024 14:38:53 -0700 Subject: [PATCH] Add two pipelines to be run on RA during OR3 - A fast observer-support pipeline matching what the single frame runner in the summit rapid analysis runs now - A commissiong validation pipeline that will help inform what changes to make the following night. Detector and Visit level tasks can be run in real time by e.g. Rapid Analysis. The rest of the pipeline is expected to be run as part of the 10am processing. --- .../LSSTComCamSim/DRP-ops-rehearsal-3.yaml | 42 +--- .../nightly-validation-ops-rehearsal-3.yaml | 191 ++++++++++++++++++ .../quickLook-ops-rehearsal-3.yaml | 70 +++++++ pipelines/_ingredients/DRP-full.yaml | 10 + pipelines/_ingredients/LSSTComCamSim/DRP.yaml | 117 +++++++++++ 5 files changed, 389 insertions(+), 41 deletions(-) create mode 100644 pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml create mode 100644 pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml create mode 100644 pipelines/_ingredients/LSSTComCamSim/DRP.yaml diff --git a/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml b/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml index 658c1c3a..b9578108 100644 --- a/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml +++ b/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml @@ -1,48 +1,8 @@ description: DRP Pipeline for Simulated ComCam Ops Rehearsal 3 instrument: lsst.obs.lsst.LsstComCamSim imports: - - $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/coaddColumnValidate.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityCore.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityExtended.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/diaTractQualityCore.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/matchedVisitQualityCore.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/visitQualityCore.yaml -tasks: - fgcmFitCycle: - class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask - config: - doMultipleCycles: true - multipleCyclesFinalCycleNumber: 4 + - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCamSim/DRP.yaml - fgcmOutputProducts: - class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask - config: - connections.cycleNumber: 4 - - selectGoodSeeingVisits: - class: lsst.pipe.tasks.selectImages.BestSeeingQuantileSelectVisitsTask - config: - connections.goodVisits: goodSeeingVisits - nVisitsMin: 12 - # For simulation 2 there are 60 visits per region - # between 60399.362 and 60402.419. - minMJD: 60399.362 - maxMJD: 60402.419 - updateVisitSummary: - class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask - config: - # TODO: Evaluate GBDES, SkyCorr, FGCM after initial DRP - wcs_provider: "tract" - photo_calib_provider: "global" - background_provider: "input_summary" - analyzeObjectTableCore: - class: lsst.analysis.tools.tasks.ObjectTableTractAnalysisTask - config: - python: | - # no z-band in comCamSim - del config.atools.yPerpPSF - del config.atools.yPerpCModel subsets: step1: subset: diff --git a/pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml b/pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml new file mode 100644 index 00000000..7bdfa09b --- /dev/null +++ b/pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml @@ -0,0 +1,191 @@ +description: | + DRP-flavored pipeline to support validation + commissiong validation pipeline that will help inform + what changes to make the following night. Detector and Visit + level tasks can be run in real time by e.g. Rapid Analysis. + The rest of the pipeline is expected to be run as part of the + 10am processing. +instrument: lsst.obs.lsst.LsstComCamSim +imports: + - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCamSim/DRP.yaml +tasks: + analyzePreSourceTableCore: + class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask + config: + connections.data: preSourceTable_visit + connections.inputName: preSourceTable_visit + connections.outputName: preSourceTableCore +subsets: + # These 3 steps can be run in real time by Rapid Analysis Framework at USDF + step1: + subset: + - isr + - characterizeImage + - calibrate + - writePreSourceTable + - transformPreSourceTable + description: | + Detector level tasks. TO DO: requests include synthetic source injection, + subtractImages, detectAndMeasureDiaSources. + step2a: + subset: + - consolidatePreSourceTable + - consolidateVisitSummary + - analyzePreSourceTableCore + - catalogMatchPreVisit + - astrometricRefCatPreSourceVisit # configure for metrics only + # TO DO: Add photometricMatchPreVisit, photometricRefCatPreSourceVisit + description: Visit-level tasks + nightlyRollup: + subset: + - makePreliminaryCcdVisitTable + - makePreliminaryVisitTable + - preliminaryVisitCoverageAnalysis + description: | + Global tasks that can be run at end of night or multiple times during + the night, to get a summary of observations taken. + # The following N steps should be run once at the end of the night + step2b: + subset: + - analyzeMatchedPreVisitCore + - isolatedStarAssociation + - gbdesAstrometricFit + description: | + Tract-level tasks + Allowed data query constraints: tract + + GbdesAstrometricFit and isolatedStarAssociation both use PreSources, + generated by consolidatePreSourceTable, for all visits that overlap a + tract. + gbdesAstrometricFit produces solutions per-tract, per-visit + isolatedStarAssociation produces solutions per-tract. + step2c: + subset: + - fgcmBuildFromIsolatedStars + - fgcmFitCycle + - fgcmOutputProducts + description: | + Global-level tasks that must not be run with any data query constraints + Can be run anytime after subset step2b. + Allowed data query constraints: instrument + step2d: + subset: + - finalizeCharacterization + - writeRecalibratedSourceTable + - transformSourceTable + - consolidateSourceTable + - updateVisitSummary + description: | + Visit-level tasks to re-estimate PSFs and apply + global calibrations to Source tables. + Allowed data query constraints: visit + step2e: + subset: + - makeCcdVisitTable + - makeVisitTable + description: | + Global-level tasks that must not be run with any data query constraints + Can be run anytime after subset step2d. + + Allowed data query constraints: instrument + + Tasks generate one data product per collection. + make[Ccd]VisitTable produces per-collection summary of the Visits + and CcdVisits. + step3: + subset: + - makeWarp + - selectDeepCoaddVisits + - assembleCoadd + - detection + - mergeDetections + - deblend + - measure + - mergeMeasurements + - forcedPhotCoadd + - transformObjectTable + - writeObjectTable + - consolidateObjectTable + - healSparsePropertyMaps + - selectGoodSeeingVisits + - templateGen + # analysis_tools tasks + - analyzeMatchedVisitCore + - analyzeObjectTableCore + - analyzeObjectTableExtended + - catalogMatchTract + - photometricCatalogMatch + - photometricRefCatObjectTract + - plotPropertyMapTract + - refCatObjectTract + - validateObjectTableCore + description: | + Patch and Tract-level tasks that make coadds and object tables that + can be run together, but only after the 'step1' and 'step2d' + subsets. + + These should be run with explicit 'tract' constraints essentially all the + time, because otherwise quanta will be created for jobs with only partial + visit coverage. + step4: + subset: + - forcedPhotCcd + - forcedPhotDiffim + - getTemplate + - subtractImages + - detectAndMeasureDiaSources + - transformDiaSourceCat + - writeForcedSourceTable + description: | + Image subtraction and forced photometry Tasks that can be run together, + but only after the 'step1', 'step2d' and 'step3' subsets + + These detector-level tasks should not be run with 'tract' or 'patch' as + part of the data ID expression if all reference catalogs or diffIm + templates that cover these detector-level quanta are desired. + step5: + subset: + - drpAssociation + - drpDiaCalculation + - forcedPhotCcdOnDiaObjects + - forcedPhotDiffOnDiaObjects + - transformForcedSourceTable + - consolidateAssocDiaSourceTable + - consolidateFullDiaObjectTable + - writeForcedSourceOnDiaObjectTable + - transformForcedSourceOnDiaObjectTable + - analyzeDiaSourceTableTract + description: | + ForcedSource and DIASource association Tasks that can be run together, + but only after the 'step1', 'step2', 'step3', and 'step4' subsets + + This step includes patch-level aggregation Tasks. These should be run + with explicit 'tract' constraints in the data query, otherwise quanta + will be created for jobs with only partial visit coverage. + step6: + subset: + - consolidateDiaSourceTable + description: | + Tasks that can be run together, but only after the 'step1', 'step2', + 'step3', and 'step4' subsets + + This step includes visit-level aggregation tasks. Running without tract + or patch in the data query is recommended, otherwise the outputs of + consolidateDiaSourceTable will not contain complete visits. + + This subset is separate from step4 to signal to operators to pause to + assess unexpected image differencing failures before these aggregation + steps. Otherwise, if run in the same quantum graph, aggregated data + products (e.g. diaObjects) would not be created if one or more of the + expected inputs is missing. + step7: + subset: + - consolidateHealSparsePropertyMaps + - analyzeObjectTableSurveyCore + description: | + Tasks that should be run as the final step that require global inputs, + and can be run after the 'step3' subset. + + This step has global aggregation tasks to run over all visits, detectors, + tracts, etc. This step should be run only with the instrument constraint + in the data query. diff --git a/pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml b/pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml new file mode 100644 index 00000000..d050ae7c --- /dev/null +++ b/pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml @@ -0,0 +1,70 @@ +description: | + A fast pipeline to be run on the rapid analysis framework at + the summit which feeds RubinTV for realtime observer feedback +instrument: lsst.obs.lsst.LsstComCamSim +imports: + - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCamSim/DRP.yaml +tasks: + isr: + class: lsst.ip.isr.IsrTask + config: + # Turn off all slow steps in ISR for + doBrighterFatter: false + # TO DO: Replace with calibrateImage in a way that we can use the + # preSources downstream. + characterizeImage: + class: lsst.pipe.tasks.characterizeImage.CharacterizeImageTask + config: + doApCorr: false + doDeblend: false + doMaskStreaks: false + calibrate: + class: lsst.pipe.tasks.calibrate.CalibrateTask + config: + python: | + config.measurement.plugins.names = ['base_CircularApertureFlux', + 'base_PsfFlux', + 'base_NaiveCentroid', + 'base_CompensatedGaussianFlux', + 'base_LocalBackground', + 'base_SdssCentroid', + 'base_SdssShape', + 'base_Variance', + 'base_Jacobian', + 'base_PixelFlags', + 'base_GaussianFlux', + 'base_SkyCoord', + 'base_FPPosition', + 'base_ClassificationSizeExtendedness', + ] + config.measurement.slots.shape = "base_SdssShape" + config.measurement.slots.psfShape = "base_SdssShape_psf" + config.measurement.plugins['base_PixelFlags'].masksFpAnywhere.remove('STREAK') + config.measurement.plugins['base_PixelFlags'].masksFpCenter.remove('STREAK') + config.doApCorr = False + config.doDeblend = False + config.astrometry.sourceSelector["science"].doRequirePrimary = False + config.astrometry.sourceSelector["science"].doIsolated = False + # TO DO: DM-41189 Get pipeline that will attach an atool here from lauren +subsets: + step1: + subset: + - isr + - characterizeImage + - calibrate + # TO DO: configure and add writePreSourceTable, transformPreSourceTable + description: | + Per-detector tasks that can be run together + step2a: + subset: + - consolidateVisitSummary + # TO DO: configure and add consolidatePreSourceTable + nightlyRollup: + # can be run at end of night or on a timer + # Need to either clobber OR use a chained collection of timestamped runs + subset: + - makePreliminaryCcdVisitTable + - makePreliminaryVisitTable + - preliminaryVisitCoverageAnalysis + description: | + Global-level tasks diff --git a/pipelines/_ingredients/DRP-full.yaml b/pipelines/_ingredients/DRP-full.yaml index 7b355d56..c94c9baf 100644 --- a/pipelines/_ingredients/DRP-full.yaml +++ b/pipelines/_ingredients/DRP-full.yaml @@ -51,6 +51,16 @@ tasks: class: lsst.pipe.tasks.multiBand.MeasureMergedCoaddSourcesTask config: connections.sourceTableHandles: preSourceTable_visit + makePreliminaryCcdVisitTable: + class: lsst.pipe.tasks.postprocess.MakeCcdVisitTableTask + config: + connections.visitSummaryRefs: visitSummary + connections.outputCatalog: preCcdVisitTable + makePreliminaryVisitTable: + class: lsst.pipe.tasks.postprocess.MakeVisitTableTask + config: + connections.visitSummaries: visitSummary + connections.outputCatalog: preVisitTable subsets: # This pipeline file only defines subsets that have a consistent or # near-consistent definition as a set of tasks, for use by downstream diff --git a/pipelines/_ingredients/LSSTComCamSim/DRP.yaml b/pipelines/_ingredients/LSSTComCamSim/DRP.yaml new file mode 100644 index 00000000..b65bce29 --- /dev/null +++ b/pipelines/_ingredients/LSSTComCamSim/DRP.yaml @@ -0,0 +1,117 @@ +description: DRP Pipeline for Simulated ComCam Ops Rehearsal 3 +instrument: lsst.obs.lsst.LsstComCamSim +imports: + - $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/coaddColumnValidate.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityExtended.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/diaTractQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/matchedVisitQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/visitQualityCore.yaml +tasks: + fgcmFitCycle: + class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask + config: + doMultipleCycles: true + multipleCyclesFinalCycleNumber: 4 + fgcmOutputProducts: + class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask + config: + connections.cycleNumber: 4 + selectGoodSeeingVisits: + class: lsst.pipe.tasks.selectImages.BestSeeingQuantileSelectVisitsTask + config: + connections.goodVisits: goodSeeingVisits + nVisitsMin: 12 + # For simulation 2 there are 60 visits per region + # between 60399.362 and 60402.419. + minMJD: 60399.362 + maxMJD: 60402.419 + updateVisitSummary: + class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask + config: + # TODO: Evaluate GBDES, SkyCorr, FGCM after initial DRP + wcs_provider: "tract" + photo_calib_provider: "global" + background_provider: "input_summary" + analyzeMatchedPreVisitCore: + class: lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask + config: + connections.sourceCatalogs: preSourceTable_visit + connections.outputName: matchedPreVisitCore + # The rest should be the same as configs in matchedVisitQualityCore.yaml + atools.stellarPhotometricRepeatability: StellarPhotometricRepeatability + atools.stellarPhotometricResiduals: StellarPhotometricResidualsFocalPlane + atools.stellarAstrometricResidualsRA: StellarAstrometricResidualsRAFocalPlanePlot + atools.stellarAstrometricResidualsDec: StellarAstrometricResidualsDecFocalPlanePlot + atools.stellarAstrometricResidualStdDevRA: StellarAstrometricResidualStdDevRAFocalPlanePlot + atools.stellarAstrometricResidualStdDevDec: StellarAstrometricResidualStdDevDecFocalPlanePlot + atools.stellarAstrometricRepeatability1: AstrometricRelativeRepeatability + atools.stellarAstrometricRepeatability1.xValue: 1 + atools.stellarAstrometricRepeatability1.process.calculateActions.rms.annulus: 5 + atools.stellarAstrometricRepeatability2: AstrometricRelativeRepeatability + atools.stellarAstrometricRepeatability2.xValue: 2 + atools.stellarAstrometricRepeatability2.process.calculateActions.rms.annulus: 20 + atools.stellarAstrometricRepeatability3: AstrometricRelativeRepeatability + atools.stellarAstrometricRepeatability3.xValue: 3 + atools.stellarAstrometricRepeatability3.process.calculateActions.rms.annulus: 200 + atools.stellarAstrometricRepeatability3.process.calculateActions.rms.threshAD: 30 + python: | + from lsst.analysis.tools.atools import * + analyzeObjectTableCore: + class: lsst.analysis.tools.tasks.ObjectTableTractAnalysisTask + config: + python: | + # no z-band in comCamSim + del config.atools.yPerpPSF + del config.atools.yPerpCModel + analyzePreSourceTableCore: + class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask + config: + connections.outputName: preSourceTableCore + atools.skyFluxVisitStatisticMetric: SkyFluxStatisticMetric + atools.skyFluxVisitStatisticMetric.applyContext: VisitContext + atools.skySourceSky: SkySourceSkyPlot + atools.skySourceFlux: SkySourceHistPlot + python: | + from lsst.analysis.tools.atools import * + from lsst.analysis.tools.contexts import * + catalogMatchPreVisit: + class: lsst.analysis.tools.tasks.astrometricCatalogMatch.AstrometricCatalogMatchVisitTask + config: + connections.catalog: "preSourceTable_visit" + connections.targetCatalog: "preSourceTable_visit" + astrometricRefCatPreSourceVisit: + class: lsst.analysis.tools.tasks.refCatSourceAnalysis.RefCatSourceAnalysisTask + config: + # Only run metrics for analyzing the preSources: + connections.data: preSourceTable_visit_uw_stars_20240228_match + atools.astromDiffMetrics: TargetRefCatDeltaMetrics + atools.astromDiffMetrics.applyContext: VisitContext + python: | + from lsst.analysis.tools.atools import * + from lsst.analysis.tools.contexts import * + # These don't actually work now: + photometricMatchPreVisit: + class: lsst.analysis.tools.tasks.photometricCatalogMatch.PhotometricCatalogMatchVisitTask + config: + extraColumns: ["x", "y", "ap09Flux", "ap09FluxErr"] + connections.refCatalog: uw_stars_20240228 + connections.catalog: preSourceTable_visit + connections.targetCatalog: preSourceTable_visit + connections.matchedCatalog: preSourceTable_visit_uw_stars_20240228_photoMatch + photometricRefCatPreSourceVisit: + class: lsst.analysis.tools.tasks.refCatSourcePhotometricAnalysis.RefCatSourcePhotometricAnalysisTask + config: + # Only run metrics for analyzing the preSources: + connections.data: preSourceTable_visit_uw_stars_20240228_photoMatch + python: from lsst.analysis.tools.atools import * + preliminaryVisitCoverageAnalysis: + class: lsst.analysis.tools.tasks.CcdVisitTableAnalysisTask + config: + connections.data: preCcdVisitTable + atools.raDecCoveragePlot: RaDecMultiVisitCoveragePlot + atools.raDecCoveragePlot.produce.plot.parametersToPlotList: ["psfSigma", "astromOffsetMean", "medianE"] + atools.raDecCoveragePlot.produce.plot.plotDetectorOutline: true + atools.focalPlaneCoveragePlot: FocalPlaneMultiVisitCoveragePlot + python: from lsst.analysis.tools.atools import *