From f687845f7d4eb3c2e377202fddc73a015e71f330 Mon Sep 17 00:00:00 2001 From: Yusra AlSayyad Date: Fri, 15 Mar 2024 14:38:53 -0700 Subject: [PATCH] Add two pipelines to be run on RA during OR3 - A fast observer-support pipeline matching what the single frame runner in the summit rapid analysis runs now. Working title = Quick Look - A commissiong validation pipeline that will help inform what changes to make the following night. Detector and Visit level tasks can be run in real time by e.g. Rapid Analysis. The rest of the pipeline is expected to be run as part of the 10am processing. Working title = Nightly Validation --- .../LSSTComCamSim/DRP-ops-rehearsal-3.yaml | 43 +------ .../nightly-validation-ops-rehearsal-3.yaml | 84 +++++++++++++ .../quickLook-ops-rehearsal-3.yaml | 70 +++++++++++ pipelines/_ingredients/DRP-full.yaml | 10 ++ pipelines/_ingredients/LSSTComCamSim/DRP.yaml | 116 ++++++++++++++++++ 5 files changed, 282 insertions(+), 41 deletions(-) create mode 100644 pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml create mode 100644 pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml create mode 100644 pipelines/_ingredients/LSSTComCamSim/DRP.yaml diff --git a/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml b/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml index 658c1c3a..4810c84a 100644 --- a/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml +++ b/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml @@ -1,48 +1,8 @@ description: DRP Pipeline for Simulated ComCam Ops Rehearsal 3 instrument: lsst.obs.lsst.LsstComCamSim imports: - - $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/coaddColumnValidate.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityCore.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityExtended.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/diaTractQualityCore.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/matchedVisitQualityCore.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/visitQualityCore.yaml -tasks: - fgcmFitCycle: - class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask - config: - doMultipleCycles: true - multipleCyclesFinalCycleNumber: 4 + - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCamSim/DRP.yaml - fgcmOutputProducts: - class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask - config: - connections.cycleNumber: 4 - - selectGoodSeeingVisits: - class: lsst.pipe.tasks.selectImages.BestSeeingQuantileSelectVisitsTask - config: - connections.goodVisits: goodSeeingVisits - nVisitsMin: 12 - # For simulation 2 there are 60 visits per region - # between 60399.362 and 60402.419. - minMJD: 60399.362 - maxMJD: 60402.419 - updateVisitSummary: - class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask - config: - # TODO: Evaluate GBDES, SkyCorr, FGCM after initial DRP - wcs_provider: "tract" - photo_calib_provider: "global" - background_provider: "input_summary" - analyzeObjectTableCore: - class: lsst.analysis.tools.tasks.ObjectTableTractAnalysisTask - config: - python: | - # no z-band in comCamSim - del config.atools.yPerpPSF - del config.atools.yPerpCModel subsets: step1: subset: @@ -60,6 +20,7 @@ subsets: select partial visits that overlap that region. step2a: subset: + - analyzeMatchedPreVisitCore - consolidatePreSourceTable - consolidateVisitSummary description: | diff --git a/pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml b/pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml new file mode 100644 index 00000000..f06db29d --- /dev/null +++ b/pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml @@ -0,0 +1,84 @@ +description: | + DRP-flavored pipeline to support validation + during commissioning, and will help inform + what changes to make the following night. Detector and Visit + level tasks can be run in real time by e.g. Rapid Analysis. + The rest of the pipeline is expected to be run as part of the + 10am processing. +instrument: lsst.obs.lsst.LsstComCamSim +imports: + - $DRP_PIPE_DIR/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml +tasks: + analyzePreSourceTableCore: + class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask + config: + connections.data: preSourceTable_visit + connections.inputName: preSourceTable_visit + connections.outputName: preSourceTableCore +subsets: + # These 3 steps can be run in real time by Rapid Analysis Framework at USDF + step1: + subset: + - isr + - characterizeImage + - calibrate + - writePreSourceTable + - transformPreSourceTable + description: | + Detector level tasks. TO DO: requests include synthetic source injection, + subtractImages, detectAndMeasureDiaSources. + step2a: + subset: + - consolidatePreSourceTable + - consolidateVisitSummary + - analyzePreSourceTableCore + - catalogMatchPreVisit + - astrometricRefCatPreSourceVisit + # TO DO: Add photometricMatchPreVisit, photometricRefCatPreSourceVisit + description: Visit-level tasks + nightlyRollup: + subset: + - makePreliminaryCcdVisitTable + - makePreliminaryVisitTable + - preliminaryVisitCoverageAnalysis + description: | + Global tasks that can be run at end of night or multiple times during + the night, to get a summary of observations taken. + # The following N steps should be run once at the end of the night + # Steps 2b,2c,2d,2e,4,5,6,7 are currently the same as DRP. + # This step3 here does not make templates. To run step4,step5,step6 you must + # provide a collection with templates as input, ideally the same one that + # was used for prompt processing. + # A complete pipeline without DIA is #step2b,step2c,step2d,step2e,step3,step7 + step3: + subset: + - makeWarp + - selectDeepCoaddVisits + - assembleCoadd + - detection + - mergeDetections + - deblend + - measure + - mergeMeasurements + - forcedPhotCoadd + - transformObjectTable + - writeObjectTable + - consolidateObjectTable + - healSparsePropertyMaps + # analysis_tools tasks + - analyzeMatchedVisitCore + - analyzeObjectTableCore + - analyzeObjectTableExtended + - catalogMatchTract + - photometricCatalogMatch + - photometricRefCatObjectTract + - plotPropertyMapTract + - refCatObjectTract + - validateObjectTableCore + description: | + This overrides the default DRP pipeline by not making nightly templates. + We may want to make daily templates in the future. + + Patch and Tract-level tasks that make coadds and object tables that + can be run together, but only after the 'step1' and 'step2d' + subsets. diff --git a/pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml b/pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml new file mode 100644 index 00000000..9746cdc2 --- /dev/null +++ b/pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml @@ -0,0 +1,70 @@ +description: | + A fast pipeline to be run on the rapid analysis framework at + the summit which feeds RubinTV for realtime observer feedback +instrument: lsst.obs.lsst.LsstComCamSim +imports: + - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCamSim/DRP.yaml +tasks: + isr: + class: lsst.ip.isr.IsrTask + config: + # Turn off slow steps in ISR + doBrighterFatter: false + # TO DO: Replace with calibrateImage in a way that we can use the + # preSources downstream. + characterizeImage: + class: lsst.pipe.tasks.characterizeImage.CharacterizeImageTask + config: + doApCorr: false + doDeblend: false + doMaskStreaks: false + calibrate: + class: lsst.pipe.tasks.calibrate.CalibrateTask + config: + python: | + config.measurement.plugins.names = ['base_CircularApertureFlux', + 'base_PsfFlux', + 'base_NaiveCentroid', + 'base_CompensatedGaussianFlux', + 'base_LocalBackground', + 'base_SdssCentroid', + 'base_SdssShape', + 'base_Variance', + 'base_Jacobian', + 'base_PixelFlags', + 'base_GaussianFlux', + 'base_SkyCoord', + 'base_FPPosition', + 'base_ClassificationSizeExtendedness', + ] + config.measurement.slots.shape = "base_SdssShape" + config.measurement.slots.psfShape = "base_SdssShape_psf" + config.measurement.plugins['base_PixelFlags'].masksFpAnywhere.remove('STREAK') + config.measurement.plugins['base_PixelFlags'].masksFpCenter.remove('STREAK') + config.doApCorr = False + config.doDeblend = False + config.astrometry.sourceSelector["science"].doRequirePrimary = False + config.astrometry.sourceSelector["science"].doIsolated = False + # TO DO: DM-41189 Get pipeline that will attach an atool here from lauren +subsets: + step1: + subset: + - isr + - characterizeImage + - calibrate + # TO DO: configure and add writePreSourceTable, transformPreSourceTable + description: | + Per-detector tasks that can be run together + step2a: + subset: + - consolidateVisitSummary + # TO DO: configure and add consolidatePreSourceTable + nightlyRollup: + # can be run at end of night or on a timer + # Need to either clobber OR use a chained collection of timestamped runs + subset: + - makePreliminaryCcdVisitTable + - makePreliminaryVisitTable + - preliminaryVisitCoverageAnalysis + description: | + Global-level tasks diff --git a/pipelines/_ingredients/DRP-full.yaml b/pipelines/_ingredients/DRP-full.yaml index 7b355d56..c94c9baf 100644 --- a/pipelines/_ingredients/DRP-full.yaml +++ b/pipelines/_ingredients/DRP-full.yaml @@ -51,6 +51,16 @@ tasks: class: lsst.pipe.tasks.multiBand.MeasureMergedCoaddSourcesTask config: connections.sourceTableHandles: preSourceTable_visit + makePreliminaryCcdVisitTable: + class: lsst.pipe.tasks.postprocess.MakeCcdVisitTableTask + config: + connections.visitSummaryRefs: visitSummary + connections.outputCatalog: preCcdVisitTable + makePreliminaryVisitTable: + class: lsst.pipe.tasks.postprocess.MakeVisitTableTask + config: + connections.visitSummaries: visitSummary + connections.outputCatalog: preVisitTable subsets: # This pipeline file only defines subsets that have a consistent or # near-consistent definition as a set of tasks, for use by downstream diff --git a/pipelines/_ingredients/LSSTComCamSim/DRP.yaml b/pipelines/_ingredients/LSSTComCamSim/DRP.yaml new file mode 100644 index 00000000..b575cf15 --- /dev/null +++ b/pipelines/_ingredients/LSSTComCamSim/DRP.yaml @@ -0,0 +1,116 @@ +description: DRP Pipeline for Simulated ComCam Ops Rehearsal 3 +instrument: lsst.obs.lsst.LsstComCamSim +imports: + - $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/coaddColumnValidate.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityExtended.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/diaTractQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/matchedVisitQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/visitQualityCore.yaml +tasks: + fgcmFitCycle: + class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask + config: + doMultipleCycles: true + multipleCyclesFinalCycleNumber: 4 + fgcmOutputProducts: + class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask + config: + connections.cycleNumber: 4 + selectGoodSeeingVisits: + class: lsst.pipe.tasks.selectImages.BestSeeingQuantileSelectVisitsTask + config: + connections.goodVisits: goodSeeingVisits + nVisitsMin: 12 + # For simulation 2 there are 60 visits per region + # between 60399.362 and 60402.419. + minMJD: 60399.362 + maxMJD: 60402.419 + updateVisitSummary: + class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask + config: + # TODO: Evaluate GBDES, SkyCorr, FGCM after initial DRP + wcs_provider: "tract" + photo_calib_provider: "global" + background_provider: "input_summary" + analyzeMatchedPreVisitCore: + class: lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask + config: + connections.sourceCatalogs: preSourceTable_visit + connections.outputName: matchedPreVisitCore + # The rest should be the same as configs in matchedVisitQualityCore.yaml + atools.stellarPhotometricRepeatability: StellarPhotometricRepeatability + atools.stellarPhotometricResiduals: StellarPhotometricResidualsFocalPlane + atools.stellarAstrometricResidualsRA: StellarAstrometricResidualsRAFocalPlanePlot + atools.stellarAstrometricResidualsDec: StellarAstrometricResidualsDecFocalPlanePlot + atools.stellarAstrometricResidualStdDevRA: StellarAstrometricResidualStdDevRAFocalPlanePlot + atools.stellarAstrometricResidualStdDevDec: StellarAstrometricResidualStdDevDecFocalPlanePlot + atools.stellarAstrometricRepeatability1: AstrometricRelativeRepeatability + atools.stellarAstrometricRepeatability1.xValue: 1 + atools.stellarAstrometricRepeatability1.process.calculateActions.rms.annulus: 5 + atools.stellarAstrometricRepeatability2: AstrometricRelativeRepeatability + atools.stellarAstrometricRepeatability2.xValue: 2 + atools.stellarAstrometricRepeatability2.process.calculateActions.rms.annulus: 20 + atools.stellarAstrometricRepeatability3: AstrometricRelativeRepeatability + atools.stellarAstrometricRepeatability3.xValue: 3 + atools.stellarAstrometricRepeatability3.process.calculateActions.rms.annulus: 200 + atools.stellarAstrometricRepeatability3.process.calculateActions.rms.threshAD: 30 + python: | + from lsst.analysis.tools.atools import * + analyzeObjectTableCore: + class: lsst.analysis.tools.tasks.ObjectTableTractAnalysisTask + config: + python: | + # no z-band in comCamSim + del config.atools.yPerpPSF + del config.atools.yPerpCModel + analyzePreSourceTableCore: + class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask + config: + connections.outputName: preSourceTableCore + atools.skyFluxVisitStatisticMetric: SkyFluxStatisticMetric + atools.skyFluxVisitStatisticMetric.applyContext: VisitContext + atools.skySourceSky: SkySourceSkyPlot + atools.skySourceFlux: SkySourceHistPlot + python: | + from lsst.analysis.tools.atools import * + from lsst.analysis.tools.contexts import * + catalogMatchPreVisit: + class: lsst.analysis.tools.tasks.astrometricCatalogMatch.AstrometricCatalogMatchVisitTask + config: + connections.catalog: "preSourceTable_visit" + connections.targetCatalog: "preSourceTable_visit" + astrometricRefCatPreSourceVisit: + class: lsst.analysis.tools.tasks.refCatSourceAnalysis.RefCatSourceAnalysisTask + config: + # Only run metrics for analyzing the preSources: + connections.data: preSourceTable_visit_uw_stars_20240228_match + atools.astromDiffMetrics: TargetRefCatDeltaMetrics + atools.astromDiffMetrics.applyContext: VisitContext + python: | + from lsst.analysis.tools.atools import * + from lsst.analysis.tools.contexts import * + # These don't actually work now: + photometricMatchPreVisit: + class: lsst.analysis.tools.tasks.photometricCatalogMatch.PhotometricCatalogMatchVisitTask + config: + extraColumns: ["x", "y", "ap09Flux", "ap09FluxErr"] + connections.refCatalog: uw_stars_20240228 + connections.catalog: preSourceTable_visit + connections.targetCatalog: preSourceTable_visit + connections.matchedCatalog: preSourceTable_visit_uw_stars_20240228_photoMatch + photometricRefCatPreSourceVisit: + class: lsst.analysis.tools.tasks.refCatSourcePhotometricAnalysis.RefCatSourcePhotometricAnalysisTask + config: + # Only run metrics for analyzing the preSources: + connections.data: preSourceTable_visit_uw_stars_20240228_photoMatch + python: from lsst.analysis.tools.atools import * + preliminaryVisitCoverageAnalysis: + class: lsst.analysis.tools.tasks.CcdVisitTableAnalysisTask + config: + connections.data: preCcdVisitTable + atools.raDecCoveragePlot: RaDecMultiVisitCoveragePlot + atools.raDecCoveragePlot.produce.plot.showExtremeOutliers: false + atools.focalPlaneCoveragePlot: FocalPlaneMultiVisitCoveragePlot + python: from lsst.analysis.tools.atools import *