diff --git a/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml b/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml index 658c1c3a..b9578108 100644 --- a/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml +++ b/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml @@ -1,48 +1,8 @@ description: DRP Pipeline for Simulated ComCam Ops Rehearsal 3 instrument: lsst.obs.lsst.LsstComCamSim imports: - - $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/coaddColumnValidate.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityCore.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityExtended.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/diaTractQualityCore.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/matchedVisitQualityCore.yaml - - $ANALYSIS_TOOLS_DIR/pipelines/visitQualityCore.yaml -tasks: - fgcmFitCycle: - class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask - config: - doMultipleCycles: true - multipleCyclesFinalCycleNumber: 4 + - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCamSim/DRP.yaml - fgcmOutputProducts: - class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask - config: - connections.cycleNumber: 4 - - selectGoodSeeingVisits: - class: lsst.pipe.tasks.selectImages.BestSeeingQuantileSelectVisitsTask - config: - connections.goodVisits: goodSeeingVisits - nVisitsMin: 12 - # For simulation 2 there are 60 visits per region - # between 60399.362 and 60402.419. - minMJD: 60399.362 - maxMJD: 60402.419 - updateVisitSummary: - class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask - config: - # TODO: Evaluate GBDES, SkyCorr, FGCM after initial DRP - wcs_provider: "tract" - photo_calib_provider: "global" - background_provider: "input_summary" - analyzeObjectTableCore: - class: lsst.analysis.tools.tasks.ObjectTableTractAnalysisTask - config: - python: | - # no z-band in comCamSim - del config.atools.yPerpPSF - del config.atools.yPerpCModel subsets: step1: subset: diff --git a/pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml b/pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml new file mode 100644 index 00000000..7bdfa09b --- /dev/null +++ b/pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml @@ -0,0 +1,191 @@ +description: | + DRP-flavored pipeline to support validation + commissiong validation pipeline that will help inform + what changes to make the following night. Detector and Visit + level tasks can be run in real time by e.g. Rapid Analysis. + The rest of the pipeline is expected to be run as part of the + 10am processing. +instrument: lsst.obs.lsst.LsstComCamSim +imports: + - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCamSim/DRP.yaml +tasks: + analyzePreSourceTableCore: + class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask + config: + connections.data: preSourceTable_visit + connections.inputName: preSourceTable_visit + connections.outputName: preSourceTableCore +subsets: + # These 3 steps can be run in real time by Rapid Analysis Framework at USDF + step1: + subset: + - isr + - characterizeImage + - calibrate + - writePreSourceTable + - transformPreSourceTable + description: | + Detector level tasks. TO DO: requests include synthetic source injection, + subtractImages, detectAndMeasureDiaSources. + step2a: + subset: + - consolidatePreSourceTable + - consolidateVisitSummary + - analyzePreSourceTableCore + - catalogMatchPreVisit + - astrometricRefCatPreSourceVisit # configure for metrics only + # TO DO: Add photometricMatchPreVisit, photometricRefCatPreSourceVisit + description: Visit-level tasks + nightlyRollup: + subset: + - makePreliminaryCcdVisitTable + - makePreliminaryVisitTable + - preliminaryVisitCoverageAnalysis + description: | + Global tasks that can be run at end of night or multiple times during + the night, to get a summary of observations taken. + # The following N steps should be run once at the end of the night + step2b: + subset: + - analyzeMatchedPreVisitCore + - isolatedStarAssociation + - gbdesAstrometricFit + description: | + Tract-level tasks + Allowed data query constraints: tract + + GbdesAstrometricFit and isolatedStarAssociation both use PreSources, + generated by consolidatePreSourceTable, for all visits that overlap a + tract. + gbdesAstrometricFit produces solutions per-tract, per-visit + isolatedStarAssociation produces solutions per-tract. + step2c: + subset: + - fgcmBuildFromIsolatedStars + - fgcmFitCycle + - fgcmOutputProducts + description: | + Global-level tasks that must not be run with any data query constraints + Can be run anytime after subset step2b. + Allowed data query constraints: instrument + step2d: + subset: + - finalizeCharacterization + - writeRecalibratedSourceTable + - transformSourceTable + - consolidateSourceTable + - updateVisitSummary + description: | + Visit-level tasks to re-estimate PSFs and apply + global calibrations to Source tables. + Allowed data query constraints: visit + step2e: + subset: + - makeCcdVisitTable + - makeVisitTable + description: | + Global-level tasks that must not be run with any data query constraints + Can be run anytime after subset step2d. + + Allowed data query constraints: instrument + + Tasks generate one data product per collection. + make[Ccd]VisitTable produces per-collection summary of the Visits + and CcdVisits. + step3: + subset: + - makeWarp + - selectDeepCoaddVisits + - assembleCoadd + - detection + - mergeDetections + - deblend + - measure + - mergeMeasurements + - forcedPhotCoadd + - transformObjectTable + - writeObjectTable + - consolidateObjectTable + - healSparsePropertyMaps + - selectGoodSeeingVisits + - templateGen + # analysis_tools tasks + - analyzeMatchedVisitCore + - analyzeObjectTableCore + - analyzeObjectTableExtended + - catalogMatchTract + - photometricCatalogMatch + - photometricRefCatObjectTract + - plotPropertyMapTract + - refCatObjectTract + - validateObjectTableCore + description: | + Patch and Tract-level tasks that make coadds and object tables that + can be run together, but only after the 'step1' and 'step2d' + subsets. + + These should be run with explicit 'tract' constraints essentially all the + time, because otherwise quanta will be created for jobs with only partial + visit coverage. + step4: + subset: + - forcedPhotCcd + - forcedPhotDiffim + - getTemplate + - subtractImages + - detectAndMeasureDiaSources + - transformDiaSourceCat + - writeForcedSourceTable + description: | + Image subtraction and forced photometry Tasks that can be run together, + but only after the 'step1', 'step2d' and 'step3' subsets + + These detector-level tasks should not be run with 'tract' or 'patch' as + part of the data ID expression if all reference catalogs or diffIm + templates that cover these detector-level quanta are desired. + step5: + subset: + - drpAssociation + - drpDiaCalculation + - forcedPhotCcdOnDiaObjects + - forcedPhotDiffOnDiaObjects + - transformForcedSourceTable + - consolidateAssocDiaSourceTable + - consolidateFullDiaObjectTable + - writeForcedSourceOnDiaObjectTable + - transformForcedSourceOnDiaObjectTable + - analyzeDiaSourceTableTract + description: | + ForcedSource and DIASource association Tasks that can be run together, + but only after the 'step1', 'step2', 'step3', and 'step4' subsets + + This step includes patch-level aggregation Tasks. These should be run + with explicit 'tract' constraints in the data query, otherwise quanta + will be created for jobs with only partial visit coverage. + step6: + subset: + - consolidateDiaSourceTable + description: | + Tasks that can be run together, but only after the 'step1', 'step2', + 'step3', and 'step4' subsets + + This step includes visit-level aggregation tasks. Running without tract + or patch in the data query is recommended, otherwise the outputs of + consolidateDiaSourceTable will not contain complete visits. + + This subset is separate from step4 to signal to operators to pause to + assess unexpected image differencing failures before these aggregation + steps. Otherwise, if run in the same quantum graph, aggregated data + products (e.g. diaObjects) would not be created if one or more of the + expected inputs is missing. + step7: + subset: + - consolidateHealSparsePropertyMaps + - analyzeObjectTableSurveyCore + description: | + Tasks that should be run as the final step that require global inputs, + and can be run after the 'step3' subset. + + This step has global aggregation tasks to run over all visits, detectors, + tracts, etc. This step should be run only with the instrument constraint + in the data query. diff --git a/pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml b/pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml new file mode 100644 index 00000000..d050ae7c --- /dev/null +++ b/pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml @@ -0,0 +1,70 @@ +description: | + A fast pipeline to be run on the rapid analysis framework at + the summit which feeds RubinTV for realtime observer feedback +instrument: lsst.obs.lsst.LsstComCamSim +imports: + - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCamSim/DRP.yaml +tasks: + isr: + class: lsst.ip.isr.IsrTask + config: + # Turn off all slow steps in ISR for + doBrighterFatter: false + # TO DO: Replace with calibrateImage in a way that we can use the + # preSources downstream. + characterizeImage: + class: lsst.pipe.tasks.characterizeImage.CharacterizeImageTask + config: + doApCorr: false + doDeblend: false + doMaskStreaks: false + calibrate: + class: lsst.pipe.tasks.calibrate.CalibrateTask + config: + python: | + config.measurement.plugins.names = ['base_CircularApertureFlux', + 'base_PsfFlux', + 'base_NaiveCentroid', + 'base_CompensatedGaussianFlux', + 'base_LocalBackground', + 'base_SdssCentroid', + 'base_SdssShape', + 'base_Variance', + 'base_Jacobian', + 'base_PixelFlags', + 'base_GaussianFlux', + 'base_SkyCoord', + 'base_FPPosition', + 'base_ClassificationSizeExtendedness', + ] + config.measurement.slots.shape = "base_SdssShape" + config.measurement.slots.psfShape = "base_SdssShape_psf" + config.measurement.plugins['base_PixelFlags'].masksFpAnywhere.remove('STREAK') + config.measurement.plugins['base_PixelFlags'].masksFpCenter.remove('STREAK') + config.doApCorr = False + config.doDeblend = False + config.astrometry.sourceSelector["science"].doRequirePrimary = False + config.astrometry.sourceSelector["science"].doIsolated = False + # TO DO: DM-41189 Get pipeline that will attach an atool here from lauren +subsets: + step1: + subset: + - isr + - characterizeImage + - calibrate + # TO DO: configure and add writePreSourceTable, transformPreSourceTable + description: | + Per-detector tasks that can be run together + step2a: + subset: + - consolidateVisitSummary + # TO DO: configure and add consolidatePreSourceTable + nightlyRollup: + # can be run at end of night or on a timer + # Need to either clobber OR use a chained collection of timestamped runs + subset: + - makePreliminaryCcdVisitTable + - makePreliminaryVisitTable + - preliminaryVisitCoverageAnalysis + description: | + Global-level tasks diff --git a/pipelines/_ingredients/DRP-full.yaml b/pipelines/_ingredients/DRP-full.yaml index 7b355d56..c94c9baf 100644 --- a/pipelines/_ingredients/DRP-full.yaml +++ b/pipelines/_ingredients/DRP-full.yaml @@ -51,6 +51,16 @@ tasks: class: lsst.pipe.tasks.multiBand.MeasureMergedCoaddSourcesTask config: connections.sourceTableHandles: preSourceTable_visit + makePreliminaryCcdVisitTable: + class: lsst.pipe.tasks.postprocess.MakeCcdVisitTableTask + config: + connections.visitSummaryRefs: visitSummary + connections.outputCatalog: preCcdVisitTable + makePreliminaryVisitTable: + class: lsst.pipe.tasks.postprocess.MakeVisitTableTask + config: + connections.visitSummaries: visitSummary + connections.outputCatalog: preVisitTable subsets: # This pipeline file only defines subsets that have a consistent or # near-consistent definition as a set of tasks, for use by downstream diff --git a/pipelines/_ingredients/LSSTComCamSim/DRP.yaml b/pipelines/_ingredients/LSSTComCamSim/DRP.yaml new file mode 100644 index 00000000..03b801c2 --- /dev/null +++ b/pipelines/_ingredients/LSSTComCamSim/DRP.yaml @@ -0,0 +1,137 @@ +description: DRP Pipeline for Simulated ComCam Ops Rehearsal 3 +instrument: lsst.obs.lsst.LsstComCamSim +imports: + - $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/coaddColumnValidate.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityExtended.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/diaTractQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/matchedVisitQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/visitQualityCore.yaml +tasks: + fgcmFitCycle: + class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask + config: + doMultipleCycles: true + multipleCyclesFinalCycleNumber: 4 + fgcmOutputProducts: + class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask + config: + connections.cycleNumber: 4 + selectGoodSeeingVisits: + class: lsst.pipe.tasks.selectImages.BestSeeingQuantileSelectVisitsTask + config: + connections.goodVisits: goodSeeingVisits + nVisitsMin: 12 + # For simulation 2 there are 60 visits per region + # between 60399.362 and 60402.419. + minMJD: 60399.362 + maxMJD: 60402.419 + updateVisitSummary: + class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask + config: + # TODO: Evaluate GBDES, SkyCorr, FGCM after initial DRP + wcs_provider: "tract" + photo_calib_provider: "global" + background_provider: "input_summary" + analyzeMatchedPreVisitCore: + class: lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask + config: + connections.sourceCatalogs: preSourceTable_visit + connections.outputName: matchedPreVisitCore + # The rest should be the same as configs in matchedVisitQualityCore.yaml + atools.stellarPhotometricRepeatability: StellarPhotometricRepeatability + atools.stellarPhotometricResiduals: StellarPhotometricResidualsFocalPlane + atools.stellarAstrometricResidualsRA: StellarAstrometricResidualsRAFocalPlanePlot + atools.stellarAstrometricResidualsDec: StellarAstrometricResidualsDecFocalPlanePlot + atools.stellarAstrometricResidualStdDevRA: StellarAstrometricResidualStdDevRAFocalPlanePlot + atools.stellarAstrometricResidualStdDevDec: StellarAstrometricResidualStdDevDecFocalPlanePlot + atools.stellarAstrometricRepeatability1: AstrometricRelativeRepeatability + atools.stellarAstrometricRepeatability1.xValue: 1 + atools.stellarAstrometricRepeatability1.process.calculateActions.rms.annulus: 5 + atools.stellarAstrometricRepeatability2: AstrometricRelativeRepeatability + atools.stellarAstrometricRepeatability2.xValue: 2 + atools.stellarAstrometricRepeatability2.process.calculateActions.rms.annulus: 20 + atools.stellarAstrometricRepeatability3: AstrometricRelativeRepeatability + atools.stellarAstrometricRepeatability3.xValue: 3 + atools.stellarAstrometricRepeatability3.process.calculateActions.rms.annulus: 200 + atools.stellarAstrometricRepeatability3.process.calculateActions.rms.threshAD: 30 + python: | + from lsst.analysis.tools.atools import * + analyzeObjectTableCore: + class: lsst.analysis.tools.tasks.ObjectTableTractAnalysisTask + config: + python: | + # no z-band in comCamSim + del config.atools.yPerpPSF + del config.atools.yPerpCModel + analyzePreSourceTableCore: + class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask + config: + connections.outputName: preSourceTableCore + atools.skyFluxVisitStatisticMetric: SkyFluxStatisticMetric + atools.skyFluxVisitStatisticMetric.applyContext: VisitContext + atools.skySourceSky: SkySourceSkyPlot + atools.skySourceFlux: SkySourceHistPlot + python: | + from lsst.analysis.tools.atools import * + from lsst.analysis.tools.contexts import * + catalogMatchPreVisit: + class: lsst.analysis.tools.tasks.astrometricCatalogMatch.AstrometricCatalogMatchVisitTask + config: + connections.catalog: "preSourceTable_visit" + connections.targetCatalog: "preSourceTable_visit" + astrometricRefCatPreSourceVisit: + class: lsst.analysis.tools.tasks.refCatSourceAnalysis.RefCatSourceAnalysisTask + config: + # Only run metrics for analyzing the preSources: + connections.data: preSourceTable_visit_uw_stars_20240228_match + atools.astromDiffMetrics: TargetRefCatDeltaMetrics + atools.astromDiffMetrics.applyContext: VisitContext + python: | + from lsst.analysis.tools.atools import * + from lsst.analysis.tools.contexts import * + # These don't actually work now: + photometricMatchPreVisit: + class: lsst.analysis.tools.tasks.photometricCatalogMatch.PhotometricCatalogMatchVisitTask + config: + extraColumns: ["x", "y", "ap09Flux", "ap09FluxErr"] + connections.refCatalog: uw_stars_20240228 + connections.catalog: preSourceTable_visit + connections.targetCatalog: preSourceTable_visit + connections.matchedCatalog: preSourceTable_visit_uw_stars_20240228_photoMatch + photometricRefCatPreSourceVisit: + class: lsst.analysis.tools.tasks.refCatSourcePhotometricAnalysis.RefCatSourcePhotometricAnalysisTask + config: + # Only run metrics for analyzing the preSources: + connections.data: preSourceTable_visit_uw_stars_20240228_photoMatch + python: from lsst.analysis.tools.atools import * + preliminaryVisitCoverageAnalysis: + class: lsst.analysis.tools.tasks.CcdVisitTableAnalysisTask + config: + connections.data: preCcdVisitTable + introspectMakeWarpConfig: false + atools.raDecCoveragePlot: RaDecMultiVisitCoveragePlot + atools.focalPlaneCoveragePlot: FocalPlaneMultiVisitCoveragePlot + atools.raDecCoveragePlot.produce.plot.parametersToPlotList: ["psfSigma", "astromOffsetMean", "medianE"] + atools.raDecCoveragePlot.produce.plot.plotDetectorOutline: true + python: | + # TO DO: Move as much of this as possible into the + from lsst.analysis.tools.atools import * + # RA/Dec Plot + config.atools.raDecCoverageAltParsPlot = RaDecMultiVisitCoveragePlot + config.atools.raDecCoverageAltParsPlot.produce.plot.tractsToPlotList = None + config.atools.raDecCoverageAltParsPlot.produce.plot.trimToTract = True + config.atools.raDecCoverageAltParsPlot.produce.plot.vectorsToLoadList.extend(["zenithDistance", "nPsfStar", "skyNoise"]) + config.atools.raDecCoverageAltParsPlot.updateLoadList() + config.atools.raDecCoverageAltParsPlot.produce.plot.parametersToPlotList = ["zenithDistance", "nPsfStar", "skyNoise"] + config.atools.raDecCoverageAltParsPlot.produce.plot.plotDetectorOutline = False + config.atools.raDecCoverageAltParsPlot.produce.plot.plotAllTractOutlines = True + config.atools.raDecCoverageAltParsPlot.produce.plot.showExtremeOutliers = False + # Focal Plane Plot + config.atools.focalPlaneCoverageAltParsPlot = FocalPlaneMultiVisitCoveragePlot + config.atools.focalPlaneCoverageAltParsPlot.produce.plot.vectorsToLoadList.extend(["zenithDistance", "nPsfStar", "skyNoise"]) + config.atools.focalPlaneCoverageAltParsPlot.updateLoadList() + config.atools.focalPlaneCoverageAltParsPlot.produce.plot.parametersToPlotList = [ + "zenithDistance", "nPsfStar", "skyNoise"] + config.atools.focalPlaneCoverageAltParsPlot.produce.plot.nPointBinThresh = 100