diff --git a/pipelines/LSSTComCam/DRP.yaml b/pipelines/LSSTComCam/DRP.yaml new file mode 100644 index 00000000..c0b4dde3 --- /dev/null +++ b/pipelines/LSSTComCam/DRP.yaml @@ -0,0 +1,181 @@ +description: DRP Pipeline for LSSTComCam +instrument: lsst.obs.lsst.LsstComCam +imports: + - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCam/DRP.yaml + +subsets: + step1: + subset: + - isr + - characterizeImage + - calibrate + - writePreSourceTable + - transformPreSourceTable + description: | + Per-detector tasks that can be run together to start the DRP pipeline. + + These should never be run with 'tract' or 'patch' as part of the data ID + expression if any later steps will also be run, because downstream steps + require full visits and 'tract' and 'patch' constraints will always + select partial visits that overlap that region. + step2a: + subset: + - analyzeMatchedPreVisitCore + - consolidatePreSourceTable + - consolidateVisitSummary + description: | + Visit-level tasks + Allowed data query constraints: visit + + Tasks aggregate all detectors for a given visit and perform + full focal plane background estimation. + + TODO: Evaluate SkyCorr after intial DRP + step2b: + subset: + - isolatedStarAssociation + - gbdesAstrometricFit + description: | + Tract-level tasks + Allowed data query constraints: tract + + GbdesAstrometricFit and isolatedStarAssociation both use PreSources, + generated by consolidatePreSourceTable, for all visits that overlap a + tract. + gbdesAstrometricFit produces solutions per-tract, per-visit + isolatedStarAssociation produces solutions per-tract. + TODO: Evaluate GBDES after first DRP + step2c: + subset: + - fgcmBuildFromIsolatedStars + - fgcmFitCycle + - fgcmOutputProducts + step2d: + subset: + - finalizeCharacterization + - writeRecalibratedSourceTable + - transformSourceTable + - consolidateSourceTable + - updateVisitSummary + description: | + Visit-level tasks. + Allowed data query constraints: visit + + writeRecalibratedSourceTable, transformSourceTable run per-detector + consolidateSourceTable produces one data product per visit. + finalizeCharacterization will eventually model full focal plane PSFs. + step2e: + subset: + - makeCcdVisitTable + - makeVisitTable + description: | + Global-level tasks that must not be run with any data query constraints + Can be run anytime after subset step2d. + + Allowed data query constraints: instrument + + Tasks generate one data product per collection. + make[Ccd]VisitTable produces per-collection summary of the Visits + and CcdVisits. + step3: + subset: + - makeWarp + - selectDeepCoaddVisits + - assembleCoadd + - detection + - mergeDetections + - deblend + - measure + - mergeMeasurements + - forcedPhotCoadd + - transformObjectTable + - writeObjectTable + - consolidateObjectTable + - healSparsePropertyMaps + - selectGoodSeeingVisits + - templateGen + # analysis_tools tasks + - analyzeMatchedVisitCore + - analyzeObjectTableCore + - catalogMatchTract + - photometricCatalogMatch + - photometricRefCatObjectTract + - plotPropertyMapTract + - refCatObjectTract + - validateObjectTableCore + description: | + Tasks that can be run together, but only after the 'step1' and 'step2d' + subsets. + + These should be run with explicit 'tract' constraints essentially all the + time, because otherwise quanta will be created for jobs with only partial + visit coverage. + + This subset is considered a workaround for missing middleware and task + functionality. It may be removed in the future. + step4: + subset: + - forcedPhotCcd + - forcedPhotDiffim + - getTemplate + - subtractImages + - detectAndMeasureDiaSources + - transformDiaSourceCat + - writeForcedSourceTable + description: | + Tasks that can be run together, but only after the 'step1', 'step2d' and + 'step3' subsets + + These detector-level tasks should not be run with 'tract' or 'patch' as + part of the data ID expression if all reference catalogs or diffIm + templates that cover these detector-level quanta are desired. + step5: + subset: + - drpAssociation + - drpDiaCalculation + - forcedPhotCcdOnDiaObjects + - forcedPhotDiffOnDiaObjects + - transformForcedSourceTable + - consolidateAssocDiaSourceTable + - consolidateFullDiaObjectTable + - writeForcedSourceOnDiaObjectTable + - transformForcedSourceOnDiaObjectTable + - analyzeDiaSourceTableTract + description: | + Tasks that can be run together, but only after the 'step1', 'step2', + 'step3', and 'step4' subsets + + This step includes patch-level aggregation Tasks. These should be run + with explicit 'tract' constraints in the data query, otherwise quanta + will be created for jobs with only partial visit coverage. + step6: + subset: + - consolidateDiaSourceTable + description: | + Tasks that can be run together, but only after the 'step1', 'step2', + 'step3', and 'step4' subsets + + This step includes visit-level aggregation tasks. Running without tract + or patch in the data query is recommended, otherwise the outputs of + consolidateDiaSourceTable will not contain complete visits. + + This subset is separate from step4 to signal to operators to pause to + assess unexpected image differencing failures before these aggregation + steps. Otherwise, if run in the same quantum graph, aggregated data + products (e.g. diaObjects) would not be created if one or more of the + expected inputs is missing. + step7: + subset: + - consolidateHealSparsePropertyMaps + - analyzeObjectTableSurveyCore + - makeMetricTableMatchedVisitCore + - makeMetricTableObjectTableCore + - matchedVisitCoreWholeSkyPlot + - objectTableCoreWholeSkyPlot + description: | + Tasks that should be run as the final step that require global inputs, + and can be run after the 'step3' subset. + + This step has global aggregation tasks to run over all visits, detectors, + tracts, etc. This step should be run only with the instrument constraint + in the data query. diff --git a/pipelines/LSSTComCam/nightly-validation.yaml b/pipelines/LSSTComCam/nightly-validation.yaml new file mode 100644 index 00000000..73e28923 --- /dev/null +++ b/pipelines/LSSTComCam/nightly-validation.yaml @@ -0,0 +1,148 @@ +description: | + DRP-flavored pipeline to support validation + during commissioning, and will help inform + what changes to make the following night. Detector and Visit + level tasks can be run in real time by e.g. Rapid Analysis. + The rest of the pipeline is expected to be run as part of the + 10am processing. +instrument: lsst.obs.lsst.LsstComCam +imports: + - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCam/DRP.yaml +tasks: + analyzePreSourceTableCore: + class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask + config: + connections.data: preSourceTable_visit + connections.inputName: preSourceTable_visit + connections.outputName: preSourceTableCore + calibrate: + class: lsst.pipe.tasks.calibrate.CalibrateTask + config: + python: | + from lsst.analysis.tools.tasks import CalexpSummaryAnalysisTask + config.createSummaryMetrics.retarget(CalexpSummaryAnalysisTask) + from lsst.analysis.tools.atools import CalexpSummaryMetrics + config.createSummaryMetrics.atools.calexpSummaryMetrics = CalexpSummaryMetrics + doCreateSummaryMetrics: true + updateVisitSummary: + class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask + config: + # No global calibration during nightly validation + wcs_provider: "input_summary" + photo_calib_provider: "input_summary" + background_provider: "input_summary" +subsets: + # These 3 steps can be run in real time by Rapid Analysis Framework at USDF + step1: + subset: + - isr + - characterizeImage + - calibrate + - writePreSourceTable + - transformPreSourceTable + description: | + Detector level tasks. TO DO: requests include synthetic source injection, + subtractImages, detectAndMeasureDiaSources. + step2a: + subset: + - consolidatePreSourceTable + - consolidateVisitSummary + - analyzePreSourceTableCore + - catalogMatchPreVisit + - astrometricRefCatPreSourceVisit + # TO DO: Add photometricMatchPreVisit, photometricRefCatPreSourceVisit + description: Visit-level tasks + nightlyRollup: + subset: + - makePreliminaryCcdVisitTable + - makePreliminaryVisitTable + - preliminaryVisitCoverageAnalysis + description: | + Global tasks that can be run at end of night or multiple times during + the night, to get a summary of observations taken. + # The following N steps should be run once at the end of the night + # A complete pipeline (without DIA): step2b,step2d,step2e,step3,step7 + step2b: + subset: + - isolatedStarAssociation + description: | + Tract-level tasks + Allowed data query constraints: tract + + isolatedStarAssociation uses PreSources, + generated by consolidatePreSourceTable, for all visits that overlap a + tract. + step2d: + subset: + - finalizeCharacterization + - writeRecalibratedSourceTable + - transformSourceTable + - consolidateSourceTable + - updateVisitSummary + description: | + Visit-level tasks. + Allowed data query constraints: visit + + writeRecalibratedSourceTable, transformSourceTable run per-detector + consolidateSourceTable produces one data product per visit. + finalizeCharacterization will eventually model full focal plane PSFs. + step2e: + subset: + - makeCcdVisitTable + - makeVisitTable + description: | + Global-level tasks that must not be run with any data query constraints + Can be run anytime after subset step2d. + + Allowed data query constraints: instrument + + Tasks generate one data product per collection. + make[Ccd]VisitTable produces per-collection summary of the Visits + and CcdVisits. + step3: + subset: + - makeWarp + - selectDeepCoaddVisits + - assembleCoadd + - detection + - mergeDetections + - deblend + - measure + - mergeMeasurements + - forcedPhotCoadd + - transformObjectTable + - writeObjectTable + - consolidateObjectTable + - healSparsePropertyMaps + # analysis_tools tasks + - analyzeMatchedVisitCore + - analyzeObjectTableCore + - analyzeObjectTableExtended + - catalogMatchTract + - photometricCatalogMatch + - photometricRefCatObjectTract + - plotPropertyMapTract + - refCatObjectTract + - validateObjectTableCore + description: | + This overrides the default DRP pipeline by not making nightly templates. + We may want to make daily templates in the future. + + Patch and Tract-level tasks that make coadds and object tables that + can be run together, but only after the 'step1' and 'step2d' + subsets. + step7: + subset: + - consolidateHealSparsePropertyMaps + - analyzeObjectTableSurveyCore + - makeMetricTableMatchedVisitCore + - makeMetricTableObjectTableCore + - matchedVisitCoreWholeSkyPlot + - objectTableCoreWholeSkyPlot + description: | + Tasks that should be run as the final step that require global inputs, + and can be run after the 'step3' subset. + + This step has global aggregation tasks to run over all visits, detectors, + tracts, etc. This step should be run only with the instrument constraint + in the data query. diff --git a/pipelines/LSSTComCam/quickLook.yaml b/pipelines/LSSTComCam/quickLook.yaml new file mode 100644 index 00000000..36fd0fe7 --- /dev/null +++ b/pipelines/LSSTComCam/quickLook.yaml @@ -0,0 +1,69 @@ +description: | + A fast pipeline to be run on the rapid analysis framework at + the summit which feeds RubinTV for realtime observer feedback +instrument: lsst.obs.lsst.LsstComCam +imports: + - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCam/DRP.yaml +tasks: + isr: + class: lsst.ip.isr.IsrTask + config: + # Turn off slow steps in ISR + doBrighterFatter: false + doCrosstalk: false + # TO DO: Replace with calibrateImage in a way that we can use the + # preSources downstream. + characterizeImage: + class: lsst.pipe.tasks.characterizeImage.CharacterizeImageTask + config: + doApCorr: false + doDeblend: false + calibrate: + class: lsst.pipe.tasks.calibrate.CalibrateTask + config: + python: | + config.measurement.plugins.names = ['base_CircularApertureFlux', + 'base_PsfFlux', + 'base_LocalBackground', + 'base_SdssCentroid', + 'base_SdssShape', + 'base_Variance', + 'base_Jacobian', + 'base_PixelFlags', + 'base_GaussianFlux', + 'base_SkyCoord', + 'base_FPPosition', + 'base_ClassificationSizeExtendedness', + 'base_CompensatedTophatFlux', + ] + config.measurement.slots.shape = "base_SdssShape" + config.measurement.slots.psfShape = "base_SdssShape_psf" + config.measurement.plugins['base_PixelFlags'].masksFpAnywhere.remove('STREAK') + config.measurement.plugins['base_PixelFlags'].masksFpCenter.remove('STREAK') + config.doApCorr = False + config.doDeblend = False + config.astrometry.sourceSelector["science"].doRequirePrimary = False + config.astrometry.sourceSelector["science"].doIsolated = False + # TO DO: DM-41189 Get pipeline that will attach an atool here from lauren +subsets: + step1: + subset: + - isr + # - characterizeImage + # - calibrate + # TO DO: configure and add writePreSourceTable, transformPreSourceTable + description: | + Per-detector tasks that can be run together + step2a: + subset: + - consolidateVisitSummary + # TO DO: configure and add consolidatePreSourceTable + nightlyRollup: + # can be run at end of night or on a timer + # Need to either clobber OR use a chained collection of timestamped runs + subset: + - makePreliminaryCcdVisitTable + - makePreliminaryVisitTable + - preliminaryVisitCoverageAnalysis + description: | + Global-level tasks diff --git a/pipelines/_ingredients/LSSTComCam/DRP.yaml b/pipelines/_ingredients/LSSTComCam/DRP.yaml new file mode 100644 index 00000000..7b8805a2 --- /dev/null +++ b/pipelines/_ingredients/LSSTComCam/DRP.yaml @@ -0,0 +1,136 @@ +description: DRP Pipeline for Simulated ComCam Ops Rehearsals +instrument: lsst.obs.lsst.LsstComCam +imports: + - $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/coaddColumnValidate.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityExtended.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/diaTractQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/matchedVisitQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/visitQualityCore.yaml + - $ANALYSIS_TOOLS_DIR/pipelines/wholeSkyCore.yaml +tasks: + fgcmFitCycle: + class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask + config: + doMultipleCycles: true + multipleCyclesFinalCycleNumber: 4 + fgcmOutputProducts: + class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask + config: + connections.cycleNumber: 4 + selectGoodSeeingVisits: + class: lsst.pipe.tasks.selectImages.BestSeeingQuantileSelectVisitsTask + config: + connections.goodVisits: goodSeeingVisits + nVisitsMin: 12 + # For simulation 2 there are 60 visits per region + # between 60399.362 and 60402.419. + minMJD: 60399.362 + maxMJD: 60402.419 + updateVisitSummary: + class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask + config: + # TODO: Evaluate GBDES, SkyCorr, FGCM after initial DRP + wcs_provider: "tract" + photo_calib_provider: "global" + background_provider: "input_summary" + analyzeMatchedPreVisitCore: + class: lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask + config: + connections.sourceCatalogs: preSourceTable_visit + connections.outputName: matchedPreVisitCore + # The rest should be the same as configs in matchedVisitQualityCore.yaml + atools.stellarPhotometricRepeatability: StellarPhotometricRepeatability + atools.stellarPhotometricResiduals: StellarPhotometricResidualsFocalPlane + atools.stellarAstrometricResidualsRA: StellarAstrometricResidualsRAFocalPlanePlot + atools.stellarAstrometricResidualsDec: StellarAstrometricResidualsDecFocalPlanePlot + atools.stellarAstrometricResidualStdDevRA: StellarAstrometricResidualStdDevRAFocalPlanePlot + atools.stellarAstrometricResidualStdDevDec: StellarAstrometricResidualStdDevDecFocalPlanePlot + atools.stellarAstrometricRepeatability1: AstrometricRelativeRepeatability + atools.stellarAstrometricRepeatability1.xValue: 1 + atools.stellarAstrometricRepeatability1.process.calculateActions.rms.annulus: 5 + atools.stellarAstrometricRepeatability2: AstrometricRelativeRepeatability + atools.stellarAstrometricRepeatability2.xValue: 2 + atools.stellarAstrometricRepeatability2.process.calculateActions.rms.annulus: 20 + atools.stellarAstrometricRepeatability3: AstrometricRelativeRepeatability + atools.stellarAstrometricRepeatability3.xValue: 3 + atools.stellarAstrometricRepeatability3.process.calculateActions.rms.annulus: 200 + atools.stellarAstrometricRepeatability3.process.calculateActions.rms.threshAD: 30 + python: | + from lsst.analysis.tools.atools import * + analyzePreSourceTableCore: + class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask + config: + connections.outputName: preSourceTableCore + atools.skyFluxVisitStatisticMetric: SkyFluxStatisticMetric + atools.skyFluxVisitStatisticMetric.applyContext: VisitContext + atools.skySourceSky: SkySourceSkyPlot + atools.skySourceFlux: SkySourceHistPlot + python: | + from lsst.analysis.tools.atools import * + from lsst.analysis.tools.contexts import * + catalogMatchPreVisit: + class: lsst.analysis.tools.tasks.astrometricCatalogMatch.AstrometricCatalogMatchVisitTask + config: + connections.catalog: "preSourceTable_visit" + connections.targetCatalog: "preSourceTable_visit" + astrometricRefCatPreSourceVisit: + class: lsst.analysis.tools.tasks.refCatSourceAnalysis.RefCatSourceAnalysisTask + config: + # Only run metrics for analyzing the preSources: + connections.data: preSourceTable_visit_gaia_dr3_20230707_match_astrom + atools.astromDiffMetrics: TargetRefCatDeltaMetrics + atools.astromDiffMetrics.applyContext: VisitContext + python: | + from lsst.analysis.tools.atools import * + from lsst.analysis.tools.contexts import * + matchedVisitCoreWholeSkyPlot: + class: lsst.analysis.tools.tasks.WholeSkyAnalysisTask + config: + connections.inputName: "matchedVisitCore_metricsTable" + connections.outputName: "matchedVisitCore_wholeSky" + atools.wholeSkyMetric: WholeSkyPlotTool + atools.wholeSkyMetric.plotKeys: [] + atools.wholeSkyMetric.keysWithBand: parameters.matchedVisitCoreKeysWithBand + python: | + from lsst.analysis.tools.atools import * + objectTableCoreWholeSkyPlot: + class: lsst.analysis.tools.tasks.WholeSkyAnalysisTask + config: + atools.wholeSkyMetric: WholeSkyPlotTool + atools.wholeSkyMetric.plotKeys: [ + "wPerpPSFP_wPerp_psfFlux_median", + "wPerpPSFP_wPerp_psfFlux_sigmaMAD", + "skippedDeblenderMetrics_numSkippedPeaks", + "skippedDeblenderMetrics_numSkippedBlends", + "skippedDeblenderMetrics_numBlendParentTooBig", + "skippedDeblenderMetrics_numBlendTooManyPeaks", + "skippedDeblenderMetrics_numBlendTooManyMasked", + ] + atools.wholeSkyMetric.keysWithBand: parameters.objectTableCoreKeysWithBand + python: | + from lsst.analysis.tools.atools import * + # These don't actually work now: + photometricMatchPreVisit: + class: lsst.analysis.tools.tasks.photometricCatalogMatch.PhotometricCatalogMatchVisitTask + config: + extraColumns: ["x", "y", "ap09Flux", "ap09FluxErr"] + connections.refCatalog: gaia_dr3_20230707 + connections.catalog: preSourceTable_visit + connections.targetCatalog: preSourceTable_visit + connections.matchedCatalog: preSourceTable_visit_gaia_dr3_20230707_photoMatch + photometricRefCatPreSourceVisit: + class: lsst.analysis.tools.tasks.refCatSourcePhotometricAnalysis.RefCatSourcePhotometricAnalysisTask + config: + # Only run metrics for analyzing the preSources: + connections.data: preSourceTable_visit_gaia_dr3_20230707_photoMatch + python: from lsst.analysis.tools.atools import * + preliminaryVisitCoverageAnalysis: + class: lsst.analysis.tools.tasks.CcdVisitTableAnalysisTask + config: + connections.data: preCcdVisitTable + atools.raDecCoveragePlot: RaDecMultiVisitCoveragePlot + atools.raDecCoveragePlot.produce.plot.showExtremeOutliers: false + atools.focalPlaneCoveragePlot: FocalPlaneMultiVisitCoveragePlot + python: from lsst.analysis.tools.atools import *