lsst · yalsayyad · Mar 21, 2024 · Mar 15, 2024
diff --git a/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml b/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml
@@ -1,48 +1,8 @@
 description: DRP Pipeline for Simulated ComCam Ops Rehearsal 3
 instrument: lsst.obs.lsst.LsstComCamSim
 imports:
-  - $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml
-  - $ANALYSIS_TOOLS_DIR/pipelines/coaddColumnValidate.yaml
-  - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityCore.yaml
-  - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityExtended.yaml
-  - $ANALYSIS_TOOLS_DIR/pipelines/diaTractQualityCore.yaml
-  - $ANALYSIS_TOOLS_DIR/pipelines/matchedVisitQualityCore.yaml
-  - $ANALYSIS_TOOLS_DIR/pipelines/visitQualityCore.yaml
-tasks:
-  fgcmFitCycle:
-    class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask
-    config:
-      doMultipleCycles: true
-      multipleCyclesFinalCycleNumber: 4
+  - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCamSim/DRP.yaml
 
-  fgcmOutputProducts:
-    class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask
-    config:
-      connections.cycleNumber: 4
-
-  selectGoodSeeingVisits:
-    class: lsst.pipe.tasks.selectImages.BestSeeingQuantileSelectVisitsTask
-    config:
-      connections.goodVisits: goodSeeingVisits
-      nVisitsMin: 12
-      # For simulation 2 there are 60 visits per region
-      # between 60399.362 and 60402.419.
-      minMJD: 60399.362
-      maxMJD: 60402.419
-  updateVisitSummary:
-    class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask
-    config:
-      # TODO: Evaluate GBDES, SkyCorr, FGCM after initial DRP
-      wcs_provider: "tract"
-      photo_calib_provider: "global"
-      background_provider: "input_summary"
-  analyzeObjectTableCore:
-    class: lsst.analysis.tools.tasks.ObjectTableTractAnalysisTask
-    config:
-      python: |
-        # no z-band in comCamSim
-        del config.atools.yPerpPSF
-        del config.atools.yPerpCModel
 subsets:
   step1:
     subset:
@@ -60,6 +20,7 @@ subsets:
       select partial visits that overlap that region.
   step2a:
     subset:
+      - analyzeMatchedPreVisitCore
       - consolidatePreSourceTable
       - consolidateVisitSummary
     description: |

diff --git a/pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml b/pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml
@@ -0,0 +1,84 @@
+description: |
+  DRP-flavored pipeline to support validation
+  during commissioning, and will help inform
+  what changes to make the following night. Detector and Visit
+  level tasks can be run in real time by e.g. Rapid Analysis.
+  The rest of the pipeline is expected to be run as part of the
+  10am processing.
+instrument: lsst.obs.lsst.LsstComCamSim
+imports:
+  - $DRP_PIPE_DIR/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml
+tasks:
+  analyzePreSourceTableCore:
+    class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask
+    config:
+      connections.data: preSourceTable_visit
+      connections.inputName: preSourceTable_visit
+      connections.outputName: preSourceTableCore
+subsets:
+  # These 3 steps can be run in real time by Rapid Analysis Framework at USDF
+  step1:
+    subset:
+      - isr
+      - characterizeImage
+      - calibrate
+      - writePreSourceTable
+      - transformPreSourceTable
+    description: |
+      Detector level tasks. TO DO: requests include synthetic source injection,
+      subtractImages, detectAndMeasureDiaSources.
+  step2a:
+    subset:
+      - consolidatePreSourceTable
+      - consolidateVisitSummary
+      - analyzePreSourceTableCore
+      - catalogMatchPreVisit
+      - astrometricRefCatPreSourceVisit
+      # TO DO: Add photometricMatchPreVisit, photometricRefCatPreSourceVisit
+    description: Visit-level tasks
+  nightlyRollup:
+    subset:
+      - makePreliminaryCcdVisitTable
+      - makePreliminaryVisitTable
+      - preliminaryVisitCoverageAnalysis
+    description: |
+      Global tasks that can be run at end of night or multiple times during
+      the night, to get a summary of observations taken.
+  # The following N steps should be run once at the end of the night
+  # Steps 2b,2c,2d,2e,4,5,6,7 are currently the same as DRP.
+  # This step3 here does not make templates. To run step4,step5,step6 you must
+  # provide a collection with templates as input, ideally the same one that
+  # was used for prompt processing.
+  # A complete pipeline without DIA is #step2b,step2c,step2d,step2e,step3,step7
+  step3:
+    subset:
+      - makeWarp
+      - selectDeepCoaddVisits
+      - assembleCoadd
+      - detection
+      - mergeDetections
+      - deblend
+      - measure
+      - mergeMeasurements
+      - forcedPhotCoadd
+      - transformObjectTable
+      - writeObjectTable
+      - consolidateObjectTable
+      - healSparsePropertyMaps
+      # analysis_tools tasks
+      - analyzeMatchedVisitCore
+      - analyzeObjectTableCore
+      - analyzeObjectTableExtended
+      - catalogMatchTract
+      - photometricCatalogMatch
+      - photometricRefCatObjectTract
+      - plotPropertyMapTract
+      - refCatObjectTract
+      - validateObjectTableCore
+    description: |
+      This overrides the default DRP pipeline by not making nightly templates.
+      We may want to make daily templates in the future.
+
+      Patch and Tract-level tasks that make coadds and object tables that
+      can be run together, but only after the 'step1' and 'step2d'
+      subsets.
diff --git a/pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml b/pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml
@@ -0,0 +1,70 @@
+description: |
+    A fast pipeline to be run on the rapid analysis framework at
+    the summit which feeds RubinTV for realtime observer feedback
+instrument: lsst.obs.lsst.LsstComCamSim
+imports:
+  - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCamSim/DRP.yaml
+tasks:
+  isr:
+    class: lsst.ip.isr.IsrTask
+    config:
+      # Turn off slow steps in ISR
+      doBrighterFatter: false
+  # TO DO: Replace with calibrateImage in a way that we can use the
+  # preSources downstream.
+  characterizeImage:
+    class: lsst.pipe.tasks.characterizeImage.CharacterizeImageTask
+    config:
+      doApCorr: false
+      doDeblend: false
+      doMaskStreaks: false
+  calibrate:
+    class: lsst.pipe.tasks.calibrate.CalibrateTask
+    config:
+      python: |
+          config.measurement.plugins.names = ['base_CircularApertureFlux',
+                                              'base_PsfFlux',
+                                              'base_NaiveCentroid',
+                                              'base_CompensatedGaussianFlux',
+                                              'base_LocalBackground',
+                                              'base_SdssCentroid',
+                                              'base_SdssShape',
+                                              'base_Variance',
+                                              'base_Jacobian',
+                                              'base_PixelFlags',
+                                              'base_GaussianFlux',
+                                              'base_SkyCoord',
+                                              'base_FPPosition',
+                                              'base_ClassificationSizeExtendedness',
+                                              ]
+          config.measurement.slots.shape = "base_SdssShape"
+          config.measurement.slots.psfShape = "base_SdssShape_psf"
+          config.measurement.plugins['base_PixelFlags'].masksFpAnywhere.remove('STREAK')
+          config.measurement.plugins['base_PixelFlags'].masksFpCenter.remove('STREAK')
+          config.doApCorr = False
+          config.doDeblend = False
+          config.astrometry.sourceSelector["science"].doRequirePrimary = False
+          config.astrometry.sourceSelector["science"].doIsolated = False
+      # TO DO: DM-41189 Get pipeline that will attach an atool here from lauren
+subsets:
+  step1:
+    subset:
+      - isr
+      - characterizeImage
+      - calibrate
+      # TO DO: configure and add writePreSourceTable, transformPreSourceTable
+    description: |
+      Per-detector tasks that can be run together
+  step2a:
+    subset:
+      - consolidateVisitSummary
+      # TO DO: configure and add consolidatePreSourceTable
+  nightlyRollup:
+    # can be run at end of night or on a timer
+    # Need to either clobber OR use a chained collection of timestamped runs
+    subset:
+      - makePreliminaryCcdVisitTable
+      - makePreliminaryVisitTable
+      - preliminaryVisitCoverageAnalysis
+    description: |
+      Global-level tasks
diff --git a/pipelines/_ingredients/DRP-full.yaml b/pipelines/_ingredients/DRP-full.yaml
@@ -51,6 +51,16 @@ tasks:
     class: lsst.pipe.tasks.multiBand.MeasureMergedCoaddSourcesTask
     config:
       connections.sourceTableHandles: preSourceTable_visit
+  makePreliminaryCcdVisitTable:
+    class: lsst.pipe.tasks.postprocess.MakeCcdVisitTableTask
+    config:
+      connections.visitSummaryRefs: visitSummary
+      connections.outputCatalog: preCcdVisitTable
+  makePreliminaryVisitTable:
+    class: lsst.pipe.tasks.postprocess.MakeVisitTableTask
+    config:
+      connections.visitSummaries: visitSummary
+      connections.outputCatalog: preVisitTable
 subsets:
   # This pipeline file only defines subsets that have a consistent or
   # near-consistent definition as a set of tasks, for use by downstream

diff --git a/pipelines/_ingredients/LSSTComCamSim/DRP.yaml b/pipelines/_ingredients/LSSTComCamSim/DRP.yaml
@@ -0,0 +1,116 @@
+description: DRP Pipeline for Simulated ComCam Ops Rehearsal 3
+instrument: lsst.obs.lsst.LsstComCamSim
+imports:
+  - $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml
+  - $ANALYSIS_TOOLS_DIR/pipelines/coaddColumnValidate.yaml
+  - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityCore.yaml
+  - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityExtended.yaml
+  - $ANALYSIS_TOOLS_DIR/pipelines/diaTractQualityCore.yaml
+  - $ANALYSIS_TOOLS_DIR/pipelines/matchedVisitQualityCore.yaml
+  - $ANALYSIS_TOOLS_DIR/pipelines/visitQualityCore.yaml
+tasks:
+  fgcmFitCycle:
+    class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask
+    config:
+      doMultipleCycles: true
+      multipleCyclesFinalCycleNumber: 4
+  fgcmOutputProducts:
+    class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask
+    config:
+      connections.cycleNumber: 4
+  selectGoodSeeingVisits:
+    class: lsst.pipe.tasks.selectImages.BestSeeingQuantileSelectVisitsTask
+    config:
+      connections.goodVisits: goodSeeingVisits
+      nVisitsMin: 12
+      # For simulation 2 there are 60 visits per region
+      # between 60399.362 and 60402.419.
+      minMJD: 60399.362
+      maxMJD: 60402.419
+  updateVisitSummary:
+    class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask
+    config:
+      # TODO: Evaluate GBDES, SkyCorr, FGCM after initial DRP
+      wcs_provider: "tract"
+      photo_calib_provider: "global"
+      background_provider: "input_summary"
+  analyzeMatchedPreVisitCore:
+    class: lsst.analysis.tools.tasks.AssociatedSourcesTractAnalysisTask
+    config:
+      connections.sourceCatalogs: preSourceTable_visit
+      connections.outputName: matchedPreVisitCore
+      # The rest should be the same as configs in matchedVisitQualityCore.yaml
+      atools.stellarPhotometricRepeatability: StellarPhotometricRepeatability
+      atools.stellarPhotometricResiduals: StellarPhotometricResidualsFocalPlane
+      atools.stellarAstrometricResidualsRA: StellarAstrometricResidualsRAFocalPlanePlot
+      atools.stellarAstrometricResidualsDec: StellarAstrometricResidualsDecFocalPlanePlot
+      atools.stellarAstrometricResidualStdDevRA: StellarAstrometricResidualStdDevRAFocalPlanePlot
+      atools.stellarAstrometricResidualStdDevDec: StellarAstrometricResidualStdDevDecFocalPlanePlot
+      atools.stellarAstrometricRepeatability1: AstrometricRelativeRepeatability
+      atools.stellarAstrometricRepeatability1.xValue: 1
+      atools.stellarAstrometricRepeatability1.process.calculateActions.rms.annulus: 5
+      atools.stellarAstrometricRepeatability2: AstrometricRelativeRepeatability
+      atools.stellarAstrometricRepeatability2.xValue: 2
+      atools.stellarAstrometricRepeatability2.process.calculateActions.rms.annulus: 20
+      atools.stellarAstrometricRepeatability3: AstrometricRelativeRepeatability
+      atools.stellarAstrometricRepeatability3.xValue: 3
+      atools.stellarAstrometricRepeatability3.process.calculateActions.rms.annulus: 200
+      atools.stellarAstrometricRepeatability3.process.calculateActions.rms.threshAD: 30
+      python: |
+        from lsst.analysis.tools.atools import *
+  analyzeObjectTableCore:
+    class: lsst.analysis.tools.tasks.ObjectTableTractAnalysisTask
+    config:
+      python: |
+        # no z-band in comCamSim
+        del config.atools.yPerpPSF
+        del config.atools.yPerpCModel
+  analyzePreSourceTableCore:
+    class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask
+    config:
+      connections.outputName: preSourceTableCore
+      atools.skyFluxVisitStatisticMetric: SkyFluxStatisticMetric
+      atools.skyFluxVisitStatisticMetric.applyContext: VisitContext
+      atools.skySourceSky: SkySourceSkyPlot
+      atools.skySourceFlux: SkySourceHistPlot
+      python: |
+        from lsst.analysis.tools.atools import *
+        from lsst.analysis.tools.contexts import *
+  catalogMatchPreVisit:
+    class: lsst.analysis.tools.tasks.astrometricCatalogMatch.AstrometricCatalogMatchVisitTask
+    config:
+      connections.catalog: "preSourceTable_visit"
+      connections.targetCatalog: "preSourceTable_visit"
+  astrometricRefCatPreSourceVisit:
+    class: lsst.analysis.tools.tasks.refCatSourceAnalysis.RefCatSourceAnalysisTask
+    config:
+      # Only run metrics for analyzing the preSources:
+      connections.data: preSourceTable_visit_uw_stars_20240228_match
+      atools.astromDiffMetrics: TargetRefCatDeltaMetrics
+      atools.astromDiffMetrics.applyContext: VisitContext
+      python: |
+        from lsst.analysis.tools.atools import *
+        from lsst.analysis.tools.contexts import *
+  # These don't actually work now:
+  photometricMatchPreVisit:
+    class: lsst.analysis.tools.tasks.photometricCatalogMatch.PhotometricCatalogMatchVisitTask
+    config:
+      extraColumns: ["x", "y", "ap09Flux", "ap09FluxErr"]
+      connections.refCatalog: uw_stars_20240228
+      connections.catalog: preSourceTable_visit
+      connections.targetCatalog: preSourceTable_visit
+      connections.matchedCatalog: preSourceTable_visit_uw_stars_20240228_photoMatch
+  photometricRefCatPreSourceVisit:
+    class: lsst.analysis.tools.tasks.refCatSourcePhotometricAnalysis.RefCatSourcePhotometricAnalysisTask
+    config:
+      # Only run metrics for analyzing the preSources:
+      connections.data: preSourceTable_visit_uw_stars_20240228_photoMatch
+      python: from lsst.analysis.tools.atools import *
+  preliminaryVisitCoverageAnalysis:
+    class: lsst.analysis.tools.tasks.CcdVisitTableAnalysisTask
+    config:
+      connections.data: preCcdVisitTable
+      atools.raDecCoveragePlot: RaDecMultiVisitCoveragePlot
+      atools.raDecCoveragePlot.produce.plot.showExtremeOutliers: false
+      atools.focalPlaneCoveragePlot: FocalPlaneMultiVisitCoveragePlot
+      python: from lsst.analysis.tools.atools import *