DM-45192: Reconfigure RA for real ComCam

pipelines/LSSTComCam/DRP.yaml

@@ -0,0 +1,181 @@
+description: DRP Pipeline for LSSTComCam
+instrument: lsst.obs.lsst.LsstComCam
+imports:
+  - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCam/DRP.yaml
+
+subsets:
+  step1:
+    subset:
+      - isr
+      - characterizeImage
+      - calibrate
+      - writePreSourceTable
+      - transformPreSourceTable
+    description: |
+      Per-detector tasks that can be run together to start the DRP pipeline.
+
+      These should never be run with 'tract' or 'patch' as part of the data ID
+      expression if any later steps will also be run, because downstream steps
+      require full visits and 'tract' and 'patch' constraints will always
+      select partial visits that overlap that region.
+  step2a:
+    subset:
+      - analyzeMatchedPreVisitCore
+      - consolidatePreSourceTable
+      - consolidateVisitSummary
+    description: |
+      Visit-level tasks
+      Allowed data query constraints: visit
+
+      Tasks aggregate all detectors for a given visit and perform
+      full focal plane background estimation.
+
+      TODO: Evaluate SkyCorr after intial DRP
+  step2b:
+    subset:
+      - isolatedStarAssociation
+      - gbdesAstrometricFit
+    description: |
+      Tract-level tasks
+      Allowed data query constraints: tract
+
+      GbdesAstrometricFit and isolatedStarAssociation both use PreSources,
+      generated by consolidatePreSourceTable, for all visits that overlap a
+      tract.
+      gbdesAstrometricFit produces solutions per-tract, per-visit
+      isolatedStarAssociation produces solutions per-tract.
+      TODO: Evaluate GBDES after first DRP
+  step2c:
+    subset:
+      - fgcmBuildFromIsolatedStars
+      - fgcmFitCycle
+      - fgcmOutputProducts
+  step2d:
+    subset:
+      - finalizeCharacterization
+      - writeRecalibratedSourceTable
+      - transformSourceTable
+      - consolidateSourceTable
+      - updateVisitSummary
+    description: |
+      Visit-level tasks.
+      Allowed data query constraints: visit
+
+      writeRecalibratedSourceTable, transformSourceTable run per-detector
+      consolidateSourceTable produces one data product per visit.
+      finalizeCharacterization will eventually model full focal plane PSFs.
+  step2e:
+    subset:
+      - makeCcdVisitTable
+      - makeVisitTable
+    description: |
+      Global-level tasks that must not be run with any data query constraints
+      Can be run anytime after subset step2d.
+
+      Allowed data query constraints: instrument
+
+      Tasks generate one data product per collection.
+      make[Ccd]VisitTable produces per-collection summary of the Visits
+      and CcdVisits.
+  step3:
+    subset:
+      - makeWarp
+      - selectDeepCoaddVisits
+      - assembleCoadd
+      - detection
+      - mergeDetections
+      - deblend
+      - measure
+      - mergeMeasurements
+      - forcedPhotCoadd
+      - transformObjectTable
+      - writeObjectTable
+      - consolidateObjectTable
+      - healSparsePropertyMaps
+      - selectGoodSeeingVisits
+      - templateGen
+      # analysis_tools tasks
+      - analyzeMatchedVisitCore
+      - analyzeObjectTableCore
+      - catalogMatchTract
+      - photometricCatalogMatch
+      - photometricRefCatObjectTract
+      - plotPropertyMapTract
+      - refCatObjectTract
+      - validateObjectTableCore
+    description: |
+      Tasks that can be run together, but only after the 'step1' and 'step2d'
+      subsets.
+
+      These should be run with explicit 'tract' constraints essentially all the
+      time, because otherwise quanta will be created for jobs with only partial
+      visit coverage.
+
+      This subset is considered a workaround for missing middleware and task
+      functionality.  It may be removed in the future.
+  step4:
+    subset:
+      - forcedPhotCcd
+      - forcedPhotDiffim
+      - getTemplate
+      - subtractImages
+      - detectAndMeasureDiaSources
+      - transformDiaSourceCat
+      - writeForcedSourceTable
+    description: |
+      Tasks that can be run together, but only after the 'step1', 'step2d' and
+      'step3' subsets
+
+      These detector-level tasks should not be run with 'tract' or 'patch' as
+      part of the data ID expression if all reference catalogs or diffIm
+      templates that cover these detector-level quanta are desired.
+  step5:
+    subset:
+      - drpAssociation
+      - drpDiaCalculation
+      - forcedPhotCcdOnDiaObjects
+      - forcedPhotDiffOnDiaObjects
+      - transformForcedSourceTable
+      - consolidateAssocDiaSourceTable
+      - consolidateFullDiaObjectTable
+      - writeForcedSourceOnDiaObjectTable
+      - transformForcedSourceOnDiaObjectTable
+      - analyzeDiaSourceTableTract
+    description: |
+      Tasks that can be run together, but only after the 'step1', 'step2',
+      'step3', and 'step4' subsets
+
+      This step includes patch-level aggregation Tasks. These should be run
+      with explicit 'tract' constraints in the data query, otherwise quanta
+      will be created for jobs with only partial visit coverage.
+  step6:
+    subset:
+      - consolidateDiaSourceTable
+    description: |
+      Tasks that can be run together, but only after the 'step1', 'step2',
+      'step3', and 'step4' subsets
+
+      This step includes visit-level aggregation tasks. Running without tract
+      or patch in the data query is recommended, otherwise the outputs of
+      consolidateDiaSourceTable will not contain complete visits.
+
+      This subset is separate from step4 to signal to operators to pause to
+      assess unexpected image differencing failures before these aggregation
+      steps. Otherwise, if run in the same quantum graph, aggregated data
+      products (e.g. diaObjects) would not be created if one or more of the
+      expected inputs is missing.
+  step7:
+    subset:
+      - consolidateHealSparsePropertyMaps
+      - analyzeObjectTableSurveyCore
+      - makeMetricTableMatchedVisitCore
+      - makeMetricTableObjectTableCore
+      - matchedVisitCoreWholeSkyPlot
+      - objectTableCoreWholeSkyPlot
+    description: |
+      Tasks that should be run as the final step that require global inputs,
+      and can be run after the 'step3' subset.
+
+      This step has global aggregation tasks to run over all visits, detectors,
+      tracts, etc.  This step should be run only with the instrument constraint
+      in the data query.

pipelines/LSSTComCam/nightly-validation.yaml

@@ -0,0 +1,148 @@
+description: |
+  DRP-flavored pipeline to support validation
+  during commissioning, and will help inform
+  what changes to make the following night. Detector and Visit
+  level tasks can be run in real time by e.g. Rapid Analysis.
+  The rest of the pipeline is expected to be run as part of the
+  10am processing.
+instrument: lsst.obs.lsst.LsstComCam
+imports:
+  - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCam/DRP.yaml
+tasks:
+  analyzePreSourceTableCore:
+    class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask
+    config:
+      connections.data: preSourceTable_visit
+      connections.inputName: preSourceTable_visit
+      connections.outputName: preSourceTableCore
+  calibrate:
+    class: lsst.pipe.tasks.calibrate.CalibrateTask
+    config:
+      python: |
+        from lsst.analysis.tools.tasks import CalexpSummaryAnalysisTask
+        config.createSummaryMetrics.retarget(CalexpSummaryAnalysisTask)
+        from lsst.analysis.tools.atools import CalexpSummaryMetrics
+        config.createSummaryMetrics.atools.calexpSummaryMetrics = CalexpSummaryMetrics
+      doCreateSummaryMetrics: true
+  updateVisitSummary:
+    class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask
+    config:
+      # No global calibration during nightly validation
+      wcs_provider: "input_summary"
+      photo_calib_provider: "input_summary"
+      background_provider: "input_summary"
+subsets:
+  # These 3 steps can be run in real time by Rapid Analysis Framework at USDF
+  step1:
+    subset:
+      - isr
+      - characterizeImage
+      - calibrate
+      - writePreSourceTable
+      - transformPreSourceTable
+    description: |
+      Detector level tasks. TO DO: requests include synthetic source injection,
+      subtractImages, detectAndMeasureDiaSources.
+  step2a:
+    subset:
+      - consolidatePreSourceTable
+      - consolidateVisitSummary
+      - analyzePreSourceTableCore
+      - catalogMatchPreVisit
+      - astrometricRefCatPreSourceVisit
+      # TO DO: Add photometricMatchPreVisit, photometricRefCatPreSourceVisit
+    description: Visit-level tasks
+  nightlyRollup:
+    subset:
+      - makePreliminaryCcdVisitTable
+      - makePreliminaryVisitTable
+      - preliminaryVisitCoverageAnalysis
+    description: |
+      Global tasks that can be run at end of night or multiple times during
+      the night, to get a summary of observations taken.
+  # The following N steps should be run once at the end of the night
+  # A complete pipeline (without DIA): step2b,step2d,step2e,step3,step7
+  step2b:
+    subset:
+      - isolatedStarAssociation
+    description: |
+      Tract-level tasks
+      Allowed data query constraints: tract
+
+      isolatedStarAssociation uses PreSources,
+      generated by consolidatePreSourceTable, for all visits that overlap a
+      tract.
+  step2d:
+    subset:
+      - finalizeCharacterization
+      - writeRecalibratedSourceTable
+      - transformSourceTable
+      - consolidateSourceTable
+      - updateVisitSummary
+    description: |
+      Visit-level tasks.
+      Allowed data query constraints: visit
+
+      writeRecalibratedSourceTable, transformSourceTable run per-detector
+      consolidateSourceTable produces one data product per visit.
+      finalizeCharacterization will eventually model full focal plane PSFs.
+  step2e:
+    subset:
+      - makeCcdVisitTable
+      - makeVisitTable
+    description: |
+      Global-level tasks that must not be run with any data query constraints
+      Can be run anytime after subset step2d.
+
+      Allowed data query constraints: instrument
+
+      Tasks generate one data product per collection.
+      make[Ccd]VisitTable produces per-collection summary of the Visits
+      and CcdVisits.
+  step3:
+    subset:
+      - makeWarp
+      - selectDeepCoaddVisits
+      - assembleCoadd
+      - detection
+      - mergeDetections
+      - deblend
+      - measure
+      - mergeMeasurements
+      - forcedPhotCoadd
+      - transformObjectTable
+      - writeObjectTable
+      - consolidateObjectTable
+      - healSparsePropertyMaps
+      # analysis_tools tasks
+      - analyzeMatchedVisitCore
+      - analyzeObjectTableCore
+      - analyzeObjectTableExtended
+      - catalogMatchTract
+      - photometricCatalogMatch
+      - photometricRefCatObjectTract
+      - plotPropertyMapTract
+      - refCatObjectTract
+      - validateObjectTableCore
+    description: |
+      This overrides the default DRP pipeline by not making nightly templates.
+      We may want to make daily templates in the future.
+
+      Patch and Tract-level tasks that make coadds and object tables that
+      can be run together, but only after the 'step1' and 'step2d'
+      subsets.
+  step7:
+    subset:
+      - consolidateHealSparsePropertyMaps
+      - analyzeObjectTableSurveyCore
+      - makeMetricTableMatchedVisitCore
+      - makeMetricTableObjectTableCore
+      - matchedVisitCoreWholeSkyPlot
+      - objectTableCoreWholeSkyPlot
+    description: |
+      Tasks that should be run as the final step that require global inputs,
+      and can be run after the 'step3' subset.
+
+      This step has global aggregation tasks to run over all visits, detectors,
+      tracts, etc.  This step should be run only with the instrument constraint
+      in the data query.