Merge branch 'main' into tickets/DM-44589

bps/resources/LSSTComCamSim/DRP-ops-rehearsals.yaml

@@ -0,0 +1,88 @@
+# This BPS configuration snippet adds requestMemory values for all DRP
+# and tasks for which the 99th percentile in memory usage exceeds 2GB from
+# the Cumulative DRP produced for Ops Rehearsal 3
+#
+# Use it by adding:
+#
+#   includeConfigs:
+#     - ${DRP_PIPE_DIR}/bps/resources/LSSTComCamSim/DRP-ops-rehearsals.yaml
+#
+# (with no outer indentation) to your BPS config file.
+#
+# NOTE: These numbers are from a campaign was run with clustering,
+# which means that the values are per cluster NOT per task.
+# Interpret these with caution, until we have a
+# a run done without clustering to pull from.
+pipetask:
+  analyzeMatchedVisitCore:
+    requestMemory: 184409
+  analyzeObjectTableCore:
+    requestMemory: 3353
+  analyzePreSourceTableCore:
+    requestMemory: 22321
+  assembleCoadd:
+    requestMemory: 32156
+  astrometricRefCatPreSourceVisit:
+    requestMemory: 21512
+  calibrate:
+    requestMemory: 7053
+  catalogMatchPreVisit:
+    requestMemory: 21512
+  characterizeImage:
+    requestMemory: 7053
+  consolidateAssocDiaSourceTable:
+    requestMemory: 2711
+  consolidateObjectTable:
+    requestMemory: 12953
+  consolidatePreSourceTable:
+    requestMemory: 21614
+  consolidateVisitSummary:
+    requestMemory: 21614
+  deblend:
+    requestMemory: 3763
+  detectAndMeasureDiaSources:
+    requestMemory: 9250
+  detection:
+    requestMemory: 31868
+  fgcmBuildFromIsolatedStars:
+    requestMemory: 4059
+  fgcmFitCycle:
+    requestMemory: 7867
+  finalizeCharacterization:
+    requestMemory: 7956
+  forcedPhotCcd:
+    requestMemory: 2310
+  forcedPhotDiffim:
+    requestMemory: 2310
+  gbdesAstrometricFit:
+    requestMemory: 3907
+  getTemplate:
+    requestMemory: 9167
+  healSparsePropertyMaps:
+    requestMemory: 32008
+  isolatedStarAssociation:
+    requestMemory: 5330
+  isr:
+    requestMemory: 7053
+  makeCcdVisitTable:
+    requestMemory: 21778
+  makeVisitTable:
+    requestMemory: 21637
+  plotPropertyMapTract:
+    requestMemory: 3682
+  selectDeepCoaddVisits:
+    requestMemory: 3291
+  subtractImages:
+    requestMemory: 9250
+  transformDiaSourceCat:
+    requestMemory: 9250
+  transformObjectTable:
+    requestMemory: 2448
+  transformPreSourceTable:
+    requestMemory: 7053
+  writeForcedSourceTable:
+    requestMemory: 2317
+  writeObjectTable:
+    requestMemory: 2448
+  writePreSourceTable:
+    requestMemory: 7053

pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml → pipelines/LSSTComCamSim/DRP.yaml

@@ -1,4 +1,4 @@
-description: DRP Pipeline for Simulated ComCam Ops Rehearsal 3
+description: DRP Pipeline for Simulated ComCam
 instrument: lsst.obs.lsst.LsstComCamSim
 imports:
   - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCamSim/DRP.yaml

pipelines/LSSTComCamSim/nightly-validation-ops-rehearsal-3.yaml → pipelines/LSSTComCamSim/nightly-validation.yaml

@@ -7,7 +7,7 @@ description: |
   10am processing.
 instrument: lsst.obs.lsst.LsstComCamSim
 imports:
-  - $DRP_PIPE_DIR/pipelines/LSSTComCamSim/DRP-ops-rehearsal-3.yaml
+  - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTComCamSim/DRP.yaml
 tasks:
   analyzePreSourceTableCore:
     class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask
@@ -24,6 +24,13 @@ tasks:
         from lsst.analysis.tools.atools import CalexpSummaryMetrics
         config.createSummaryMetrics.atools.calexpSummaryMetrics = CalexpSummaryMetrics
       doCreateSummaryMetrics: true
+  updateVisitSummary:
+    class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask
+    config:
+      # No global calibration during nightly validation
+      wcs_provider: "input_summary"
+      photo_calib_provider: "input_summary"
+      background_provider: "input_summary"
 subsets:
   # These 3 steps can be run in real time by Rapid Analysis Framework at USDF
   step1:
@@ -54,11 +61,44 @@ subsets:
       Global tasks that can be run at end of night or multiple times during
       the night, to get a summary of observations taken.
   # The following N steps should be run once at the end of the night
-  # Steps 2b,2c,2d,2e,4,5,6,7 are currently the same as DRP.
-  # This step3 here does not make templates. To run step4,step5,step6 you must
-  # provide a collection with templates as input, ideally the same one that
-  # was used for prompt processing.
-  # A complete pipeline without DIA is #step2b,step2c,step2d,step2e,step3,step7
+  # A complete pipeline (without DIA): step2b,step2d,step2e,step3,step7
+  step2b:
+    subset:
+      - isolatedStarAssociation
+    description: |
+      Tract-level tasks
+      Allowed data query constraints: tract
+
+      isolatedStarAssociation uses PreSources,
+      generated by consolidatePreSourceTable, for all visits that overlap a
+      tract.
+  step2d:
+    subset:
+      - finalizeCharacterization
+      - writeRecalibratedSourceTable
+      - transformSourceTable
+      - consolidateSourceTable
+      - updateVisitSummary
+    description: |
+      Visit-level tasks.
+      Allowed data query constraints: visit
+
+      writeRecalibratedSourceTable, transformSourceTable run per-detector
+      consolidateSourceTable produces one data product per visit.
+      finalizeCharacterization will eventually model full focal plane PSFs.
+  step2e:
+    subset:
+      - makeCcdVisitTable
+      - makeVisitTable
+    description: |
+      Global-level tasks that must not be run with any data query constraints
+      Can be run anytime after subset step2d.
+
+      Allowed data query constraints: instrument
+
+      Tasks generate one data product per collection.
+      make[Ccd]VisitTable produces per-collection summary of the Visits
+      and CcdVisits.
   step3:
     subset:
       - makeWarp
@@ -91,3 +131,14 @@ subsets:
       Patch and Tract-level tasks that make coadds and object tables that
       can be run together, but only after the 'step1' and 'step2d'
       subsets.
+  step7:
+    subset:
+      - consolidateHealSparsePropertyMaps
+      - analyzeObjectTableSurveyCore
+    description: |
+      Tasks that should be run as the final step that require global inputs,
+      and can be run after the 'step3' subset.
+
+      This step has global aggregation tasks to run over all visits, detectors,
+      tracts, etc.  This step should be run only with the instrument constraint
+      in the data query.

pipelines/LSSTComCamSim/quickLook-ops-rehearsal-3.yaml → pipelines/LSSTComCamSim/quickLook.yaml

@@ -17,7 +17,6 @@ tasks:
     config:
       doApCorr: false
       doDeblend: false
-      doMaskStreaks: false
   calibrate:
     class: lsst.pipe.tasks.calibrate.CalibrateTask
     config:

pipelines/_ingredients/DECam/DRP.yaml

@@ -57,6 +57,7 @@ tasks:
       # Legacy DECam config to write out warps, even if they are empty
       doWriteEmptyWarps: true
       python: |
+        config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize = 29
         config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss = [1.0, 2.0, 4.5]
         from lsst.pipe.tasks.selectImages import PsfWcsSelectImagesTask
         config.select.retarget(PsfWcsSelectImagesTask)

pipelines/_ingredients/DRP-minimal-calibration.yaml

@@ -252,3 +252,4 @@ contracts:
   - "'calib_psf_used' not in measure.propagateFlags.source_flags if makeWarp.useVisitSummaryPsf else True"
   - "'calib_psf_reserved' not in measure.propagateFlags.source_flags if makeWarp.useVisitSummaryPsf else True"
   - "makeWarp.warpAndPsfMatch.warp.cacheSize == assembleCoadd.coaddPsf.cacheSize"
+  - "makeWarp.matchingKernelSize == assembleCoadd.matchingKernelSize"  # TODO: Remove this in DM-44688.

pipelines/_ingredients/LSSTComCamSim/DRP.yaml

@@ -1,4 +1,4 @@
-description: DRP Pipeline for Simulated ComCam Ops Rehearsal 3
+description: DRP Pipeline for Simulated ComCam Ops Rehearsals
 instrument: lsst.obs.lsst.LsstComCamSim
 imports:
   - $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml
@@ -85,7 +85,7 @@ tasks:
     class: lsst.analysis.tools.tasks.refCatSourceAnalysis.RefCatSourceAnalysisTask
     config:
       # Only run metrics for analyzing the preSources:
-      connections.data: preSourceTable_visit_uw_stars_20240228_match
+      connections.data: preSourceTable_visit_uw_stars_20240524_match
       atools.astromDiffMetrics: TargetRefCatDeltaMetrics
       atools.astromDiffMetrics.applyContext: VisitContext
       python: |
@@ -96,15 +96,15 @@ tasks:
     class: lsst.analysis.tools.tasks.photometricCatalogMatch.PhotometricCatalogMatchVisitTask
     config:
       extraColumns: ["x", "y", "ap09Flux", "ap09FluxErr"]
-      connections.refCatalog: uw_stars_20240228
+      connections.refCatalog: uw_stars_20240524
       connections.catalog: preSourceTable_visit
       connections.targetCatalog: preSourceTable_visit
-      connections.matchedCatalog: preSourceTable_visit_uw_stars_20240228_photoMatch
+      connections.matchedCatalog: preSourceTable_visit_uw_stars_20240524_photoMatch
   photometricRefCatPreSourceVisit:
     class: lsst.analysis.tools.tasks.refCatSourcePhotometricAnalysis.RefCatSourcePhotometricAnalysisTask
     config:
       # Only run metrics for analyzing the preSources:
-      connections.data: preSourceTable_visit_uw_stars_20240228_photoMatch
+      connections.data: preSourceTable_visit_uw_stars_20240524_photoMatch
       python: from lsst.analysis.tools.atools import *
   preliminaryVisitCoverageAnalysis:
     class: lsst.analysis.tools.tasks.CcdVisitTableAnalysisTask

tests/test_pipelines.py

@@ -66,6 +66,15 @@
     "yBackground",
 }
 
+# LATISS common inputs, in addition to COMMON_INPUTS
+LATISS_INPUTS = {
+    "atlas_refcat2_20220201",
+    "defects",
+    "gaia_dr3_20230707",
+    "sky",
+    "transmission_atmosphere_fgcm",
+}
+
 # LSSTCam-imSim common inputs, in addition to COMMON_INPUTS
 LSSTCAM_IMSIM_INPUTS = {
     "bfk",
@@ -87,8 +96,6 @@
     "deepCoadd_forced_src",
     "deepCoadd_inputMap",
     "deepCoadd_meas",
-    "deepCoadd_measMatch",
-    "deepCoadd_measMatchFull",
     "deepCoadd_mergeDet",
     "deepCoadd_nImage",
     "deepCoadd_obj",
@@ -118,6 +125,8 @@
 HSC_OUTPUTS = {
     "calexp_skyCorr_visit_mosaic",
     "calexpBackground_skyCorr_visit_mosaic",
+    "deepCoadd_measMatch",
+    "deepCoadd_measMatchFull",
     "forced_src",
     "preSource",
     "preSourceTable",
@@ -126,8 +135,52 @@
     "srcMatchFull",
 }
 
+# LATISS common outputs, in addition to COMMON_OUTPUTS
+LATISS_OUTPUTS = {
+    "diaObjectTable_tract",
+    "diaSourceTable",
+    "diaSourceTable_tract",
+    "fgcm_Cycle0_FitParameters",
+    "fgcm_Cycle1_FitParameters",
+    "fgcm_Cycle2_FitParameters",
+    "fgcm_Cycle3_FitParameters",
+    "fgcm_Cycle4_FitParameters",
+    "fgcm_Cycle0_FlaggedStars",
+    "fgcm_Cycle1_FlaggedStars",
+    "fgcm_Cycle2_FlaggedStars",
+    "fgcm_Cycle3_FlaggedStars",
+    "fgcm_Cycle4_FlaggedStars",
+    "fgcm_reference_stars",
+    "fgcm_star_ids",
+    "fgcm_star_observations",
+    "forcedSourceOnDiaObjectTable",
+    "forcedSourceOnDiaObjectTable_tract",
+    "forcedSourceTable",
+    "forcedSourceTable_tract",
+    "forced_diff",
+    "forced_diff_diaObject",
+    "forced_src_diaObject",
+    "goodSeeingCoadd",
+    "goodSeeingCoadd_nImage",
+    "goodSeeingDiff_assocDiaSrcTable",
+    "goodSeeingDiff_diaObjTable",
+    "goodSeeingDiff_diaSrc",
+    "goodSeeingDiff_diaSrcTable",
+    "goodSeeingDiff_differenceExp",
+    "goodSeeingDiff_differenceTempExp",
+    "goodSeeingDiff_fullDiaObjTable",
+    "goodSeeingDiff_matchedExp",
+    "goodSeeingDiff_templateExp",
+    "goodSeeingVisits",
+    "mergedForcedSource",
+    "mergedForcedSourceOnDiaObject",
+    "transmission_atmosphere_fgcm",
+}
+
 # LSSTCam-imSim common outputs, in addition to COMMON_OUTPUTS
 LSSTCAM_IMSIM_OUTPUTS = {
+    "deepCoadd_measMatch",
+    "deepCoadd_measMatchFull",
     "diaObjectTable_tract",
     "diaSourceTable",
     "diaSourceTable_tract",
@@ -219,6 +272,8 @@ def test_decam_drp_merian(self):
             },
             expected_outputs=COMMON_OUTPUTS
             | {
+                "deepCoadd_measMatch",
+                "deepCoadd_measMatchFull",
                 "goodSeeingCoadd",
                 "goodSeeingCoadd_nImage",
                 "goodSeeingVisits",
@@ -438,6 +493,32 @@ def test_hsc_drp_rc2_subset(self):
         )
         tester.run(butler, self)
 
+    def test_latiss_drp(self):
+        butler = self.makeButler(writeable=True)
+        tester = PipelineStepTester(
+            os.path.join(PIPELINES_DIR, "LATISS", "DRP.yaml"),
+            [
+                "#step1",
+                "#step2a",
+                "#step2bcde",
+                "#step3a",
+                "#step3b",
+                "#step3c",
+                "#step4",
+                "#step5",
+                "#step6",
+            ],
+            [
+                ("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False),
+                ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False),
+                ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False),
+                ("atlas_refcat2_20220201", {"htm7"}, "SimpleCatalog", False),
+            ],
+            expected_inputs=COMMON_INPUTS | LATISS_INPUTS | {"fgcmLookUpTable"},
+            expected_outputs=COMMON_OUTPUTS | LATISS_OUTPUTS,
+        )
+        tester.run(butler, self)
+
     def test_lsstcam_imsim_drp_ci_imsim(self):
         butler = self.makeButler(writeable=True)
         tester = PipelineStepTester(