Merge pull request #214 from lsst/tickets/DM-40388

DM-40388: Switch AP pipelines to use new CalibrateImageTask

pipelines/DECam/ApVerify.yaml

@@ -15,9 +15,3 @@ tasks:
     class: lsst.ap.association.DiaPipelineTask
     config:
       doPackageAlerts: True
-contracts:
-  # Must re-declare contracts that cross apPipe and metrics boundary, as
-  # these were removed on import.
-  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
-  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
-      fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name

pipelines/DECam/ApVerifyCalibrateImage.yaml → pipelines/DECam/ApVerifyCalibrate.yaml

@@ -1,18 +1,23 @@
-# TODO DM-40388: This file will replace ApVerify.yaml, once the new pipeline is vetted.
 # Verification pipeline specialized for DECam.
 # This concatenates various lsst.verify metrics to an AP pipeline
 
 description: Fully instrumented AP pipeline specialized for DECam
 imports:
-  - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyCalibrateImage.yaml
+  - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyCalibrate.yaml
     # Include all metrics from standard pipeline. It's not practical to create
     # a metrics subset because it would require constant micromanagement.
     exclude:
       - apPipe
-  - location: $AP_PIPE_DIR/pipelines/DECam/ApPipeCalibrateImage.yaml
+  - location: $AP_PIPE_DIR/pipelines/DECam/ApPipeCalibrate.yaml
 tasks:
   # ApVerify override removed by excluding apPipe.
   diaPipe:
     class: lsst.ap.association.DiaPipelineTask
     config:
       doPackageAlerts: True
+contracts:
+  # Must re-declare contracts that cross apPipe and metrics boundary, as
+  # these were removed on import.
+  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
+  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
+      fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name

pipelines/HSC/ApVerifyCalibrateImage.yaml → pipelines/HSC/ApVerifyCalibrate.yaml

@@ -1,15 +1,14 @@
-# TODO DM-40388: This file will replace ApVerify.yaml, once the new pipeline is vetted.
 # Verification pipeline specialized for HSC.
 # This concatenates various lsst.verify metrics to an AP pipeline
 
 description: Fully instrumented AP pipeline specialized for HSC
 imports:
-  - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyCalibrateImage.yaml
+  - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyCalibrate.yaml
     # Include all metrics from standard pipeline. It's not practical to create
     # a metrics subset because it would require constant micromanagement.
     exclude:
       - apPipe
-  - location: $AP_PIPE_DIR/pipelines/HSC/ApPipeCalibrateImage.yaml
+  - location: $AP_PIPE_DIR/pipelines/HSC/ApPipeCalibrate.yaml
 tasks:
   # ApVerify override removed by excluding apPipe.
   diaPipe:

pipelines/LSSTCam-imSim/ApVerify.yaml

@@ -15,9 +15,3 @@ tasks:
     class: lsst.ap.association.DiaPipelineTask
     config:
       doPackageAlerts: True
-contracts:
-  # Must re-declare contracts that cross apPipe and metrics boundary, as
-  # these were removed on import.
-  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
-  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
-      fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name

pipelines/LSSTCam-imSim/ApVerifyCalibrateImage.yaml → pipelines/LSSTCam-imSim/ApVerifyCalibrate.yaml

@@ -1,18 +1,23 @@
-# TODO DM-40388: This file will replace ApVerify.yaml, once the new pipeline is vetted.
 # Verification pipeline specialized for the DC2 ImSim simulation.
 # This concatenates various lsst.verify metrics to an AP pipeline
 
 description: Fully instrumented AP pipeline specialized for LSSTCam-imSim
 imports:
-  - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyCalibrateImage.yaml
+  - location: $AP_VERIFY_DIR/pipelines/_ingredients/ApVerifyCalibrate.yaml
     # Include all metrics from standard pipeline. It's not practical to create
     # a metrics subset because it would require constant micromanagement.
     exclude:
       - apPipe
-  - location: $AP_PIPE_DIR/pipelines/LSSTCam-imSim/ApPipeCalibrateImage.yaml
+  - location: $AP_PIPE_DIR/pipelines/LSSTCam-imSim/ApPipeCalibrate.yaml
 tasks:
   # ApVerify override removed by excluding apPipe.
   diaPipe:
     class: lsst.ap.association.DiaPipelineTask
     config:
       doPackageAlerts: True
+contracts:
+  # Must re-declare contracts that cross apPipe and metrics boundary, as
+  # these were removed on import.
+  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
+  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
+      fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name

pipelines/_ingredients/ApVerify.yaml

@@ -1,14 +1,14 @@
-# Gen 3 pipeline for ap_verify
 # This concatenates various lsst.verify metrics to an AP pipeline
 
-description: Fully instrumented AP pipeline
+description: Fully instrumented AP pipeline.
 imports:
   - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipe.yaml
-  # Metrics that should be run without fakes
   - location: $AP_VERIFY_DIR/pipelines/_ingredients/MetricsRuntime.yaml
     exclude:
-      - timing_calibrateImage
-      - cputiming_calibrateImage
+      - timing_calibrate
+      - timing_characterizeImage
+      - cputiming_calibrate
+      - cputiming_characterizeImage
   - location: $AP_VERIFY_DIR/pipelines/_ingredients/MetricsMisc.yaml
   - location: $AP_VERIFY_DIR/pipelines/_ingredients/Conversions.yaml
 tasks:
@@ -18,8 +18,3 @@ tasks:
       # TODO: needed for "providing bulk sample alerts to brokers"; remove once
       # we have an alternative.
       doPackageAlerts: True
-contracts:
-  # Metric inputs must match pipeline outputs
-  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
-  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
-      fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name

pipelines/_ingredients/ApVerifyCalibrateImage.yaml → pipelines/_ingredients/ApVerifyCalibrate.yaml

@@ -1,20 +1,25 @@
+# Gen 3 pipeline for ap_verify
 # This concatenates various lsst.verify metrics to an AP pipeline
 
-description: Fully instrumented AP pipeline.
+description: Fully instrumented AP pipeline
 imports:
-  - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeCalibrateImage.yaml
+  - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeCalibrate.yaml
+  # Metrics that should be run without fakes
   - location: $AP_VERIFY_DIR/pipelines/_ingredients/MetricsRuntime.yaml
     exclude:
-      - timing_calibrate
-      - timing_characterizeImage
-      - cputiming_calibrate
-      - cputiming_characterizeImage
-  - location: $AP_VERIFY_DIR/pipelines/_ingredients/MetricsMiscCalibrateImage.yaml
-  - location: $AP_VERIFY_DIR/pipelines/_ingredients/ConversionsCalibrateImage.yaml
+      - timing_calibrateImage
+      - cputiming_calibrateImage
+  - location: $AP_VERIFY_DIR/pipelines/_ingredients/MetricsMiscCalibrate.yaml
+  - location: $AP_VERIFY_DIR/pipelines/_ingredients/ConversionsCalibrate.yaml
 tasks:
   diaPipe:
     class: lsst.ap.association.DiaPipelineTask
     config:
       # TODO: needed for "providing bulk sample alerts to brokers"; remove once
       # we have an alternative.
       doPackageAlerts: True
+contracts:
+  # Metric inputs must match pipeline outputs
+  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
+  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
+      fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name

pipelines/_ingredients/Conversions.yaml

@@ -2,20 +2,16 @@
 #
 # This pipeline depends on an external pipeline for the coaddName and fakesType
 # pipeline parameters.
+description: Type conversion tasks customized for AP pipeline, with CalibrateImage.
 
-description: Type conversion tasks customized for AP pipeline
 tasks:
-  # Conversion of src [afw.table] to source [Parquet]
-  writeSourceTable:
-    class: lsst.pipe.tasks.postprocess.WriteSourceTableTask
-  # TODO: TransformSourceTableTask can't be run until we create a functor
-  # config that doesn't depend on shapeHSM.
-  # Merging of source [detector-level] to sourceTable_visit [visit-level]
+  # Merging of initial_stars_detector [detector-level] to initial_stars [visit-level]
   consolidateSourceTable:
     class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask
     config:
-      # Skip DPDD-ified sourceTable and just use source directly
-      connections.inputCatalogs: "{catalogType}source"
+      # This is output directly by CalibrateImageTask.
+      connections.inputCatalogs: initial_stars_detector
+      connections.outputCatalog: initial_stars
 
   # Merging of *Diff_diaSrcTable [detector-level Parquet] to diaSourceTable [visit-level]
   consolidateDiaSourceTable:
@@ -26,10 +22,10 @@ tasks:
       connections.inputCatalogs: "{catalogType}Diff_diaSrcTable"
       connections.outputCatalog: diaSourceTable
 
-  # Creation of visitSummary
   consolidateVisitSummary:
     class: lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask
-  # Conversion of visitSummary [visit-level afw.table] to visitTable [instrument-level Parquet]
+    config:
+      connections.calexp: initial_pvi
   makeVisitTable:
     class: lsst.pipe.tasks.postprocess.MakeVisitTableTask
     config:

pipelines/_ingredients/ConversionsCalibrate.yaml

@@ -0,0 +1,41 @@
+# Add-on pipeline designed to be added to ApVerify or ApPipe for QA purposes.
+#
+# This pipeline depends on an external pipeline for the coaddName and fakesType
+# pipeline parameters.
+
+description: Type conversion tasks customized for AP pipeline
+tasks:
+  # Conversion of src [afw.table] to source [Parquet]
+  writeSourceTable:
+    class: lsst.pipe.tasks.postprocess.WriteSourceTableTask
+  # TODO: TransformSourceTableTask can't be run until we create a functor
+  # config that doesn't depend on shapeHSM.
+  # Merging of source [detector-level] to sourceTable_visit [visit-level]
+  consolidateSourceTable:
+    class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask
+    config:
+      # Skip DPDD-ified sourceTable and just use source directly
+      connections.inputCatalogs: "{catalogType}source"
+
+  # Merging of *Diff_diaSrcTable [detector-level Parquet] to diaSourceTable [visit-level]
+  consolidateDiaSourceTable:
+    class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask
+    config:
+      # Task doesn't support coaddName, so coopt catalogType instead.
+      connections.catalogType: parameters.coaddName
+      connections.inputCatalogs: "{catalogType}Diff_diaSrcTable"
+      connections.outputCatalog: diaSourceTable
+
+  # Creation of visitSummary
+  consolidateVisitSummary:
+    class: lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask
+  # Conversion of visitSummary [visit-level afw.table] to visitTable [instrument-level Parquet]
+  makeVisitTable:
+    class: lsst.pipe.tasks.postprocess.MakeVisitTableTask
+    config:
+      connections.visitSummaries: "{calexpType}visitSummary"
+  # Conversion of visitSummary [visit-level afw.table] to ccdVisitTable [instrument-level Parquet]
+  makeCcdVisitTable:
+    class: lsst.pipe.tasks.postprocess.MakeCcdVisitTableTask
+    config:
+      connections.visitSummaryRefs: "{calexpType}visitSummary"

pipelines/_ingredients/MetricsMisc.yaml

@@ -1,8 +1,7 @@
 # Miscellaneous metrics for Alert Production
 # In the future, these might be placed in task-specific pipelines (for debugging)
 # or grouped by their datasets (to optimize expensive Butler reads)
-
-description: Miscelaneous AP Pipeline metrics
+description: Miscelaneous AP Pipeline metrics.
 tasks:
   numNewDiaObjects:
     class: lsst.ap.association.metrics.NumberNewDiaObjectsMetricTask
@@ -26,13 +25,3 @@ tasks:
       connections.labelName: diaPipe
   totalUnassociatedDiaObjects:
     class: lsst.ap.association.metrics.TotalUnassociatedDiaObjectsMetricTask
-  numSciSources:
-    class: lsst.ip.diffim.metrics.NumberSciSourcesMetricTask
-  fracDiaSourcesToSciSources:
-    class: lsst.ip.diffim.metrics.FractionDiaSourcesToSciSourcesMetricTask
-    config:
-      connections.coaddName: parameters.coaddName
-  numDeblendedSciSources:
-    class: lsst.pipe.tasks.metrics.NumberDeblendedSourcesMetricTask
-  numDeblendChildSciSources:
-    class: lsst.pipe.tasks.metrics.NumberDeblendChildSourcesMetricTask

pipelines/_ingredients/MetricsMiscCalibrate.yaml

@@ -0,0 +1,38 @@
+# Miscellaneous metrics for Alert Production
+# In the future, these might be placed in task-specific pipelines (for debugging)
+# or grouped by their datasets (to optimize expensive Butler reads)
+
+description: Miscelaneous AP Pipeline metrics
+tasks:
+  numNewDiaObjects:
+    class: lsst.ap.association.metrics.NumberNewDiaObjectsMetricTask
+    config:
+      connections.labelName: diaPipe  # partial name of metadata dataset
+  numUnassociatedDiaObjects:
+    class: lsst.ap.association.metrics.NumberUnassociatedDiaObjectsMetricTask
+    config:
+      connections.labelName: diaPipe
+  fracUpdatedDiaObjects:
+    class: lsst.ap.association.metrics.FractionUpdatedDiaObjectsMetricTask
+    config:
+      connections.labelName: diaPipe
+  numTotalSolarSystemObjects:
+    class: lsst.ap.association.metrics.NumberSolarSystemObjectsMetricTask
+    config:
+      connections.labelName: diaPipe
+  numAssociatedSsObjects:
+    class: lsst.ap.association.metrics.NumberAssociatedSolarSystemObjectsMetricTask
+    config:
+      connections.labelName: diaPipe
+  totalUnassociatedDiaObjects:
+    class: lsst.ap.association.metrics.TotalUnassociatedDiaObjectsMetricTask
+  numSciSources:
+    class: lsst.ip.diffim.metrics.NumberSciSourcesMetricTask
+  fracDiaSourcesToSciSources:
+    class: lsst.ip.diffim.metrics.FractionDiaSourcesToSciSourcesMetricTask
+    config:
+      connections.coaddName: parameters.coaddName
+  numDeblendedSciSources:
+    class: lsst.pipe.tasks.metrics.NumberDeblendedSourcesMetricTask
+  numDeblendChildSciSources:
+    class: lsst.pipe.tasks.metrics.NumberDeblendChildSourcesMetricTask

pipelines/_ingredients/MetricsRuntime.yaml

@@ -10,6 +10,7 @@ tasks:
       connections.labelName: isr   # partial name of metadata dataset
       metadataDimensions: [instrument, exposure, detector]  # TimingMetricTask assumes visit
       target: isr.run              # method name in metadata. Usually matches label for top-level tasks
+  # TODO XXXX: These two can be removed once the pipeline that uses characterize/calibrate is gone.
   timing_characterizeImage:
     class: lsst.verify.tasks.commonMetrics.TimingMetricTask
     config:
@@ -24,6 +25,7 @@ tasks:
       connections.metric: CalibrateTime
       connections.labelName: calibrate
       target: calibrate.run
+  ###
   timing_calibrateImage:
     class: lsst.verify.tasks.commonMetrics.TimingMetricTask
     config:
@@ -264,4 +266,4 @@ tasks:
       connections.package: ap_association
       connections.metric: PackageAlertsMemory
       connections.labelName: diaPipe
-      target: diaPipe:alertPackager.run
+      target: diaPipe:alertPackager.run