RCAL-965 Invoke converter from_tvac_raw to enable processing of TVAC/FPS data

changes/1596.dq_init.rst

@@ -0,0 +1 @@
+Invoke converter from_tvac_raw to enable processing of TVAC/FPS data

docs/roman/dq_init/description.rst

@@ -1,5 +1,13 @@
 Description
 ============
+The Data Quality (DQ) initialization step in the calibration pipeline performs
+two functions: Population of the DQ mask and the conversion of the science raw
+data, the "_uncal" file, to the ramp model by the majority of steps in the
+exposure level pipeline.
+
+Data Quality Initialization
+---------------------------
+
 The Data Quality (DQ) initialization step in the calibration pipeline
 populates the DQ mask for the input dataset. Flag values from the
 appropriate static mask ("MASK") reference file in CRDS are copied into the
@@ -23,3 +31,20 @@ Note that when applying the ``dq_init`` step to guide star data, the flags from
 file are propagated into the guide star dataset "dq" array, instead of the "pixeldq" array.
 The step identifies guide star data based on the following exposure type (exposure.type keyword attribute) values:
 WFI_WIM_ACQ, WFI_WIM_TRACK, WFI_WSM_ACQ1, WFI_WSM_ACQ2, WFI_WSM_TRACK.
+
+Conversion from Level 1 uncal files
+-----------------------------------
+
+The output of the initialization step is the RampModel. This is the form of the
+data used throughout most of the exposure pipeline steps. For the most part, the
+meta information between the input raw, or "uncal", model and the ramp model is
+complete.
+
+However, romancal supports processing a selection of files which use an
+outdated schema. It supports these with a bespoke method that converts the files
+to the new format when they are read in dq_init. This conversion does not do a
+detailed mapping between all of the new and old metadata, but instead
+opportunistically looks for fields with common names and assigns them. Other
+metadata with non-matching names is simply copied in place. This allows
+processing to proceed and preserves the original metadata, but the resulting
+files have duplicates of many entries.

pyproject.toml

@@ -23,8 +23,8 @@ dependencies = [
     "photutils >=1.13.0",
     "pyparsing >=2.4.7",
     "requests >=2.26",
-    "roman_datamodels>=0.23.0,<0.24.0",
-    #"roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git",
+    #"roman_datamodels>=0.23.0,<0.24.0",
+    "roman_datamodels @ git+https://github.com/stscieisenhamer/roman_datamodels.git@rcal-965-tvac",
     "scipy >=1.14.1",
     # "stcal>=1.10.0,<1.11.0",
     "stcal @ git+https://github.com/spacetelescope/stcal.git@main",

romancal/dq_init/dq_init_step.py

@@ -4,7 +4,7 @@
 from typing import TYPE_CHECKING
 
 import roman_datamodels as rdm
-from roman_datamodels.datamodels import RampModel
+from roman_datamodels.datamodels import RampModel, ScienceRawModel
 from roman_datamodels.dqflags import pixel
 
 from romancal.dq_init import dq_initialization
@@ -47,6 +47,10 @@
         """
         # Open datamodel
         input_model = rdm.open(input)
+        try:
+            input_model = ScienceRawModel.from_tvac_raw(input_model)
+        except ValueError:
+            pass
 
         # Convert to RampModel
         output_model = RampModel.from_science_raw(input_model)

romancal/regtest/test_elp_tvac.py

@@ -0,0 +1,123 @@
+"""Test that TVAC/FPS run in the pipeline"""
+
+import pytest
+import roman_datamodels as rdm
+from gwcs.wcstools import grid_from_bounding_box
+
+from romancal.pipeline.exposure_pipeline import ExposurePipeline
+
+from .regtestdata import compare_asdf
+
+# mark all tests in this module
+pytestmark = pytest.mark.bigdata
+
+
+@pytest.fixture(scope="module")
+def run_elp(rtdata_module):
+    rtdata = rtdata_module
+
+    # Get reference
+    rtdata.get_data("references/dark_ma510.asdf")
+
+    input_data = "TVAC2_NOMOPS_WFIFLA_20240419194120_WFI01_uncal.asdf"
+    rtdata.get_data(f"WFI/image/{input_data}")
+    rtdata.input = input_data
+
+    # Test Pipeline
+    output = "TVAC2_NOMOPS_WFIFLA_20240419194120_WFI01_cal.asdf"
+    rtdata.output = output
+    args = [
+        "roman_elp",
+        rtdata.input,
+        "--steps.dark_current.override_dark=dark_ma510.asdf",
+        "--steps.rampfit.override_dark=dark_ma510.asdf",
+    ]
+    ExposurePipeline.from_cmdline(args)
+
+    # get truth file
+    rtdata.get_truth(f"truth/WFI/image/{output}")
+    return rtdata
+
+
+@pytest.fixture(scope="module")
+def output_filename(run_elp):
+    return run_elp.output
+
+
+@pytest.fixture(scope="module")
+def output_model(output_filename):
+    with rdm.open(output_filename) as model:
+        yield model
+
+
+@pytest.fixture(scope="module")
+def truth_filename(run_elp):
+    return run_elp.truth
+
+
+def test_output_matches_truth(output_filename, truth_filename, ignore_asdf_paths):
+    diff = compare_asdf(output_filename, truth_filename, **ignore_asdf_paths)
+    assert diff.identical, diff.report()
+
+
+def test_output_is_image_model(output_model):
+    assert isinstance(output_model, rdm.datamodels.ImageModel)
+
+
+@pytest.mark.parametrize(
+    "step_name",
+    (
+        "assign_wcs",
+        "flat_field",
+        "dark",
+        "dq_init",
+        "linearity",
+        "ramp_fit",
+        "saturation",
+    ),
+)
+def test_steps_ran(output_model, step_name):
+    assert getattr(output_model.meta.cal_step, step_name) == "COMPLETE"
+
+
+def test_has_a_wcs(output_model):
+    assert output_model.meta.wcs is not None
+
+
+@pytest.mark.soctests
+def test_wcs_has_distortion_information(output_model):
+    assert "v2v3" in output_model.meta.wcs.available_frames
+
+
+def test_wcs_applies_distortion_correction(output_model):
+    # compare coordinates before and after distortion correction has been applied
+    # 1 - get new image array based on the model
+    x0, y0 = grid_from_bounding_box(output_model.meta.wcs.bounding_box)
+    # 2 - apply the distortion-corrected WCS solution to new image array
+    corrected_coords = output_model.meta.wcs(x0, y0)
+    # 3 - apply the transformation from 'v2v3' to 'world' without distortion correction
+    original_coords = output_model.meta.wcs.get_transform("v2v3", "world")(x0, y0)
+    # compare both results to make sure they don't match
+    # (which means the distortion correction was actually applied to the model)
+    assert (corrected_coords[0] != original_coords[0]).all()
+    assert (corrected_coords[1] != original_coords[1]).all()
+
+
+@pytest.mark.parametrize(
+    "arr_name", ("dq", "err", "var_poisson", "var_rnoise", "var_flat")
+)
+def test_array_exists(output_model, arr_name):
+    assert hasattr(output_model, arr_name)
+
+
+def test_has_exposure_time(output_model):
+    assert "exposure_time" in output_model.meta.exposure
+
+
+@pytest.mark.parametrize("meta_attribute", ("detector", "optical_element"))
+def test_instrument_meta(output_model, meta_attribute):
+    assert meta_attribute in output_model.meta.instrument
+
+
+def test_wcs_has_bounding_box(output_model):
+    assert len(output_model.meta.wcs.bounding_box) == 2