From a905bc0e3f2d8a70aa971cf6bb200021534943a0 Mon Sep 17 00:00:00 2001 From: Laszlo Dobos Date: Fri, 24 May 2024 15:37:44 -0400 Subject: [PATCH 1/6] Minor updates to work with gapipe --- python/pfs/datamodel/pfsFiberArray.py | 4 +++- python/pfs/datamodel/pfsFluxReference.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/pfs/datamodel/pfsFiberArray.py b/python/pfs/datamodel/pfsFiberArray.py index 56105f25..bc7127ce 100644 --- a/python/pfs/datamodel/pfsFiberArray.py +++ b/python/pfs/datamodel/pfsFiberArray.py @@ -44,6 +44,7 @@ class PfsFiberArray(PfsSimpleSpectrum): """ filenameFormat = None # Subclasses should override NotesClass: Type[Notes] # Subclasses should override + FluxTableClass = FluxTable # Subclasses may override def __init__( self, @@ -114,7 +115,7 @@ def _readImpl(cls, fits): data["covar"] = fits["COVAR"].data.astype(np.float32) data["covar2"] = fits["COVAR2"].data.astype(np.float32) try: - fluxTable = FluxTable.fromFits(fits) + fluxTable = cls.FluxTableClass.fromFits(fits) except KeyError as exc: # Only want to catch "Extension XXX not found." if not exc.args[0].startswith("Extension"): @@ -151,3 +152,4 @@ def _writeImpl(self, fits): self.notes.writeFits(fits) if self.fluxTable is not None: self.fluxTable.toFits(fits) + return header \ No newline at end of file diff --git a/python/pfs/datamodel/pfsFluxReference.py b/python/pfs/datamodel/pfsFluxReference.py index 97262a02..bc8c159a 100644 --- a/python/pfs/datamodel/pfsFluxReference.py +++ b/python/pfs/datamodel/pfsFluxReference.py @@ -210,7 +210,7 @@ def fits_getdata(hdulist, name, dtype=None, needHeader=False): data["identity"] = Identity.fromFits(fd) data["metadata"] = astropyHeaderToDict(fd[0].header) data["wavelength"] = WavelengthArray.fromFitsHeader( - wcsHeader, data["flux"].shape[1], dtype=np.float + wcsHeader, data["flux"].shape[1], dtype=float ) data["fitFlagNames"] = MaskHelper.fromFitsHeader(flagHeader) From 8ca821f45c050ccebae08c6110856279e388cd83 Mon Sep 17 00:00:00 2001 From: Laszlo Dobos Date: Fri, 31 May 2024 18:42:15 -0400 Subject: [PATCH 2/6] Added first version of GA datamodel with unit tests. --- python/pfs/datamodel/__init__.py | 1 + python/pfs/datamodel/fluxTable.py | 21 +- python/pfs/datamodel/ga.py | 308 ++++++++++++++++++++++++++ python/pfs/datamodel/pfsFiberArray.py | 2 +- tests/test_pfsGAObject.py | 183 +++++++++++++++ 5 files changed, 509 insertions(+), 6 deletions(-) create mode 100644 python/pfs/datamodel/ga.py create mode 100644 tests/test_pfsGAObject.py diff --git a/python/pfs/datamodel/__init__.py b/python/pfs/datamodel/__init__.py index 1f2b6c1b..8d7f19f1 100644 --- a/python/pfs/datamodel/__init__.py +++ b/python/pfs/datamodel/__init__.py @@ -15,3 +15,4 @@ from .pfsTable import * from .pfsFocalPlaneFunction import * from .pfsFiberNorms import * +from .ga import * \ No newline at end of file diff --git a/python/pfs/datamodel/fluxTable.py b/python/pfs/datamodel/fluxTable.py index c60c1cd4..48fde0d6 100644 --- a/python/pfs/datamodel/fluxTable.py +++ b/python/pfs/datamodel/fluxTable.py @@ -38,11 +38,11 @@ class FluxTable: _hduName = "FLUX_TABLE" # HDU name to use def __init__(self, wavelength, flux, error, mask, flags): - dims = np.array([len(wavelength.shape), len(flux.shape), len(error.shape), len(mask.shape)]) - lengths = set([wavelength.shape, flux.shape, error.shape, mask.shape]) - if np.any(dims != 1) or len(lengths) > 1: - raise RuntimeError("Bad shapes for wavelength,flux,error,mask: %s,%s,%s,%s" % - (wavelength.shape, flux.shape, error.shape, mask.shape)) + self.checkShapes(wavelength=wavelength, + flux=flux, + error=error, + mask=mask) + self.wavelength = wavelength self.flux = flux self.error = error @@ -52,6 +52,17 @@ def __init__(self, wavelength, flux, error, mask, flags): def __len__(self): """Return number of elements""" return len(self.wavelength) + + def checkShapes(self, **kwargs): + keys = list(sorted(kwargs.keys())) + dims = np.array([ len(kwargs[k].shape) for k in keys ]) + lengths = set([ kwargs[k].shape for k in keys ]) + + if np.any(dims != 1) or len(lengths) > 1: + names = ','.join(keys) + shapes = ','.join([str(kwargs[k].shape) for k in keys]) + raise RuntimeError("Bad shapes for %s: %s" % + (names, shapes)) def toFits(self, fits): """Write to a FITS file diff --git a/python/pfs/datamodel/ga.py b/python/pfs/datamodel/ga.py new file mode 100644 index 00000000..c65d94f1 --- /dev/null +++ b/python/pfs/datamodel/ga.py @@ -0,0 +1,308 @@ +from typing import Type +import numpy as np + +from .notes import makeNotesClass, Notes +from .pfsFiberArray import PfsFiberArray +from .fluxTable import FluxTable +from .pfsTable import PfsTable, Column +from .utils import inheritDocstrings +from .utils import astropyHeaderToDict, astropyHeaderFromDict +from .masks import MaskHelper + +__all__ = [ + "VelocityCorrections", + "StellarParams", + "Abundances", + "GAFluxTable", + "PfsGAObjectNotes", + "PfsGAObject", +] + +class VelocityCorrections(PfsTable): + """A table of velocity corrections applied to the individual visits.""" + + damdVer = 2 + schema = [ + Column("visit", np.int32, "ID of the visit these corrections apply for", -1), + Column("JD", np.float32, "Julian date of the visit", -1), + Column("helio", np.float32, "Heliocentric correction", np.nan), + Column("bary", np.float32, "Barycentric correction", np.nan), + ] + fitsExtName = 'VELCORR' + +class StellarParams(PfsTable): + """List of measured stellar parameters for a target.""" + + damdVer = 2 + schema = [ + Column("method", str, "Line-of-sight velocity measurement method", ""), + Column("frame", str, "Reference frame of velocity: helio, bary", ""), + Column("param", str, "Stellar parameter: v_los, M_H, T_eff, log_g, a_M", ""), + Column("covarId", np.uint8, "Param position within covariance matrix", -1), + Column("unit", str, "Physical unit of parameter", ""), + Column("value", np.float32, "Stellar parameter value", np.nan), + Column("valueErr", np.float32, "Stellar parameter error", np.nan), + # TODO: add quantiles or similar for MCMC results + Column("flag", bool, "Measurement flag (true means bad)", False), + Column("status", str, "Measurement flags", ""), + ] + fitsExtName = 'STELLARPARAM' + +class Abundances(PfsTable): + """List of measured abundance parameters for stellar targets.""" + + damdVer = 2 + schema = [ + Column("method", str, "Abundance measurement method", ""), + Column("element", str, "Chemical element the abundance is measured for", ""), + Column("covarId", np.uint8, "Param position within covariance matrix", -1), + Column("value", np.float32, "Abundance value", np.nan), + Column("valueErr", np.float32, "Abundance error", np.nan), + ] + fitsExtName = 'ABUND' + +class GAFluxTable(FluxTable): + """Table of coadded fluxes at near-original sampling and model fits + + Merged and coadded spectra have been resampled to a standard wavelength + sampling. This representation provides coadded fluxes at approximately the + native wavelength sampling, for those that want the data with a minimum of + resampling. This is mostly of use for single exposures and coadds made from + back-to-back exposures with the same top-end configuration. For coadds made + from exposures with different top-end configurations, the different + wavelength samplings obtained from the different fibers means there's no + single native wavelength sampling, and so this is less useful. + + This is like a `pfs.datamodel.PfsSimpleSpectrum`, except that it includes a + variance array, and is written to a FITS HDU rather than a file (so it can + be incorporated within a `pfs.datamodel.PfsSpectrum`). + + Parameters + ---------- + wavelength : `numpy.ndarray` of `float` + Array of wavelengths. + flux : `numpy.ndarray` of `float` + Array of fluxes. + error : `numpy.ndarray` of `float` + Array of flux errors. + model : `numpy.ndarray` of `float` + Array of best-fit model flux. + cont : `numpy.ndarray` of `float` + Array of continuum model. + norm_flux : `numpy.ndarray` of `float` + Array of continuum-normalized flux. + norm_error : `numpy.ndarray` of `float` + Array of continuum-normalized flux error. + norm_model : `numpy.ndarray` of `float` + Array of continuum-normalized model. + mask : `numpy.ndarray` of `int` + Array of mask pixels. + flags : `pfs.datamodel.MaskHelper` + Helper for dealing with symbolic names for mask values. + """ + _hduName = "FLUX_TABLE" # HDU name to use + + def __init__(self, wavelength, flux, error, model, cont, norm_flux, norm_error, norm_model, mask, flags): + self.checkShapes(wavelength=wavelength, + flux=flux, + error=error, + model=model, + cont=cont, + norm_flux=norm_flux, + norm_error=norm_error, + norm_model=norm_model, + mask=mask) + + self.wavelength = wavelength + self.flux = flux + self.error = error + self.model = model + self.cont = cont + self.norm_flux = norm_flux + self.norm_error = norm_error + self.norm_model = norm_model + self.mask = mask + self.flags = flags + + def toFits(self, fits): + """Write to a FITS file + + Parameters + ---------- + fits : `astropy.io.fits.HDUList` + Opened FITS file. + """ + # NOTE: When making any changes to this method that modify the output + # format, increment the DAMD_VER header value and record the change in + # the versions.txt file. + from astropy.io.fits import BinTableHDU, Column + header = astropyHeaderFromDict(self.flags.toFitsHeader()) + header['DAMD_VER'] = (1, "GAFluxTable datamodel version") + hdu = BinTableHDU.from_columns([ + Column("wavelength", "D", array=self.wavelength), + Column("flux", "E", array=self.flux), + Column("error", "E", array=self.error), + Column("model", "E", array=self.model), + Column("cont", "E", array=self.cont), + Column("norm_flux", "E", array=self.norm_flux), + Column("norm_error", "E", array=self.norm_error), + Column("norm_model", "E", array=self.norm_model), + Column("mask", "K", array=self.mask), + ], header=header, name=self._hduName) + fits.append(hdu) + + @classmethod + def fromFits(cls, fits): + """Construct from a FITS file + + Parameters + ---------- + fits : `astropy.io.fits.HDUList` + Opened FITS file. + + Returns + ------- + self : `FluxTable` + Constructed `FluxTable`. + """ + hdu = fits[cls._hduName] + header = astropyHeaderToDict(hdu.header) + flags = MaskHelper.fromFitsHeader(header) + return cls(hdu.data["wavelength"].astype(float), + hdu.data["flux"].astype(np.float32), + hdu.data["error"].astype(np.float32), + hdu.data["model"].astype(np.float32), + hdu.data["cont"].astype(np.float32), + hdu.data["norm_flux"].astype(np.float32), + hdu.data["norm_error"].astype(np.float32), + hdu.data["norm_model"].astype(np.float32), + hdu.data["mask"].astype(np.int32), + flags) + +PfsGAObjectNotes = makeNotesClass( + "PfsGAObjectNotes", + [] +) + +@inheritDocstrings +class PfsGAObject(PfsFiberArray): + """Coadded spectrum of a GA target with derived quantities. + + Produced by ˙˙gapipe`` + + Parameters + ---------- + target : `pfs.datamodel.Target` + Target information. + observations : `pfs.datamodel.Observations` + Observations of the target. + wavelength : `numpy.ndarray` of `float` + Array of wavelengths. + flux : `numpy.ndarray` of `float` + Array of fluxes. + mask : `numpy.ndarray` of `int` + Array of mask pixels. + sky : `numpy.ndarray` of `float` + Array of sky values. + covar : `numpy.ndarray` of `float` + Near-diagonal (diagonal and either side) part of the covariance matrix. + covar2 : `numpy.ndarray` of `float` + Low-resolution non-sparse covariance estimate. + flags : `MaskHelper` + Helper for dealing with symbolic names for mask values. + metadata : `dict` (`str`: POD), optional + Keyword-value pairs for the header. + fluxTable : `pfs.datamodel.GAFluxTable`, optional + Table of coadded fluxes and continuum-normalized flux from contributing observations. + stellarParams: `pfs.datamodel.StellarParams`, optional + Table of measured stellar parameters. + velocityCorrections: `pfs.datamodel.VelocityCorrections`, optional + Table of velocity corrections applied to the individual visits. + abundances: `pfs.datamodel.Abundances`, optional + Table of measured abundance parameters. + paramsCovar: `numpy.ndarray` of `float`, optional + Covariance matrix for stellar parameters. + abundCovar: `numpy.ndarray` of `float`, optional + Covariance matrix for abundance parameters. + notes : `Notes`, optional + Reduction notes. + """ + + filenameFormat = ("pfsGAObject-%(catId)05d-%(tract)05d-%(patch)s-%(objId)016x" + "-%(nVisit)03d-0x%(pfsVisitHash)016x.fits") + filenameRegex = r"^pfsGAObject-(\d{5})-(\d{5})-(.*)-([0-9a-f]{16})-(\d{3})-0x([0-9a-f]{16})\.fits.*$" + filenameKeys = [("catId", int), ("tract", int), ("patch", str), ("objId", int), + ("nVisit", int), ("pfsVisitHash", int)] + NotesClass = PfsGAObjectNotes + FluxTableClass = GAFluxTable + + StellarParamsFitsExtName = "STELLARCOVAR" + AbundancesFitsExtName = "ABUNDCOVAR" + + def __init__( + self, + target, + observations, + wavelength, + flux, + mask, + sky, + covar, + covar2, + flags, + metadata=None, + fluxTable=None, + stellarParams=None, + velocityCorrections=None, + abundances=None, + paramsCovar=None, + abundCovar=None, + notes: Notes = None, + ): + super().__init__(target, observations, wavelength, flux, mask, sky, covar, covar2, flags, metadata=metadata, fluxTable=fluxTable, notes=notes) + + self.stellarParams = stellarParams + self.velocityCorrections = velocityCorrections + self.abundances = abundances + self.paramsCovar = paramsCovar + self.abundCovar = abundCovar + + def validate(self): + """Validate that all the arrays are of the expected shape""" + super().validate() + + # TODO: write any validation code + + @classmethod + def _readImpl(cls, fits): + data = super()._readImpl(fits) + + # TODO: handle missing extensions + + data["stellarParams"] = StellarParams.readHdu(fits) + data["velocityCorrections"] = VelocityCorrections.readHdu(fits) + data["abundances"] = Abundances.readHdu(fits) + if cls.StellarParamsFitsExtName in fits: + data["paramsCovar"] = fits[cls.StellarParamsFitsExtName].data.astype(np.float32) + if cls.AbundancesFitsExtName in fits: + data["abundCovar"] = fits[cls.AbundancesFitsExtName].data.astype(np.float32) + + return data + + def _writeImpl(self, fits): + from astropy.io.fits import ImageHDU + + header = super()._writeImpl(fits) + + if self.stellarParams is not None: + self.stellarParams.writeHdu(fits) + if self.velocityCorrections is not None: + self.velocityCorrections.writeHdu(fits) + if self.abundances is not None: + self.abundances.writeHdu(fits) + if self.paramsCovar is not None: + fits.append(ImageHDU(self.paramsCovar.astype(np.float32), header=header, name=self.StellarParamsFitsExtName)) + if self.abundCovar is not None: + fits.append(ImageHDU(self.abundCovar.astype(np.float32), header=header, name=self.AbundancesFitsExtName)) + + return header \ No newline at end of file diff --git a/python/pfs/datamodel/pfsFiberArray.py b/python/pfs/datamodel/pfsFiberArray.py index bc7127ce..cb024c74 100644 --- a/python/pfs/datamodel/pfsFiberArray.py +++ b/python/pfs/datamodel/pfsFiberArray.py @@ -44,7 +44,7 @@ class PfsFiberArray(PfsSimpleSpectrum): """ filenameFormat = None # Subclasses should override NotesClass: Type[Notes] # Subclasses should override - FluxTableClass = FluxTable # Subclasses may override + FluxTableClass: Type[FluxTable] = FluxTable # Subclasses may override def __init__( self, diff --git a/tests/test_pfsGAObject.py b/tests/test_pfsGAObject.py new file mode 100644 index 00000000..f720dc8b --- /dev/null +++ b/tests/test_pfsGAObject.py @@ -0,0 +1,183 @@ +import os +import re +import numpy as np +from unittest import TestCase + +from pfs.datamodel import Target, TargetType +from pfs.datamodel import Observations +from pfs.datamodel import MaskHelper +from pfs.datamodel import GAFluxTable +from pfs.datamodel import PfsGAObject, StellarParams, Abundances, VelocityCorrections + +class PfsGAObjectTestCase(TestCase): + """ Check the format of example datamodel files are + consistent with that specified in the corresponding + datamodel classes. + """ + + def makePfsGAObject(self): + """Construct a PfsGAObject with dummy values for testing.""" + + catId = 12345 + tract = 1 + patch = '1,1' + objId = 123456789 + ra = -100.63654 + dec = -68.591576 + targetType = TargetType.SCIENCE + + target = Target(catId, tract, patch, objId, ra, dec, targetType) + + visit = np.array([ 83219, 83220 ]) + arm = np.array([ 'b', 'm', ]) + spectrograph = np.array([1, 1]) + pfsDesignId = np.array([8854764194165386399, 8854764194165386400]) + fiberId = np.array([476, 476]) + pfiNominal = np.array([[ra, dec], [ra, dec]]) + pfiCenter = np.array([[ra, dec], [ra, dec]]) + + observations = Observations(visit, arm, spectrograph, pfsDesignId, fiberId, pfiNominal, pfiCenter) + + npix = 4096 + wavelength = np.concatenate([ + np.linspace(380, 650, npix, dtype=np.float32), + np.linspace(710, 885, npix, dtype=np.float32) + ]) + flux = np.zeros_like(wavelength) + error = np.zeros_like(wavelength) + model = np.zeros_like(wavelength) + cont = np.zeros_like(wavelength) + norm_flux = np.zeros_like(wavelength) + norm_error = np.zeros_like(wavelength) + norm_model = np.zeros_like(wavelength) + mask = np.zeros_like(wavelength, dtype=np.int32) + sky = np.zeros_like(wavelength) + covar = np.zeros((3, wavelength.size), dtype=np.float32) # Tridiagonal covariance matrix of flux + covar2 = np.zeros((1, 1), dtype=np.float32) # ? + + flags = MaskHelper() # {'BAD': 0, 'BAD_FIBERTRACE': 11, 'BAD_FLAT': 9, 'BAD_FLUXCAL': 13, 'BAD_SKY': 12, 'CR': 3, 'DETECTED': 5, 'DETECTED_NEGATIVE': 6, 'EDGE': 4, 'FIBERTRACE': 10, 'INTRP': 2, 'IPC': 14, 'NO_DATA': 8, 'REFLINE': 15, 'SAT': 1, 'SUSPECT': 7, 'UNMASKEDNAN': 16}) + metadata = {} # Key-value pairs to put in the header + fluxTable = GAFluxTable(wavelength, flux, error, model, cont, + norm_flux, norm_error, norm_model, + mask, flags) + + stellarParams = StellarParams( + method=np.array(['rvfit', 'rvfit', 'rvfit', 'rvfit', 'rvfit']), + frame=np.array(['helio', '', '', '', '']), + param=np.array(['v_los', 'Fe_H', 'T_eff', 'log_g', 'a_Fe']), + covarId=np.array([0, 1, 2, 3, 4]), + unit=np.array(['km s-1', 'dex', 'K', '', 'dex']), + value=np.array([0.0, 0.0, 0.0, 0.0, 0.0]), + valueErr=np.array([0.0, 0.0, 0.0, 0.0, 0.0]), + flag=np.array([False, False, False, False, False]), + status=np.array(['', '', '', '', '']), + ) + + velocityCorrections = VelocityCorrections( + visit=visit, + JD=np.zeros_like(visit, dtype=np.float32), + helio=np.zeros_like(visit, dtype=np.float32), + bary=np.zeros_like(visit, dtype=np.float32), + ) + + abundances = Abundances( + method=np.array(['chemfit', 'chemfit', 'chemfit']), + element=np.array(['Mg', 'Ti', 'Si']), + covarId=np.array([0, 1, 2]), + value=np.array([0.0, 0.0, 0.0]), + valueErr=np.array([0.0, 0.0, 0.0]), + ) + + paramsCovar = np.eye(3, dtype=np.float32) + abundCovar = np.eye(4, dtype=np.float32) + notes = None + + return PfsGAObject(target, observations, + wavelength, flux, mask, sky, covar, covar2, + flags, metadata, + fluxTable, + stellarParams, + velocityCorrections, + abundances, + paramsCovar, + abundCovar, + notes) + + def assertPfsGAObject(self, lhs, rhs): + np.testing.assert_array_equal(lhs.observations.visit, rhs.observations.visit) + + # TODO: add more tests here + + def extractAttributes(self, cls, fileName): + matches = re.search(cls.filenameRegex, fileName) + if not matches: + self.fail( + "Unable to parse filename: {} using regex {}" + .format(fileName, cls.filenameRegex)) + + # Cannot use algorithm in PfsSpectra._parseFilename(), + # specifically cls.filenameKeys, due to ambiguity in parsing + # integers in hex format (eg objId). Need to parse cls.filenameFormat + ff = re.search(r'^[a-zA-Z]+(.*)\.fits', cls.filenameFormat)[1] + cmps = re.findall(r'-{0,1}(0x){0,1}\%\((\w+)\)\d*(\w)', ff) + fmts = [(kk, tt) for ox, kk, tt in cmps] + + d = {} + for (kk, tt), vv in zip(fmts, matches.groups()): + if tt == 'd': + ii = int(vv) + elif tt == 'x': + ii = int(vv, 16) + elif tt == 's': + ii = vv + d[kk] = ii + return d + + def test_filenameRegex(self): + d = self.extractAttributes( + PfsGAObject, + 'pfsGAObject-07621-01234-2,2-02468ace1234abcd-003-0x0123456789abcdef.fits') + self.assertEqual(d['catId'], 7621) + self.assertEqual(d['tract'], 1234) + self.assertEqual(d['patch'], '2,2') + self.assertEqual(d['objId'], 163971054118939597) + self.assertEqual(d['nVisit'], 3) + self.assertEqual(d['pfsVisitHash'], 81985529216486895) + + def test_getIdentity(self): + """Construct a PfsGAObject and get its identity.""" + + pfsGAObject = self.makePfsGAObject() + identity = pfsGAObject.getIdentity() + filename = pfsGAObject.filenameFormat % identity + + self.assertEqual('pfsGAObject-12345-00001-1,1-00000000075bcd15-002-0x05a95bc24d8ce16f.fits', filename) + + def test_validate(self): + """Construct a PfsGAObject and run validation.""" + + pfsGAObject = self.makePfsGAObject() + pfsGAObject.validate() + + def test_writeFits_fromFits(self): + """Construct a PfsGAObject and save it to a FITS file.""" + + pfsGAObject = self.makePfsGAObject() + + dirName = os.path.splitext(__file__)[0] + if not os.path.exists(dirName): + os.makedirs(dirName) + + id = pfsGAObject.getIdentity() + filename = os.path.join(dirName, pfsGAObject.filenameFormat % id) + if os.path.exists(filename): + os.unlink(filename) + + try: + pfsGAObject.writeFits(filename) + other = PfsGAObject.readFits(filename) + self.assertPfsGAObject(pfsGAObject, other) + except Exception as e: + raise + finally: + os.unlink(filename) From 3b81f298acd45d2aeebe206a25a4ba82bdf209f4 Mon Sep 17 00:00:00 2001 From: Laszlo Dobos Date: Fri, 31 May 2024 18:43:18 -0400 Subject: [PATCH 3/6] Do not fail on members defining class type of other members. --- tests/test_docstrings.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 4e590fd9..88eb14f3 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -5,18 +5,20 @@ import lsst.utils.tests from pfs.datamodel.drp import PfsArm, PfsMerged, PfsReference, PfsSingle, PfsObject +from pfs.datamodel.ga import PfsGAObject class DocstringsTestCase(unittest.TestCase): def testDocstrings(self): - for cls in (PfsArm, PfsMerged, PfsReference, PfsSingle, PfsObject): + for cls in (PfsArm, PfsMerged, PfsReference, PfsSingle, PfsObject, PfsGAObject): for name, attr in inspect.getmembers(cls): if not hasattr(attr, "__doc__") or not attr.__doc__: continue docstring = attr.__doc__ for base in cls.__mro__[1:-1]: - self.assertNotIn(base.__name__, docstring, - f"{cls.__name__}.{name}.__doc__ contains {base.__name__}: {docstring}") + if not name.endswith("Class"): + self.assertNotIn(base.__name__, docstring, + f"{cls.__name__}.{name}.__doc__ contains {base.__name__}: {docstring}") class TestMemory(lsst.utils.tests.MemoryTestCase): From a967df1a3c2f36f930b7e41af37605047ccc1159 Mon Sep 17 00:00:00 2001 From: Laszlo Dobos Date: Fri, 14 Jun 2024 15:50:42 -0400 Subject: [PATCH 4/6] Flags and status added to abundances, reorganized order of HDUs. --- python/pfs/datamodel/ga.py | 15 +++++++++------ tests/test_pfsGAObject.py | 2 ++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/python/pfs/datamodel/ga.py b/python/pfs/datamodel/ga.py index c65d94f1..e4bf10f7 100644 --- a/python/pfs/datamodel/ga.py +++ b/python/pfs/datamodel/ga.py @@ -58,6 +58,9 @@ class Abundances(PfsTable): Column("covarId", np.uint8, "Param position within covariance matrix", -1), Column("value", np.float32, "Abundance value", np.nan), Column("valueErr", np.float32, "Abundance error", np.nan), + # TODO: will we have systematic errors? + Column("flag", bool, "Measurement flag (true means bad)", False), + Column("status", str, "Measurement flags", ""), ] fitsExtName = 'ABUND' @@ -279,11 +282,11 @@ def _readImpl(cls, fits): # TODO: handle missing extensions - data["stellarParams"] = StellarParams.readHdu(fits) data["velocityCorrections"] = VelocityCorrections.readHdu(fits) - data["abundances"] = Abundances.readHdu(fits) + data["stellarParams"] = StellarParams.readHdu(fits) if cls.StellarParamsFitsExtName in fits: data["paramsCovar"] = fits[cls.StellarParamsFitsExtName].data.astype(np.float32) + data["abundances"] = Abundances.readHdu(fits) if cls.AbundancesFitsExtName in fits: data["abundCovar"] = fits[cls.AbundancesFitsExtName].data.astype(np.float32) @@ -294,14 +297,14 @@ def _writeImpl(self, fits): header = super()._writeImpl(fits) - if self.stellarParams is not None: - self.stellarParams.writeHdu(fits) if self.velocityCorrections is not None: self.velocityCorrections.writeHdu(fits) - if self.abundances is not None: - self.abundances.writeHdu(fits) + if self.stellarParams is not None: + self.stellarParams.writeHdu(fits) if self.paramsCovar is not None: fits.append(ImageHDU(self.paramsCovar.astype(np.float32), header=header, name=self.StellarParamsFitsExtName)) + if self.abundances is not None: + self.abundances.writeHdu(fits) if self.abundCovar is not None: fits.append(ImageHDU(self.abundCovar.astype(np.float32), header=header, name=self.AbundancesFitsExtName)) diff --git a/tests/test_pfsGAObject.py b/tests/test_pfsGAObject.py index f720dc8b..4ec10261 100644 --- a/tests/test_pfsGAObject.py +++ b/tests/test_pfsGAObject.py @@ -86,6 +86,8 @@ def makePfsGAObject(self): covarId=np.array([0, 1, 2]), value=np.array([0.0, 0.0, 0.0]), valueErr=np.array([0.0, 0.0, 0.0]), + flag=np.array([False, False, False]), + status=np.array(['', '', '']), ) paramsCovar = np.eye(3, dtype=np.float32) From 13913d290541934fe2d1ea3bb4d3aac7d0418ac6 Mon Sep 17 00:00:00 2001 From: Laszlo Dobos Date: Fri, 14 Jun 2024 15:51:33 -0400 Subject: [PATCH 5/6] Added docs on GA data model. --- datamodel.txt | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/datamodel.txt b/datamodel.txt index f61271ab..5284d2b5 100644 --- a/datamodel.txt +++ b/datamodel.txt @@ -673,6 +673,64 @@ The format will be identical to that of the pfsObject files. -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- +Physical parameter measurements for Galactic Archeology targets including flux-calibrated +combined spectra and synthetic stellar template fits. + + "pfsGAObject-%05d-%05d-%s-%016x-%03d-0x%016x.fits" + % (catId, tract, patch, objId, nVisit % 1000, pfsVisitHash) + +The format is similar to pfsObject files, but with additional HDUs and a different fluxTable format. + +HDU #j VELCORR Velocity corrections used at each visit [BINARY FITS TABLE] +HDU #i STELLARPARAMS Fundamental stellar parameter measurements [BINARY FITS TABLE] +HDU #l STELLARCOVAR Covariance matrix of stellar parameters [BINARY FITS TABLE] n*n +HDU #k ABUND Single element abundances [BINARY FITS TABLE] +HDU #m ABUNDCOVAR Covarinace matrix of single element abundances [32-bit FLOAT] n*n + +In the data tables outlined below, we allow for the possibility of multiple measurements of the same +physical parameters indicated by the `method` field but we provide the full covariance matrix for +the primary method only. The column `covarId` indicates the position of the parameter within the +corresponding covariance matrix. + +The VELCORR table lists the velocity corrections used at each visit: + + visit visit identifier 32-bit int + JD Julian Date of the visit 32-bit float + helio Heliocentric correction at the visit [km s-1] 32-bit float + bary Barycentric correction at the visit [km s-2] 32-bit float + +The STELLARPARAMS table lists the measured fundamental parameters + + method Method of measuring the parameters string + frame Reference frame for measuring the velocity string + param Stellar parameter string + covarId Index of parameter within the covariance matrix 8-bit uint + unit Physical unit of the parameter string + value Measured value of the parameter 32-bit float + valueErr Uncertainty of the measured value 32-bit float + flag Measurement flag (true means bad) bool + status Flags describing the quality of the measurement string + +The ABUND table lists the measured single element abundances + + method Method of measuring the parameters string + element Element name string + covarId Index of parameter within the covariance matrix 8-bit uint + value Measured abundance 32-bit float + valueErr Uncertainty of the measured abundance 32-bit float + flag Measurement flag (true means bad) bool + status Flags describing the quality of the measurement string + +The FLUXTABLE of pfsGAObject files is extended and includes the following additional columns: + + model Best fit fluxed template model 32-bit float + cont Continuum model 32-bit float + norm_flux Continuum-normalized flux 32-bit float + norm_error Error of continuum-normalized flux 32-bit float + norm_model Best fit continuum-normalized model 32-bit float + +-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + 2D spectral line measurements The 2D pipeline measures the position and flux of sky and arc lines as part of the reduction process. From bc36cab84df74ba77e783fc8310c53e78ef4087e Mon Sep 17 00:00:00 2001 From: Laszlo Dobos Date: Fri, 14 Jun 2024 17:45:24 -0400 Subject: [PATCH 6/6] Fixed all flake8 errors. --- python/pfs/datamodel/fluxTable.py | 8 +++--- python/pfs/datamodel/ga.py | 26 ++++++++++++++------ python/pfs/datamodel/pfsFiberArray.py | 2 +- tests/test_docstrings.py | 5 ++-- tests/test_pfsGAObject.py | 35 ++++++++++++++------------- 5 files changed, 44 insertions(+), 32 deletions(-) diff --git a/python/pfs/datamodel/fluxTable.py b/python/pfs/datamodel/fluxTable.py index 48fde0d6..192827f7 100644 --- a/python/pfs/datamodel/fluxTable.py +++ b/python/pfs/datamodel/fluxTable.py @@ -42,7 +42,7 @@ def __init__(self, wavelength, flux, error, mask, flags): flux=flux, error=error, mask=mask) - + self.wavelength = wavelength self.flux = flux self.error = error @@ -52,11 +52,11 @@ def __init__(self, wavelength, flux, error, mask, flags): def __len__(self): """Return number of elements""" return len(self.wavelength) - + def checkShapes(self, **kwargs): keys = list(sorted(kwargs.keys())) - dims = np.array([ len(kwargs[k].shape) for k in keys ]) - lengths = set([ kwargs[k].shape for k in keys ]) + dims = np.array([len(kwargs[k].shape) for k in keys]) + lengths = set([kwargs[k].shape for k in keys]) if np.any(dims != 1) or len(lengths) > 1: names = ','.join(keys) diff --git a/python/pfs/datamodel/ga.py b/python/pfs/datamodel/ga.py index e4bf10f7..69ad3a70 100644 --- a/python/pfs/datamodel/ga.py +++ b/python/pfs/datamodel/ga.py @@ -1,4 +1,3 @@ -from typing import Type import numpy as np from .notes import makeNotesClass, Notes @@ -18,6 +17,7 @@ "PfsGAObject", ] + class VelocityCorrections(PfsTable): """A table of velocity corrections applied to the individual visits.""" @@ -30,6 +30,7 @@ class VelocityCorrections(PfsTable): ] fitsExtName = 'VELCORR' + class StellarParams(PfsTable): """List of measured stellar parameters for a target.""" @@ -48,6 +49,7 @@ class StellarParams(PfsTable): ] fitsExtName = 'STELLARPARAM' + class Abundances(PfsTable): """List of measured abundance parameters for stellar targets.""" @@ -64,6 +66,7 @@ class Abundances(PfsTable): ] fitsExtName = 'ABUND' + class GAFluxTable(FluxTable): """Table of coadded fluxes at near-original sampling and model fits @@ -115,7 +118,7 @@ def __init__(self, wavelength, flux, error, model, cont, norm_flux, norm_error, norm_error=norm_error, norm_model=norm_model, mask=mask) - + self.wavelength = wavelength self.flux = flux self.error = error @@ -182,17 +185,19 @@ def fromFits(cls, fits): hdu.data["mask"].astype(np.int32), flags) + PfsGAObjectNotes = makeNotesClass( "PfsGAObjectNotes", [] ) + @inheritDocstrings class PfsGAObject(PfsFiberArray): """Coadded spectrum of a GA target with derived quantities. - + Produced by ˙˙gapipe`` - + Parameters ---------- target : `pfs.datamodel.Target` @@ -262,7 +267,8 @@ def __init__( abundCovar=None, notes: Notes = None, ): - super().__init__(target, observations, wavelength, flux, mask, sky, covar, covar2, flags, metadata=metadata, fluxTable=fluxTable, notes=notes) + super().__init__(target, observations, wavelength, flux, mask, sky, + covar, covar2, flags, metadata=metadata, fluxTable=fluxTable, notes=notes) self.stellarParams = stellarParams self.velocityCorrections = velocityCorrections @@ -302,10 +308,14 @@ def _writeImpl(self, fits): if self.stellarParams is not None: self.stellarParams.writeHdu(fits) if self.paramsCovar is not None: - fits.append(ImageHDU(self.paramsCovar.astype(np.float32), header=header, name=self.StellarParamsFitsExtName)) + fits.append(ImageHDU(self.paramsCovar.astype(np.float32), + header=header, + name=self.StellarParamsFitsExtName)) if self.abundances is not None: self.abundances.writeHdu(fits) if self.abundCovar is not None: - fits.append(ImageHDU(self.abundCovar.astype(np.float32), header=header, name=self.AbundancesFitsExtName)) + fits.append(ImageHDU(self.abundCovar.astype(np.float32), + header=header, + name=self.AbundancesFitsExtName)) - return header \ No newline at end of file + return header diff --git a/python/pfs/datamodel/pfsFiberArray.py b/python/pfs/datamodel/pfsFiberArray.py index cb024c74..6486efb9 100644 --- a/python/pfs/datamodel/pfsFiberArray.py +++ b/python/pfs/datamodel/pfsFiberArray.py @@ -152,4 +152,4 @@ def _writeImpl(self, fits): self.notes.writeFits(fits) if self.fluxTable is not None: self.fluxTable.toFits(fits) - return header \ No newline at end of file + return header diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 88eb14f3..8b1553eb 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -17,8 +17,9 @@ def testDocstrings(self): docstring = attr.__doc__ for base in cls.__mro__[1:-1]: if not name.endswith("Class"): - self.assertNotIn(base.__name__, docstring, - f"{cls.__name__}.{name}.__doc__ contains {base.__name__}: {docstring}") + self.assertNotIn( + base.__name__, docstring, + f"{cls.__name__}.{name}.__doc__ contains {base.__name__}: {docstring}") class TestMemory(lsst.utils.tests.MemoryTestCase): diff --git a/tests/test_pfsGAObject.py b/tests/test_pfsGAObject.py index 4ec10261..2353c64c 100644 --- a/tests/test_pfsGAObject.py +++ b/tests/test_pfsGAObject.py @@ -9,6 +9,7 @@ from pfs.datamodel import GAFluxTable from pfs.datamodel import PfsGAObject, StellarParams, Abundances, VelocityCorrections + class PfsGAObjectTestCase(TestCase): """ Check the format of example datamodel files are consistent with that specified in the corresponding @@ -28,8 +29,8 @@ def makePfsGAObject(self): target = Target(catId, tract, patch, objId, ra, dec, targetType) - visit = np.array([ 83219, 83220 ]) - arm = np.array([ 'b', 'm', ]) + visit = np.array([83219, 83220]) + arm = np.array(['b', 'm']) spectrograph = np.array([1, 1]) pfsDesignId = np.array([8854764194165386399, 8854764194165386400]) fiberId = np.array([476, 476]) @@ -55,7 +56,7 @@ def makePfsGAObject(self): covar = np.zeros((3, wavelength.size), dtype=np.float32) # Tridiagonal covariance matrix of flux covar2 = np.zeros((1, 1), dtype=np.float32) # ? - flags = MaskHelper() # {'BAD': 0, 'BAD_FIBERTRACE': 11, 'BAD_FLAT': 9, 'BAD_FLUXCAL': 13, 'BAD_SKY': 12, 'CR': 3, 'DETECTED': 5, 'DETECTED_NEGATIVE': 6, 'EDGE': 4, 'FIBERTRACE': 10, 'INTRP': 2, 'IPC': 14, 'NO_DATA': 8, 'REFLINE': 15, 'SAT': 1, 'SUSPECT': 7, 'UNMASKEDNAN': 16}) + flags = MaskHelper() # metadata = {} # Key-value pairs to put in the header fluxTable = GAFluxTable(wavelength, flux, error, model, cont, norm_flux, norm_error, norm_model, @@ -95,16 +96,16 @@ def makePfsGAObject(self): notes = None return PfsGAObject(target, observations, - wavelength, flux, mask, sky, covar, covar2, - flags, metadata, - fluxTable, - stellarParams, - velocityCorrections, - abundances, - paramsCovar, - abundCovar, - notes) - + wavelength, flux, mask, sky, covar, covar2, + flags, metadata, + fluxTable, + stellarParams, + velocityCorrections, + abundances, + paramsCovar, + abundCovar, + notes) + def assertPfsGAObject(self, lhs, rhs): np.testing.assert_array_equal(lhs.observations.visit, rhs.observations.visit) @@ -134,11 +135,11 @@ def extractAttributes(self, cls, fileName): ii = vv d[kk] = ii return d - + def test_filenameRegex(self): d = self.extractAttributes( - PfsGAObject, - 'pfsGAObject-07621-01234-2,2-02468ace1234abcd-003-0x0123456789abcdef.fits') + PfsGAObject, + 'pfsGAObject-07621-01234-2,2-02468ace1234abcd-003-0x0123456789abcdef.fits') self.assertEqual(d['catId'], 7621) self.assertEqual(d['tract'], 1234) self.assertEqual(d['patch'], '2,2') @@ -180,6 +181,6 @@ def test_writeFits_fromFits(self): other = PfsGAObject.readFits(filename) self.assertPfsGAObject(pfsGAObject, other) except Exception as e: - raise + raise e finally: os.unlink(filename)