From 8280b5a513efa27d807aaec620a69ee63701ce59 Mon Sep 17 00:00:00 2001 From: psferguson Date: Tue, 16 Jul 2024 15:16:01 -0700 Subject: [PATCH] updated draft --- python/lsst/the/monster/assemble.py | 219 +++++++++++++++------------- python/lsst/the/monster/refcats.py | 35 +++-- python/lsst/the/monster/utils.py | 5 +- 3 files changed, 139 insertions(+), 120 deletions(-) diff --git a/python/lsst/the/monster/assemble.py b/python/lsst/the/monster/assemble.py index 39ee969..bf0ac65 100644 --- a/python/lsst/the/monster/assemble.py +++ b/python/lsst/the/monster/assemble.py @@ -38,7 +38,8 @@ class AssembleMonsterRefcat: photometry from the DES-calibrated catalogs. The catalogs are read in reverse priority order so that the final fluxes for any object come from the top priority catalog. At the end the intermediate - calibrated catalogs are converted to the final fluxes (e.g. LSST). + calibrated catalogs are converted to the final fluxes in the target + systems specified in target_catalog_info_class_list. Parameters ---------- @@ -46,22 +47,23 @@ class AssembleMonsterRefcat: The input Gaia DR3 RefcatInfo object. catalog_info_class_list : `list` [`RefcatInfo`] Reverse-priority list of catalog info classes for assembly. + target_catalog_info_class_list : `list` [`RefcatInfo`] + List of catalog info classes that will be output in final monster. monster_path_inp : `str`, optional Output monster path, overriding the class config. + do_u_band_slr : `bool`, optional + Perform u-band SLR using SDSS u and DES g & r bands testing_mode : `bool`, optional Enter testing mode for read_stars? - synth_system : `str`, optional - Synthetic system to do final conversion. """ def __init__(self, gaia_reference_class=GaiaDR3Info, catalog_info_class_list=[VSTInfo, SkyMapperInfo, - PS1Info, GaiaXPInfo, DESInfo, GaiaXPuInfo], - monster_path_inp=None, - testing_mode=False, + PS1Info, GaiaXPInfo, GaiaXPuInfo, DESInfo], target_catalog_info_class_list=[SynthLSSTInfo, LATISSInfo, DESInfo, SDSSuInfo], + monster_path_inp=None, do_u_band_slr=True, - + testing_mode=False, ): self.gaia_reference_info = gaia_reference_class() @@ -111,9 +113,8 @@ def run(self, target_bands = set(self.all_bands).intersection(cat_info_target.bands) # create all tuples of target catalog band for band in target_bands: - if (band == "u") & (target_system_name != "SDSS"): - # for u band where we are not using null transform - # (null is SDSS to SDSS) we want to create monster and then + if (band == "u"): + # for u band we want to create monster and then # use monster DES g and r band to transform to u band target_systems_u_transform.append((target_system_name, band)) else: @@ -125,7 +126,6 @@ def run(self, # create output columns for target_system_name, band in target_systems + target_systems_u_transform: - gaia_stars_all.add_column(nan_column, name=f"monster_{target_system_name}_{band}_flux") gaia_stars_all.add_column(nan_column, @@ -156,36 +156,21 @@ def run(self, # for each band do transformations skip u band for target_system_name, band in target_systems: if band in bands: - if band == "u": - if target_system_name not in ['SDSS']: - raise ValueError( - "this portion of the code should only be the null transform for u band" - ) - # since this is the null transform we dont need to - # apply a spline. I am struggling to figure out how - # to get the gaiaxpu in since it either has only - # u-band or will propogate other bands, open to input. - orig_flux = cat_stars[cat_info.get_transformed_flux_field(band)] - orig_flux_err = cat_stars[cat_info.get_transformed_flux_field(band)+'Err'] - model_flux = orig_flux - model_flux_err = orig_flux_err - - else: - # Transform from the DES to the synthetic system: - colorterm_file_string = 'DES_to_'+str(target_system_name)+'_band' - colorterm_spline = self.get_colorterm_spline(colorterm_file_string, band) - # apply colorterms to transform to target system mag - band_1, band_2 = cat_info.get_color_bands(band) - orig_flux = cat_stars[cat_info.get_transformed_flux_field(band)] - orig_flux_err = cat_stars[cat_info.get_transformed_flux_field(band)+'Err'] - model_flux = colorterm_spline.apply( - cat_stars[cat_info.get_transformed_flux_field(band_1)], - cat_stars[cat_info.get_transformed_flux_field(band_2)], - orig_flux, - ) - - # Rescale flux error to keep S/N constant - model_flux_err = model_flux * (orig_flux_err/orig_flux) + # Transform from the DES to the target system: + colorterm_file_string = 'DES_to_'+str(target_system_name)+'_band' + colorterm_spline = self.get_colorterm_spline(colorterm_file_string, band) + # apply colorterms to transform to target system mag + band_1, band_2 = cat_info.get_color_bands(band) + orig_flux = cat_stars[cat_info.get_transformed_flux_field(band)] + orig_flux_err = cat_stars[cat_info.get_transformed_flux_field(band)+'Err'] + model_flux = colorterm_spline.apply( + cat_stars[cat_info.get_transformed_flux_field(band_1)], + cat_stars[cat_info.get_transformed_flux_field(band_2)], + orig_flux, + ) + + # Rescale flux error to keep S/N constant + model_flux_err = model_flux * (orig_flux_err/orig_flux) # Add the fluxes and their errors to the catalog: cat_stars[f"monster_{target_system_name}_{band}_flux"] = model_flux @@ -196,12 +181,6 @@ def run(self, # catalogs already had their survey-specific cuts applied, # so these cuts should be for the Synth{synth_system} # transformations.) - # To DO: figure out what is going on here - # color_range = output_system.get_color_range(band) - # colors = - # cat_info.get_transformed_mag_colors(cat_stars, band) - # selected = - # (colors >= color_range[0]) & (colors <= color_range[1]) flux_not_nan = np.isfinite(cat_stars[f"monster_{target_system_name}_{band}_flux"]) if band == "u": # if u band we are only doing null transform @@ -234,84 +213,122 @@ def run(self, gaia_stars_all[ f"monster_{target_system_name}_{band}_source_flag" ][idx1] = cat_info.flag - + # First u band SLR and transformations to target systems if self.do_u_band_slr & ("u" in self.all_bands): - # for u band slr we use des g and r bands to - # transform to sdss u band + # for u band slr we use DES g and r bands to + # transform to SDSS u band target_system_name = 'SDSS' band = "u" - # we should have all of the sdss u band in monster - # format at this point + colorterm_file_string = 'DES_to_SDSS_band' colorterm_spline = self.get_colorterm_spline(colorterm_file_string, band) - # apply colorterms to transform to target system mag + # apply colorterms to transform to target system mag flux_col = f'monster_DES_{colorterm_spline.source_field}_flux' flux_col_1 = f'monster_DES_{colorterm_spline.source_color_field_1}_flux' flux_col_2 = f'monster_DES_{colorterm_spline.source_color_field_2}_flux' orig_flux = gaia_stars_all[flux_col] orig_flux_err = gaia_stars_all[flux_col + 'Err'] - model_flux = colorterm_spline.apply( + slr_model_flux = colorterm_spline.apply( gaia_stars_all[flux_col_1], gaia_stars_all[flux_col_2], orig_flux, ) # Rescale flux error to keep S/N constant - model_flux_err = model_flux * (orig_flux_err/orig_flux) - - # not sure what colors should be here - # colors = cat_info.get_transformed_mag_colors(cat_stars, band) - # selected = (colors >= -10) & (colors <= 10) - - # only want to insert values that are not nan and are nan - # in gaia_stars_all - model_flux_not_nan = np.isfinite(model_flux) - monster_flag_not_set = gaia_stars_all[f"monster_{target_system_name}_{band}_source_flag"] == -1 - flag = model_flux_not_nan & monster_flag_not_set - gaia_stars_all[flux_col][flag] = model_flux[flag] - gaia_stars_all[flux_col + 'Err'][flag] = model_flux_err[flag] - # Update the flags to denote which survey the flux came - # from: - gaia_stars_all[f"monster_{target_system_name}_{band}_source_flag"][flag] = FLAG_DICT["SLR"] + slr_model_flux_err = slr_model_flux * (orig_flux_err/orig_flux) + + # only want to insert values that are not nan in SLR + slr_model_flux_not_nan = np.isfinite(slr_model_flux) - if len(target_systems_u_transform) > 0: for target_system_name, band in target_systems_u_transform: - colorterm_file_string = 'SDSS_to_'+str(target_system_name)+'_band' - colorterm_spline = self.get_colorterm_spline(colorterm_file_string, band) - # apply colorterms to transform to target system mag - flux_col = f'monster_SDSS_{colorterm_spline.source_field}_flux' - flux_col_1 = f'monster_DES_{colorterm_spline.source_color_field_1}_flux' - flux_col_2 = f'monster_DES_{colorterm_spline.source_color_field_2}_flux' - - orig_flux = gaia_stars_all[flux_col] - orig_flux_err = gaia_stars_all[flux_col + 'Err'] - orig_flags = gaia_stars_all["monster_SDSS_u_source_flag"] - - model_flux = colorterm_spline.apply( - gaia_stars_all[flux_col_1], - gaia_stars_all[flux_col_2], - orig_flux, - ) - - # Rescale flux error to keep S/N constant - model_flux_err = model_flux * (orig_flux_err/orig_flux) + flux_col = f"monster_{target_system_name}_{band}_flux" + if target_system_name == 'SDSS': + # this is null transform + flag = slr_model_flux_not_nan + + gaia_stars_all[flux_col][flag] = slr_model_flux[flag] + gaia_stars_all[flux_col + "Err"][flag] = slr_model_flux_err[flag] + # Update the flags to denote which survey the flux came + gaia_stars_all[flux_col.replace('flux', 'source_flag')][flag] = FLAG_DICT["SLR"] + else: + # we take our SDSS_u flux and transform to target system + colorterm_file_string = 'SDSS_to_'+str(target_system_name)+'_band' + colorterm_spline = self.get_colorterm_spline(colorterm_file_string, band) + + flux_col_1 = f'monster_DES_{colorterm_spline.source_color_field_1}_flux' + flux_col_2 = f'monster_DES_{colorterm_spline.source_color_field_2}_flux' + + orig_flux = slr_model_flux + orig_flux_err = slr_model_flux_err + + model_flux = colorterm_spline.apply( + gaia_stars_all[flux_col_1], + gaia_stars_all[flux_col_2], + orig_flux, + ) + # Rescale flux error to keep S/N constant + model_flux_err = model_flux * (orig_flux_err/orig_flux) + model_flux_not_nan = np.isfinite(model_flux) + flag = model_flux_not_nan + + gaia_stars_all[flux_col][flag] = model_flux[flag] + gaia_stars_all[flux_col + "Err"][flag] = model_flux_err[flag] + # Update the flags to denote which survey the flux came + gaia_stars_all[flux_col.replace('flux', 'source_flag')][flag] = FLAG_DICT["SLR"] + + # next perform non SLR u band transformations to target systems + if len(target_systems_u_transform) > 0: + for cat_info in self.catalog_info_class_list: + # get set of bands for each catalog + bands = set("u").intersection(cat_info.bands) + # if catalog does not have u-band skip + if "u" not in cat_info.bands: + continue + # Read in star cat that has already been transformed to the DES + # system (if it exists). + if os.path.isfile(cat_info.transformed_path+'/'+str(htmid)+'.fits'): + cat_stars = read_stars(cat_info.transformed_path, [htmid], + allow_missing=self.testing_mode) + else: + continue + + for target_system_name, band in target_systems_u_transform: + orig_flux = cat_stars[cat_info.get_transformed_flux_field(band)] + orig_flux_err = cat_stars[cat_info.get_transformed_flux_field(band)+'Err'] + # Match the transformed catalog to Gaia. + idx1, idx2 = esutil.numpy_util.match(gaia_stars_all['id'], + cat_stars['GaiaDR3_id']) + if target_system_name == 'SDSS': + # null transform no color terms in xpu catalog + model_flux = orig_flux[idx2] + model_flux_err = orig_flux_err[idx2] + else: + # we match with monster to get g and r + # and use u band from cat_info + colorterm_file_string = 'SDSS_to_'+str(target_system_name)+'_band' + colorterm_spline = self.get_colorterm_spline(colorterm_file_string, band) + + # apply colorterms to transform to target system mag + + flux_col_1 = f'monster_DES_{colorterm_spline.source_color_field_1}_flux' + flux_col_2 = f'monster_DES_{colorterm_spline.source_color_field_2}_flux' + model_flux = colorterm_spline.apply( + gaia_stars_all[flux_col_1][idx1], + gaia_stars_all[flux_col_2][idx1], + orig_flux[idx2], + ) + # Rescale flux error to keep S/N constant + model_flux_err = model_flux * (orig_flux_err[idx2]/orig_flux[idx2]) model_flux_not_nan = np.isfinite(model_flux) flag = model_flux_not_nan # & selected # Add the fluxes and their errors to the catalog: - gaia_stars_all[f"monster_{target_system_name}_{band}_flux"][flag] = model_flux[flag] - gaia_stars_all[f"monster_{target_system_name}_{band}_fluxErr"][flag] = model_flux_err[flag] - - # not sure what this should be here - # colors = - # cat_info.get_transformed_mag_colors(gaia_stars_all, band) - # selected = - # (colors >= -10) & (colors <= 10) - - # I think new flags should be the same as sdss u band flags - gaia_stars_all[f"monster_{target_system_name}_{band}_source_flag"] = orig_flags + flux_col = f"monster_{target_system_name}_{band}_flux" + gaia_stars_all[flux_col][idx1[flag]] = model_flux[flag] + gaia_stars_all[flux_col + "Err"][idx1[flag]] = model_flux_err[flag] + gaia_stars_all[flux_col.replace('flux', 'source_flag')][idx1[flag]] = cat_info.flag if self.monster_path_inp is None: monster_path = "/sdf/data/rubin/shared/the_monster/sharded_refcats/monster_v2" diff --git a/python/lsst/the/monster/refcats.py b/python/lsst/the/monster/refcats.py index 2da7a7e..c4d8c4d 100644 --- a/python/lsst/the/monster/refcats.py +++ b/python/lsst/the/monster/refcats.py @@ -8,6 +8,7 @@ __all__ = [ + "FLAG_DICT", "DESInfo", "GaiaDR3Info", "GaiaXPInfo", @@ -23,14 +24,13 @@ "GaiaDR3": 0, "DES": 2, "GaiaXP": 4, - "GaiaXPu": 8, # should this be 4? - "PS1": 16, - "SkyMapper": 32, - "VST": 128, - "SDSS": 256, - "SynthLSST": 512, - "LATISS": 1024, - "SLR": 2048, + "PS1": 8, + "SkyMapper": 16, + "VST": 32, + "SDSS": 64, + "SLR": 128, + "SynthLSST": 256, + "LATISS": 512, } @@ -384,7 +384,7 @@ def select_stars(self, catalog, band): class GaiaDR3Info(RefcatInfo): PATH = "/sdf/data/rubin/shared/the_monster/GAIA_DR3/gaia_dr3" NAME = "GaiaDR3" - FLAG = 0 + FLAG = FLAG_DICT[NAME] def get_flux_field(self, band): return f"phot_{band.lower()}_mean_flux" @@ -399,7 +399,7 @@ def get_imz_color_range(self): class GaiaXPInfo(RefcatInfo): PATH = "/sdf/data/rubin/shared/the_monster/sharded_refcats/gaia_xp_ps_des_sdss_sm_20240116" NAME = "GaiaXP" - FLAG = 8 + FLAG = FLAG_DICT[NAME] bands = ["g", "r", "i", "z", "y"] def get_flux_field(self, band): @@ -432,7 +432,7 @@ def get_mag_range(self, band): class DESInfo(RefcatInfo): PATH = "/sdf/data/rubin/shared/the_monster/sharded_refcats/des_y6_calibration_stars_20230511" NAME = "DES" - FLAG = 16 + FLAG = FLAG_DICT[NAME] bands = ["g", "r", "i", "z", "y"] def get_flux_field(self, band): @@ -490,7 +490,7 @@ def select_stars(self, catalog, band): class SkyMapperInfo(RefcatInfo): PATH = "/sdf/data/rubin/shared/the_monster/sharded_refcats/sky_mapper_dr2_20221205" NAME = "SkyMapper" - FLAG = 2 + FLAG = FLAG_DICT[NAME] bands = ["g", "r", "i", "z"] def get_flux_field(self, band): @@ -531,7 +531,7 @@ class PS1Info(RefcatInfo): PATH = "/sdf/group/rubin/datasets/refcats/htm/v1/ps1_pv3_3pi_20170110" TRANSFORMED_PATH = "/sdf/data/rubin/shared/the_monster/sharded_refcats/ps1_transformed" NAME = "PS1" - FLAG = 4 + FLAG = FLAG_DICT[NAME] bands = ["g", "r", "i", "z", "y"] def get_flux_field(self, band): @@ -586,7 +586,7 @@ def select_stars(self, catalog, band): class VSTInfo(RefcatInfo): PATH = "/sdf/data/rubin/shared/the_monster/sharded_refcats/vst_atlas_20221205" NAME = "VST" - FLAG = 1 + FLAG = FLAG_DICT[NAME] bands = ["g", "r", "i", "z"] def get_flux_field(self, band): @@ -625,7 +625,7 @@ def get_sn_range(self, band): class SynthLSSTInfo(RefcatInfo): NAME = "SynthLSST" - FLAG = 32 + FLAG = FLAG_DICT[NAME] bands = ["u", "g", "r", "i", "z", "y"] def get_flux_field(self, band): @@ -640,7 +640,6 @@ def get_imz_color_range(self): class GaiaXPuInfo(GaiaXPInfo): - FLAG = 64 NAME = "GaiaXPu" bands = ["u"] TRANSFORMED_PATH = "/sdf/data/rubin/shared/the_monster/sharded_refcats/gaia_xp_u_20240116_transformed" @@ -675,7 +674,7 @@ def get_transformed_flux_field(self, band): class SDSSInfo(RefcatInfo): PATH = "/sdf/data/rubin/shared/the_monster/sharded_refcats/sdss_16_standards_20221205" NAME = "SDSS" - FLAG = 128 + FLAG = FLAG_DICT[NAME] bands = ["u", "g", "r", "i", "z"] def get_flux_field(self, band): @@ -713,7 +712,7 @@ class SDSSuInfo(SDSSInfo): class LATISSInfo(RefcatInfo): NAME = "LATISS" - FLAG = 256 + FLAG = FLAG_DICT[NAME] bands = ["g", "r", "i", "z", "y"] def get_flux_field(self, band): diff --git a/python/lsst/the/monster/utils.py b/python/lsst/the/monster/utils.py index c8e58a2..b3c8074 100644 --- a/python/lsst/the/monster/utils.py +++ b/python/lsst/the/monster/utils.py @@ -4,6 +4,7 @@ from lsst.afw.table import SimpleCatalog import lsst.afw.table as afwTable +from .refcats import FLAG_DICT __all__ = ["read_stars", "makeRefSchema", "makeRefCat", "makeMonsterSchema", "makeMonsterCat"] @@ -211,8 +212,10 @@ def makeMonsterSchema(gaia_catalog_columns, target_systems): monsterSchema.addField(fluxcolname_err, type='D', doc='error on flux transformed to synthetic system', units='nJy') + + flag_doc = ", ".join([str(FLAG_DICT[key]) + ":" + key for key in FLAG_DICT.keys()]) monsterSchema.addField(flagcolname, type='I', - doc='source of flux (1:VST, 2:Skymapper, 4:PS1, 8:GaiaXP, 16:DES)', + doc=f'source of flux ({flag_doc})', # TO DO: source of flux flags need to be updated units='')