From 5eb42df71e36ad8a1ed9f66e7ae75574202fdfe6 Mon Sep 17 00:00:00 2001 From: Peter Yoachim Date: Thu, 12 Sep 2024 16:29:55 -0700 Subject: [PATCH 1/2] remove json block, add ScheduledObservationArray --- .../basis_functions/mask_basis_funcs.py | 2 +- .../scheduler/surveys/ddf_presched.py | 2 - .../scheduler/surveys/long_gap_survey.py | 1 - rubin_scheduler/scheduler/utils/utils.py | 229 ++++++++---------- 4 files changed, 108 insertions(+), 126 deletions(-) diff --git a/rubin_scheduler/scheduler/basis_functions/mask_basis_funcs.py b/rubin_scheduler/scheduler/basis_functions/mask_basis_funcs.py index a4a25eb7..9bc7e253 100644 --- a/rubin_scheduler/scheduler/basis_functions/mask_basis_funcs.py +++ b/rubin_scheduler/scheduler/basis_functions/mask_basis_funcs.py @@ -232,7 +232,7 @@ def _calc_value(self, conditions, indx=None): # so the results are cached and can be used by other surveys is # needed. Technically this could fail if the masked region is # very narrow or shadow time is very large. - future_alt, future_az = conditions.future_alt_az(np.max(conditions.mjd + self.shadow_time)) + future_alt, future_az = conditions.future_alt_az(float(np.max(conditions.mjd)) + self.shadow_time) r_future_alt = IntRounded(future_alt) r_current_alt = IntRounded(conditions.alt) diff --git a/rubin_scheduler/scheduler/surveys/ddf_presched.py b/rubin_scheduler/scheduler/surveys/ddf_presched.py index 880bc037..a138b8cd 100644 --- a/rubin_scheduler/scheduler/surveys/ddf_presched.py +++ b/rubin_scheduler/scheduler/surveys/ddf_presched.py @@ -482,7 +482,6 @@ def generate_ddf_scheduled_obs( obs["target_name"] = "DD:%s" % ddf_name.replace("_a", "_b") obs["science_program"] = "DD" obs["observation_reason"] = "FBS" - obs["json_block"] = "Imaging" obs["mjd_tol"] = mjd_tol obs["dist_tol"] = dist_tol @@ -508,7 +507,6 @@ def generate_ddf_scheduled_obs( obs["target_name"] = "DD:%s" % ddf_name obs["science_program"] = "DD" obs["observation_reason"] = "FBS" - obs["json_block"] = "Imaging" obs["mjd_tol"] = mjd_tol obs["dist_tol"] = dist_tol diff --git a/rubin_scheduler/scheduler/surveys/long_gap_survey.py b/rubin_scheduler/scheduler/surveys/long_gap_survey.py index 15e487a8..0bef330f 100644 --- a/rubin_scheduler/scheduler/surveys/long_gap_survey.py +++ b/rubin_scheduler/scheduler/surveys/long_gap_survey.py @@ -193,7 +193,6 @@ def _schedule_obs(self, observations): sched_array["scheduler_note"] = self.long_name sched_array["target_name"] = "" sched_array["observation_reason"] = "FBS" - sched_array["json_block"] = "Imaging" # Don't let the desired rotSkyPos block the observation. sched_array["rotSkyPos_desired"] = sched_array["rotSkyPos"] sched_array["rotSkyPos"] = np.nan diff --git a/rubin_scheduler/scheduler/utils/utils.py b/rubin_scheduler/scheduler/utils/utils.py index aca407b2..aed3cf7e 100644 --- a/rubin_scheduler/scheduler/utils/utils.py +++ b/rubin_scheduler/scheduler/utils/utils.py @@ -12,6 +12,8 @@ "set_default_nside", "restore_scheduler", "warm_start", + "ObservationArray", + "ScheduledObservationArray", "empty_observation", "scheduled_observation", "gnomonic_project_toxy", @@ -558,21 +560,15 @@ def opsim2obs(self, filename): return self.opsimdf2obs(df) -def empty_observation(n=1): - """Return a numpy array that could be a handy observation record +class ObservationArray(np.ndarray): + """Class to work as an array of observations Parameters ---------- n : `int` Size of array to return. Default 1. - Returns - ------- - empty_observation : `np.array` - - The numpy fields have the following labels. These fields are - required to be set to be a valid observation the model observatory - can execute. + The numpy fields have the following labels. RA : `float` The Right Acension of the observation (center of the field) @@ -627,9 +623,6 @@ def empty_observation(n=1): This maps to observation_reason in the ConsDB, although could be overwritten in JSON BLOCK. Most likely this is just "science" or "FBS" when using the FBS. - json_block : `str` (optional) - The JSON BLOCK id to use to acquire observations. - This is for use by the SchedulerCSC. Notes ----- @@ -649,68 +642,77 @@ def empty_observation(n=1): """ - dtypes = [ - ("ID", int), - ("RA", float), - ("dec", float), - ("mjd", float), - ("flush_by_mjd", float), - ("exptime", float), - ("filter", "U40"), - ("rotSkyPos", float), - ("rotSkyPos_desired", float), - ("nexp", int), - ("airmass", float), - ("FWHM_500", float), - ("FWHMeff", float), - ("FWHM_geometric", float), - ("skybrightness", float), - ("night", int), - ("slewtime", float), - ("visittime", float), - ("slewdist", float), - ("fivesigmadepth", float), - ("alt", float), - ("az", float), - ("pa", float), - ("pseudo_pa", float), - ("clouds", float), - ("moonAlt", float), - ("sunAlt", float), - ("note", "U40"), - ("scheduler_note", "U40"), - ("target_name", "U40"), - ("block_id", int), - ("lmst", float), - ("rotTelPos", float), - ("rotTelPos_backup", float), - ("moonAz", float), - ("sunAz", float), - ("sunRA", float), - ("sunDec", float), - ("moonRA", float), - ("moonDec", float), - ("moonDist", float), - ("solarElong", float), - ("moonPhase", float), - ("cummTelAz", float), - ("scripted_id", int), - ("observation_reason", "U40"), - ("science_program", "U40"), - ("json_block", "U40"), - ] - - result = np.zeros(n, dtype=dtypes) + def __new__(cls, n=1): + dtypes = [ + ("ID", int), + ("RA", float), + ("dec", float), + ("mjd", float), + ("flush_by_mjd", float), + ("exptime", float), + ("filter", "U40"), + ("rotSkyPos", float), + ("rotSkyPos_desired", float), + ("nexp", int), + ("airmass", float), + ("FWHM_500", float), + ("FWHMeff", float), + ("FWHM_geometric", float), + ("skybrightness", float), + ("night", int), + ("slewtime", float), + ("visittime", float), + ("slewdist", float), + ("fivesigmadepth", float), + ("alt", float), + ("az", float), + ("pa", float), + ("pseudo_pa", float), + ("clouds", float), + ("moonAlt", float), + ("sunAlt", float), + ("note", "U40"), + ("scheduler_note", "U40"), + ("target_name", "U40"), + ("block_id", int), + ("lmst", float), + ("rotTelPos", float), + ("rotTelPos_backup", float), + ("moonAz", float), + ("sunAz", float), + ("sunRA", float), + ("sunDec", float), + ("moonRA", float), + ("moonDec", float), + ("moonDist", float), + ("solarElong", float), + ("moonPhase", float), + ("cummTelAz", float), + ("scripted_id", int), + ("observation_reason", "U40"), + ("science_program", "U40"), + ] + obj = np.zeros(n, dtype=dtypes).view(cls) + return obj + + +def empty_observation(n=1): + """For backwards compatibility""" + warnings.warn("Function empty_observation is deprecated, use ObservationArray", FutureWarning) + result = ObservationArray(n=n) return result def scheduled_observation(n=1): - """Make an array to hold pre-scheduling observations + warnings.warn( + "Function scheduled_observation is deprecated, use ScheduledObservationArray", FutureWarning + ) + result = ScheduledObservationArray(n=n) + return result - Returns - ------- - result : np.array +class ScheduledObservationArray(np.ndarray): + """Make an array to hold pre-scheduling observations Note ---- @@ -748,60 +750,43 @@ def scheduled_observation(n=1): """ - # Standard things from the usual observations - names = [ - "ID", - "RA", - "dec", - "mjd", - "flush_by_mjd", - "exptime", - "filter", - "rotSkyPos", - "rotTelPos", - "rotTelPos_backup", - "rotSkyPos_desired", - "nexp", - "scheduler_note", - "target_name", - "science_program", - "observation_reason", - "json_block", - ] - types = [ - int, - float, - float, - float, - float, - float, - "U1", - float, - float, - float, - float, - int, - "U40", - "U40", - "U40", - "U40", - "U40", - ] - names += [ - "mjd_tol", - "dist_tol", - "alt_min", - "alt_max", - "HA_max", - "HA_min", - "sun_alt_max", - "moon_min_distance", - "observed", - "scripted_id", - ] - types += [float, float, float, float, float, float, float, float, bool, int] - result = np.zeros(n, dtype=list(zip(names, types))) - return result + def __new__(cls, n=1): + # Standard things from the usual observations + dtypes1 = [ + ("ID", int), + ("RA", float), + ("dec", float), + ("mjd", float), + ("flush_by_mjd", float), + ("exptime", float), + ("filter", "U1"), + ("rotSkyPos", float), + ("rotTelPos", float), + ("rotTelPos_backup", float), + ("rotSkyPos_desired", float), + ("nexp", int), + ("scheduler_note", "U40"), + ("target_name", "U40"), + ("science_program", "U40"), + ("observation_reason", "U40"), + ] + + # New things not in standard ObservationArray + dtype2 = [ + ("mjd_tol", float), + ("dist_tol", float), + ("alt_min", float), + ("alt_max", float), + ("HA_max", float), + ("HA_min", float), + ("sun_alt_max", float), + ("moon_min_distance", float), + ("observed", bool), + ("scripted_id", int), + ] + + obj = np.zeros(n, dtype=dtypes1 + dtype2).view(cls) + return obj def hp_kd_tree(nside=None, leafsize=100, scale=1e5): From d15c6c50025c8c871b159c22615956d1bec94c79 Mon Sep 17 00:00:00 2001 From: Peter Yoachim Date: Thu, 12 Sep 2024 17:05:03 -0700 Subject: [PATCH 2/2] updating empty_observation to ObservationArray --- .../basis_functions/basis_functions.py | 2 +- .../scheduler/example/example_scheduler.py | 2 - .../scheduler/features/features.py | 8 +-- .../scheduler/schedulers/core_scheduler.py | 4 +- rubin_scheduler/scheduler/sim_runner.py | 6 +- .../scheduler/surveys/base_survey.py | 11 ++-- .../scheduler/surveys/dd_surveys.py | 6 +- .../scheduler/surveys/ddf_presched.py | 8 +-- rubin_scheduler/scheduler/surveys/desc_ddf.py | 4 +- .../scheduler/surveys/field_survey.py | 11 ++-- .../scheduler/surveys/long_gap_survey.py | 4 +- .../scheduler/surveys/pointings_survey.py | 2 +- .../scheduler/surveys/scripted_surveys.py | 4 +- rubin_scheduler/scheduler/surveys/surveys.py | 6 +- .../scheduler/surveys/too_scripted_surveys.py | 10 ++- rubin_scheduler/scheduler/utils/utils.py | 9 +-- tests/scheduler/test_basisfuncs.py | 12 ++-- tests/scheduler/test_coresched.py | 4 +- tests/scheduler/test_detailers.py | 4 +- tests/scheduler/test_example.py | 64 +++++++++++++++++++ tests/scheduler/test_features.py | 18 +++--- tests/scheduler/test_surveys.py | 6 +- tests/scheduler/test_utils.py | 53 +-------------- 23 files changed, 131 insertions(+), 127 deletions(-) create mode 100644 tests/scheduler/test_example.py diff --git a/rubin_scheduler/scheduler/basis_functions/basis_functions.py b/rubin_scheduler/scheduler/basis_functions/basis_functions.py index 7650191a..bbf99da7 100644 --- a/rubin_scheduler/scheduler/basis_functions/basis_functions.py +++ b/rubin_scheduler/scheduler/basis_functions/basis_functions.py @@ -110,7 +110,7 @@ def add_observations_array(self, observations_array, observations_hpid): ---------- observations_array_in : `np.array` An array of completed observations (with columns like - rubin_scheduler.scheduler.utils.empty_observation). + rubin_scheduler.scheduler.utils.ObservationArray). Should be sorted by MJD. observations_hpid_in : `np.array` Same as observations_array_in, but larger and with an diff --git a/rubin_scheduler/scheduler/example/example_scheduler.py b/rubin_scheduler/scheduler/example/example_scheduler.py index 307d1e99..b649e86d 100644 --- a/rubin_scheduler/scheduler/example/example_scheduler.py +++ b/rubin_scheduler/scheduler/example/example_scheduler.py @@ -1184,7 +1184,6 @@ def generate_twilight_near_sun( shadow_minutes=60.0, max_alt=76.0, max_elong=60.0, - az_range=180.0, ignore_obs=["DD", "pair", "long", "blob", "greedy"], filter_dist_weight=0.3, time_to_12deg=25.0, @@ -1323,7 +1322,6 @@ def generate_twilight_near_sun( dither=True, nexp=nexp, detailers=detailer_list, - az_range=az_range, twilight_scale=False, area_required=area_required, ) diff --git a/rubin_scheduler/scheduler/features/features.py b/rubin_scheduler/scheduler/features/features.py index bd44bd7c..b9668cc1 100644 --- a/rubin_scheduler/scheduler/features/features.py +++ b/rubin_scheduler/scheduler/features/features.py @@ -103,7 +103,7 @@ def add_observation(self, observation, indx=None, **kwargs): observation : `np.array`, (1,N) Array of observation information, containing `mjd` for the time. See - `rubin_scheduler.scheduler.utils.empty_observation`. + `rubin_scheduler.scheduler.utils.ObservationArray`. indx : `list`-like of [`int`] The healpixel indices that the observation overlaps. See `rubin_scheduler.utils.HpInLsstFov`. @@ -366,7 +366,7 @@ def __init__(self, scheduler_note=None, survey_name=None): else: self.scheduler_note = scheduler_note # Start out with an empty observation - self.feature = utils.empty_observation() + self.feature = utils.ObservationArray() def add_observations_array(self, observations_array, observations_hpid): if self.scheduler_note is not None: @@ -394,7 +394,7 @@ def __init__(self, sequence_ids=""): self.sequence_ids = sequence_ids # The ids of all sequence # observations... # Start out with an empty observation - self.feature = utils.empty_observation() + self.feature = utils.ObservationArray() send_unused_deprecation_warning(self.__class__.__name__) def add_observation(self, observation, indx=None): @@ -672,7 +672,7 @@ def season_update(self, observation=None, conditions=None): observation : `np.array`, (1,N) Array of observation information, containing `mjd` for the time. See - `rubin_scheduler.scheduler.utils.empty_observation`. + `rubin_scheduler.scheduler.utils.ObservationArray`. conditions : `rubin_scheduler.scheduler.Conditions`, optional A conditions object, containing `mjd`. diff --git a/rubin_scheduler/scheduler/schedulers/core_scheduler.py b/rubin_scheduler/scheduler/schedulers/core_scheduler.py index 8f7c1099..155f6dd7 100644 --- a/rubin_scheduler/scheduler/schedulers/core_scheduler.py +++ b/rubin_scheduler/scheduler/schedulers/core_scheduler.py @@ -15,7 +15,7 @@ HpInComcamFov, HpInLsstFov, IntRounded, - empty_observation, + ObservationArray, set_default_nside, ) from rubin_scheduler.utils import _approx_altaz2pa, _approx_ra_dec2_alt_az, _hpid2_ra_dec, rotation_converter @@ -155,7 +155,7 @@ def add_observation(self, observation): # Catch if someone passed in a slice of an observation # rather than a full observation array if len(observation.shape) == 0: - full_obs = empty_observation() + full_obs = ObservationArray() full_obs[0] = observation observation = full_obs diff --git a/rubin_scheduler/scheduler/sim_runner.py b/rubin_scheduler/scheduler/sim_runner.py index 12588a73..edbeb602 100644 --- a/rubin_scheduler/scheduler/sim_runner.py +++ b/rubin_scheduler/scheduler/sim_runner.py @@ -10,7 +10,7 @@ import pandas as pd from rubin_scheduler.scheduler.schedulers import SimpleFilterSched -from rubin_scheduler.scheduler.utils import SchemaConverter, empty_observation, run_info_table +from rubin_scheduler.scheduler.utils import ObservationArray, SchemaConverter, run_info_table from rubin_scheduler.utils import Site, _approx_altaz2pa, pseudo_parallactic_angle, rotation_converter @@ -81,7 +81,7 @@ def sim_runner( observatory.mjd = mjd end_mjd = mjd + survey_length - observations = empty_observation(n=start_result_size) + observations = ObservationArray(n=start_result_size) mjd_track = mjd + 0 step = 1.0 / 24.0 step_none = step_none / 60.0 / 24.0 # to days @@ -132,7 +132,7 @@ def sim_runner( filter_scheduler.add_observation(completed_obs) counter += 1 if counter == observations.size: - add_observations = empty_observation(n=append_result_size) + add_observations = ObservationArray(n=append_result_size) observations = np.concatenate([observations, add_observations]) if record_rewards: diff --git a/rubin_scheduler/scheduler/surveys/base_survey.py b/rubin_scheduler/scheduler/surveys/base_survey.py index 50625779..d64939d1 100644 --- a/rubin_scheduler/scheduler/surveys/base_survey.py +++ b/rubin_scheduler/scheduler/surveys/base_survey.py @@ -10,8 +10,8 @@ from rubin_scheduler.scheduler.detailers import TrackingInfoDetailer, ZeroRotDetailer from rubin_scheduler.scheduler.utils import ( HpInLsstFov, + ObservationArray, comcam_tessellate, - empty_observation, set_default_nside, thetaphi2xyz, xyz2thetaphi, @@ -151,10 +151,9 @@ def add_observations_array(self, observations_array_in, observations_hpid_in): Parameters ---------- - observations_array_in : np.array - An array of completed observations - (with columns like - rubin_scheduler.scheduler.utils.empty_observation). + observations_array_in : ObservationArray + An array of completed observations, + rubin_scheduler.scheduler.utils.ObservationArray observations_hpid_in : np.array Same as observations_array_in, but larger and with an additional column for HEALpix id. Each observation is @@ -249,7 +248,7 @@ def generate_observations_rough(self, conditions): # latest info, calculate it if not self.reward_checked: self.reward = self.calc_reward_function(conditions) - obs = empty_observation() + obs = ObservationArray() return [obs] def generate_observations(self, conditions): diff --git a/rubin_scheduler/scheduler/surveys/dd_surveys.py b/rubin_scheduler/scheduler/surveys/dd_surveys.py index eebb978e..57381235 100644 --- a/rubin_scheduler/scheduler/surveys/dd_surveys.py +++ b/rubin_scheduler/scheduler/surveys/dd_surveys.py @@ -10,7 +10,7 @@ import rubin_scheduler.scheduler.basis_functions as basis_functions from rubin_scheduler.scheduler import features from rubin_scheduler.scheduler.surveys import BaseSurvey -from rubin_scheduler.scheduler.utils import empty_observation +from rubin_scheduler.scheduler.utils import ObservationArray from rubin_scheduler.utils import ddf_locations, ra_dec2_hpid log = logging.getLogger(__name__) @@ -84,7 +84,7 @@ def __init__( self.observations = [] for num, filtername in zip(nvis, sequence): for j in range(num): - obs = empty_observation() + obs = ObservationArray() obs["filter"] = filtername if filtername == "u": obs["exptime"] = u_exptime @@ -401,7 +401,7 @@ def generate_dd_surveys( for filtername, nvis in zip(filters, nviss): for ra, dec, suffix in zip(r_as, decs, suffixes): for num in range(nvis): - obs = empty_observation() + obs = ObservationArray() obs["filter"] = filtername if filtername == "u": obs["exptime"] = u_exptime diff --git a/rubin_scheduler/scheduler/surveys/ddf_presched.py b/rubin_scheduler/scheduler/surveys/ddf_presched.py index a138b8cd..5d253339 100644 --- a/rubin_scheduler/scheduler/surveys/ddf_presched.py +++ b/rubin_scheduler/scheduler/surveys/ddf_presched.py @@ -6,7 +6,7 @@ import numpy as np from rubin_scheduler.data import get_data_dir -from rubin_scheduler.scheduler.utils import scheduled_observation +from rubin_scheduler.scheduler.utils import ScheduledObservationArray from rubin_scheduler.site_models import Almanac from rubin_scheduler.utils import calc_season, ddf_locations, survey_start_mjd @@ -449,7 +449,7 @@ def generate_ddf_scheduled_obs( for mjd in mjds: for filtername, nvis, nexp in zip(filters, nvis_master, nsnaps): if "EDFS" in ddf_name: - obs = scheduled_observation(n=int(nvis / 2)) + obs = ScheduledObservationArray(n=int(nvis / 2)) obs["RA"] = np.radians(ddfs[ddf_name][0]) obs["dec"] = np.radians(ddfs[ddf_name][1]) obs["mjd"] = mjd @@ -470,7 +470,7 @@ def generate_ddf_scheduled_obs( obs["sun_alt_max"] = sun_alt_max all_scheduled_obs.append(obs) - obs = scheduled_observation(n=int(nvis / 2)) + obs = ScheduledObservationArray(n=int(nvis / 2)) obs["RA"] = np.radians(ddfs[ddf_name.replace("_a", "_b")][0]) obs["dec"] = np.radians(ddfs[ddf_name.replace("_a", "_b")][1]) obs["mjd"] = mjd @@ -495,7 +495,7 @@ def generate_ddf_scheduled_obs( all_scheduled_obs.append(obs) else: - obs = scheduled_observation(n=nvis) + obs = ScheduledObservationArray(n=nvis) obs["RA"] = np.radians(ddfs[ddf_name][0]) obs["dec"] = np.radians(ddfs[ddf_name][1]) obs["mjd"] = mjd diff --git a/rubin_scheduler/scheduler/surveys/desc_ddf.py b/rubin_scheduler/scheduler/surveys/desc_ddf.py index 8121985f..6ab750c9 100644 --- a/rubin_scheduler/scheduler/surveys/desc_ddf.py +++ b/rubin_scheduler/scheduler/surveys/desc_ddf.py @@ -4,7 +4,7 @@ import rubin_scheduler.scheduler.basis_functions as basis_functions from rubin_scheduler.scheduler.surveys import BaseSurvey -from rubin_scheduler.scheduler.utils import empty_observation +from rubin_scheduler.scheduler.utils import ObservationArray class DescDdf(BaseSurvey): @@ -42,7 +42,7 @@ def __init__( self.reward_value = reward_value self.flush_pad = flush_pad / 60.0 / 24.0 # To days - self.simple_obs = empty_observation() + self.simple_obs = ObservationArray() self.simple_obs["RA"] = np.radians(RA) self.simple_obs["dec"] = np.radians(dec) self.simple_obs["exptime"] = exptime diff --git a/rubin_scheduler/scheduler/surveys/field_survey.py b/rubin_scheduler/scheduler/surveys/field_survey.py index ad31d689..74dd9693 100644 --- a/rubin_scheduler/scheduler/surveys/field_survey.py +++ b/rubin_scheduler/scheduler/surveys/field_survey.py @@ -9,7 +9,7 @@ from rubin_scheduler.utils import ra_dec2_hpid from ..features import LastObservation, NObsSurvey -from ..utils import empty_observation +from ..utils import ObservationArray from . import BaseSurvey @@ -176,7 +176,7 @@ def __init__( self.observations = [] for filtername in sequence: for j in range(nvisits[filtername]): - obs = empty_observation() + obs = ObservationArray() obs["filter"] = filtername obs["exptime"] = exptimes[filtername] obs["RA"] = self.ra @@ -266,10 +266,9 @@ def add_observations_array(self, observations_array_in, observations_hpid_in): Parameters ---------- - observations_array_in : np.array - An array of completed observations - (with columns like - rubin_scheduler.scheduler.utils.empty_observation). + observations_array_in : ObservationArray + An array of completed observations, + rubin_scheduler.scheduler.utils.ObservationArray observations_hpid_in : np.array Same as observations_array_in, but larger and with an additional column for HEALpix id. Each observation is diff --git a/rubin_scheduler/scheduler/surveys/long_gap_survey.py b/rubin_scheduler/scheduler/surveys/long_gap_survey.py index 0bef330f..0d9def08 100644 --- a/rubin_scheduler/scheduler/surveys/long_gap_survey.py +++ b/rubin_scheduler/scheduler/surveys/long_gap_survey.py @@ -7,7 +7,7 @@ import pandas as pd from rubin_scheduler.scheduler.surveys import BaseSurvey -from rubin_scheduler.scheduler.utils import scheduled_observation +from rubin_scheduler.scheduler.utils import ScheduledObservationArray from rubin_scheduler.utils import Site, _approx_ra_dec2_alt_az log = logging.getLogger(__name__) @@ -125,7 +125,7 @@ def _schedule_obs(self, observations): # If the incoming observation needs to have something # scheduled later if np.size(need_to_observe) > 0: - sched_array = scheduled_observation(n=need_to_observe.size) + sched_array = ScheduledObservationArray(n=need_to_observe.size) for dt in np.intersect1d(observations.dtype.names, sched_array.dtype.names): if np.size(observations) == 1: sched_array[dt] = observations[dt] diff --git a/rubin_scheduler/scheduler/surveys/pointings_survey.py b/rubin_scheduler/scheduler/surveys/pointings_survey.py index 2ec13ecd..d86a7f70 100644 --- a/rubin_scheduler/scheduler/surveys/pointings_survey.py +++ b/rubin_scheduler/scheduler/surveys/pointings_survey.py @@ -21,7 +21,7 @@ class PointingsSurvey(BaseSurvey): ---------- observations : `np.array` An array of observations, from e.g., - rubin_scheduler.scheduler.utils.empty_observation + rubin_scheduler.scheduler.utils.ObservationArray expect "RA", "dec", and "note" to be filled, other columns ignored. gap_min : `float` The minimum gap to force between observations of the same diff --git a/rubin_scheduler/scheduler/surveys/scripted_surveys.py b/rubin_scheduler/scheduler/surveys/scripted_surveys.py index c75acaf1..9254a09c 100644 --- a/rubin_scheduler/scheduler/surveys/scripted_surveys.py +++ b/rubin_scheduler/scheduler/surveys/scripted_surveys.py @@ -6,7 +6,7 @@ import numpy as np from rubin_scheduler.scheduler.surveys import BaseSurvey -from rubin_scheduler.scheduler.utils import empty_observation, set_default_nside +from rubin_scheduler.scheduler.utils import ObservationArray, set_default_nside from rubin_scheduler.utils import _angular_separation, _approx_ra_dec2_alt_az log = logging.getLogger(__name__) @@ -164,7 +164,7 @@ def calc_reward_function(self, conditions): def _slice2obs(self, obs_row): """take a slice and return a full observation object""" - observation = empty_observation() + observation = ObservationArray() for key in [ "RA", "dec", diff --git a/rubin_scheduler/scheduler/surveys/surveys.py b/rubin_scheduler/scheduler/surveys/surveys.py index 787f4ce7..4032694f 100644 --- a/rubin_scheduler/scheduler/surveys/surveys.py +++ b/rubin_scheduler/scheduler/surveys/surveys.py @@ -7,7 +7,7 @@ import numpy as np from rubin_scheduler.scheduler.surveys import BaseMarkovSurvey -from rubin_scheduler.scheduler.utils import empty_observation, int_binned_stat, order_observations +from rubin_scheduler.scheduler.utils import ObservationArray, int_binned_stat, order_observations from rubin_scheduler.utils import _angular_separation, _hpid2_ra_dec, hp_grow_argsort @@ -87,7 +87,7 @@ def generate_observations_rough(self, conditions): best_fields = np.unique(self.hp2fields[best_hp]) observations = [] for field in best_fields: - obs = empty_observation() + obs = ObservationArray() obs["RA"] = self.fields["RA"][field] obs["dec"] = self.fields["dec"][field] obs["rotSkyPos"] = 0.0 @@ -505,7 +505,7 @@ def generate_observations_rough(self, conditions): for i, indx in enumerate(better_order): field = self.best_fields[indx] - obs = empty_observation() + obs = ObservationArray() obs["RA"] = self.fields["RA"][field] obs["dec"] = self.fields["dec"][field] obs["rotSkyPos"] = 0.0 diff --git a/rubin_scheduler/scheduler/surveys/too_scripted_surveys.py b/rubin_scheduler/scheduler/surveys/too_scripted_surveys.py index 6aa0e1b2..20e9bdcf 100644 --- a/rubin_scheduler/scheduler/surveys/too_scripted_surveys.py +++ b/rubin_scheduler/scheduler/surveys/too_scripted_surveys.py @@ -6,9 +6,9 @@ from rubin_scheduler.scheduler.surveys import BaseMarkovSurvey, ScriptedSurvey from rubin_scheduler.scheduler.utils import ( + ScheduledObservationArray, comcam_tessellate, order_observations, - scheduled_observation, thetaphi2xyz, xyz2thetaphi, ) @@ -169,10 +169,8 @@ def set_script(self, obs_wanted, append=True): """ Parameters ---------- - obs_wanted : np.array - The observations that should be executed. Needs to have - columns with dtype names: - Should be from lsst.sim.scheduler.utils.scheduled_observation + obs_wanted : rubin_scheduler.scheduler.utils.ScheduledObservationArray + The observations that should be executed. append : bool Should the obs_wanted be appended to any script already set? """ @@ -363,7 +361,7 @@ def _new_event(self, target_o_o, conditions): if exptime > 119: nexp = int(np.round(exptime / 30.0)) - obs = scheduled_observation(ras.size) + obs = ScheduledObservationArray(ras.size) obs["RA"] = ras obs["dec"] = decs obs["mjd"] = mjd0 + time diff --git a/rubin_scheduler/scheduler/utils/utils.py b/rubin_scheduler/scheduler/utils/utils.py index aed3cf7e..72718cab 100644 --- a/rubin_scheduler/scheduler/utils/utils.py +++ b/rubin_scheduler/scheduler/utils/utils.py @@ -194,7 +194,7 @@ def restore_scheduler(observation_id, scheduler, observatory, in_obs, filter_sch The observaotry object in_obs : np.array or str Array of observations (formated like - rubin_scheduler.scheduler.empty_observation). If a string, + rubin_scheduler.scheduler.ObservationArray). If a string, assumed to be a file and SchemaConverter is used to load it. filter_sched : rubin_scheduler.scheduler.scheduler object The filter scheduler. Note that we don't look up the official @@ -214,10 +214,6 @@ def restore_scheduler(observation_id, scheduler, observatory, in_obs, filter_sch good_obs = np.where(observations["ID"] <= observation_id)[0] observations = observations[good_obs] - # replay the observations back into the scheduler - # In the future, may be able to replace this with a - # faster .add_observations_array method. - if fast: scheduler.add_observations_array(observations) obs = observations[-1] @@ -422,7 +418,6 @@ def __init__(self): "target_name": "target_name", "science_program": "science_program", "observation_reason": "observation_reason", - "json_block": "json_block", } # For backwards compatibility self.backwards = {"target": "target_name"} @@ -535,7 +530,7 @@ def opsimdf2obs(self, df) -> np.recarray: df = df.rename(index=str, columns=self.convert_dict) - blank = empty_observation() + blank = ObservationArray() final_result = np.empty(df.shape[0], dtype=blank.dtype) # XXX-ugh, there has to be a better way. for key in final_result.dtype.names: diff --git a/tests/scheduler/test_basisfuncs.py b/tests/scheduler/test_basisfuncs.py index 3c91f659..b6d5be28 100644 --- a/tests/scheduler/test_basisfuncs.py +++ b/tests/scheduler/test_basisfuncs.py @@ -5,7 +5,7 @@ import rubin_scheduler.scheduler.basis_functions as basis_functions from rubin_scheduler.scheduler.features import Conditions -from rubin_scheduler.scheduler.utils import empty_observation +from rubin_scheduler.scheduler.utils import ObservationArray class TestBasis(unittest.TestCase): @@ -18,7 +18,7 @@ def test_visit_repeat_basis_function(self): delta = 30.0 / 60.0 / 24.0 # Add 1st observation, should still be zero - obs = empty_observation() + obs = ObservationArray() obs["filter"] = "r" obs["mjd"] = 59000.0 conditions = Conditions() @@ -38,7 +38,7 @@ def test_visit_repeat_basis_function(self): def test_force_delay(self): bf = basis_functions.ForceDelayBasisFunction(days_delay=3.0, scheduler_note="survey") - obs = empty_observation() + obs = ObservationArray() obs["scheduler_note"] = "not_match" obs["mjd"] = 10 bf.add_observation(obs) @@ -67,7 +67,7 @@ def test_visit_gap(self): # default is feasible assert visit_gap.check_feasibility(conditions=conditions) - observation = empty_observation() + observation = ObservationArray() observation["filter"] = "r" observation["scheduler_note"] = "foo" observation["mjd"] = 59000.0 @@ -97,7 +97,7 @@ def test_visit_gap_with_filter(self): # default is feasible assert visit_gap.check_feasibility(conditions=conditions) - observation = empty_observation() + observation = ObservationArray() observation["filter"] = "r" observation["scheduler_note"] = "foo" observation["mjd"] = 59000.0 @@ -133,7 +133,7 @@ def test_visit_gap_with_multiple_filters(self): # default is feasible assert visit_gap.check_feasibility(conditions=conditions) - observation = empty_observation() + observation = ObservationArray() observation["filter"] = "r" observation["scheduler_note"] = "foo" observation["mjd"] = 59000.0 diff --git a/tests/scheduler/test_coresched.py b/tests/scheduler/test_coresched.py index 9e94ac97..c1f6faf7 100644 --- a/tests/scheduler/test_coresched.py +++ b/tests/scheduler/test_coresched.py @@ -7,7 +7,7 @@ import rubin_scheduler.scheduler.surveys as surveys from rubin_scheduler.scheduler.model_observatory import ModelObservatory from rubin_scheduler.scheduler.schedulers import CoreScheduler -from rubin_scheduler.scheduler.utils import empty_observation, generate_all_sky +from rubin_scheduler.scheduler.utils import ObservationArray, generate_all_sky class TestCoreSched(unittest.TestCase): @@ -73,7 +73,7 @@ def test_add_obs(self): survey = surveys.GreedySurvey(bfs, [0.0]) scheduler = CoreScheduler([survey]) - obs = empty_observation() + obs = ObservationArray() obs["scheduler_note"] = "survey" obs["mjd"] = 100 diff --git a/tests/scheduler/test_detailers.py b/tests/scheduler/test_detailers.py index 984058a7..9adcac82 100644 --- a/tests/scheduler/test_detailers.py +++ b/tests/scheduler/test_detailers.py @@ -2,13 +2,13 @@ import rubin_scheduler.scheduler.detailers as detailers from rubin_scheduler.scheduler.features import Conditions -from rubin_scheduler.scheduler.utils import empty_observation +from rubin_scheduler.scheduler.utils import ObservationArray class TestDetailers(unittest.TestCase): def test_random_filter(self): - obs = empty_observation(1) + obs = ObservationArray(1) obs["filter"] = "r" det = detailers.RandomFilterDetailer(filters="iz") diff --git a/tests/scheduler/test_example.py b/tests/scheduler/test_example.py new file mode 100644 index 00000000..5d548774 --- /dev/null +++ b/tests/scheduler/test_example.py @@ -0,0 +1,64 @@ +import os +import unittest + +import numpy as np + +from rubin_scheduler.data import get_data_dir +from rubin_scheduler.scheduler.example import example_scheduler, run_sched +from rubin_scheduler.utils import survey_start_mjd + + +class TestExample(unittest.TestCase): + + @unittest.skipUnless( + os.path.isfile(os.path.join(get_data_dir(), "scheduler/dust_maps/dust_nside_32.npz")), + "Test data not available.", + ) + def test_example(self): + """Test the example scheduler executes all the expected surveys""" + mjd_start = survey_start_mjd() + scheduler = example_scheduler(mjd_start=mjd_start) + observatory, scheduler, observations = run_sched(scheduler, mjd_start=mjd_start, survey_length=7) + u_notes = np.unique(observations["scheduler_note"]) + + # Note that some of these may change and need to be updated if + # survey start date changes, e.g., different DDFs in season, + # or different lunar phase means different filters get picked + # for the blobs + notes_to_check = [ + "DD:COSMOS", + "blob_long, gr, a", + "blob_long, gr, b", + "greedy", + "long", + "pair_15, iz, a", + "pair_15, iz, b", + "pair_15, ri, a", + "pair_15, ri, b", + "pair_15, yy, a", + "pair_15, yy, b", + "pair_33, gr, a", + "pair_33, gr, b", + "pair_33, ri, a", + "pair_33, ug, a", + "pair_33, ug, b", + "pair_33, yy, a", + "pair_33, yy, b", + "pair_33, zy, a", + "pair_33, zy, b", + "twilight_near_sun, 0", + "twilight_near_sun, 1", + "twilight_near_sun, 2", + "twilight_near_sun, 3", + ] + + for note in notes_to_check: + assert note in u_notes + + for note in u_notes: + # If this fails, time to add something to notes_to_check + assert note in u_notes + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/scheduler/test_features.py b/tests/scheduler/test_features.py index 45a6ff37..80d943b1 100644 --- a/tests/scheduler/test_features.py +++ b/tests/scheduler/test_features.py @@ -5,7 +5,7 @@ import rubin_scheduler.scheduler.features as features from rubin_scheduler.scheduler.model_observatory import ModelObservatory -from rubin_scheduler.scheduler.utils import HpInLsstFov, empty_observation +from rubin_scheduler.scheduler.utils import HpInLsstFov, ObservationArray from rubin_scheduler.skybrightness_pre import dark_sky from rubin_scheduler.utils import survey_start_mjd @@ -13,7 +13,7 @@ def make_observations_list(nobs=1): observations_list = [] for i in range(0, nobs): - observation = empty_observation() + observation = ObservationArray() observation["mjd"] = survey_start_mjd() + i * 30 / 60 / 60 / 24 observation["RA"] = np.radians(30) observation["dec"] = np.radians(-20) @@ -66,7 +66,7 @@ def test_pair_in_night(self): delta = 30.0 / 60.0 / 24.0 # Add 1st observation, feature should still be zero - obs = empty_observation() + obs = ObservationArray() obs["filter"] = "r" obs["mjd"] = 59000.0 pin.add_observation(obs, indx=indx) @@ -82,7 +82,7 @@ def test_pair_in_night(self): self.assertEqual(np.max(pin.feature), 2.0) def test_note_in_night(self): - obs = empty_observation() + obs = ObservationArray() plain_feature = features.NoteInNight() plain_feature.add_observation(obs) @@ -135,7 +135,7 @@ def test_conditions(self): def test_note_last_observed(self): note_last_observed = features.NoteLastObserved(note="test") - observation = empty_observation() + observation = ObservationArray() observation["mjd"] = 59000.0 note_last_observed.add_observation(observation=observation) @@ -158,7 +158,7 @@ def test_note_last_observed_with_filter(self): filtername="r", ) - observation = empty_observation() + observation = ObservationArray() observation["mjd"] = 59000.0 note_last_observed.add_observation(observation=observation) @@ -209,7 +209,7 @@ def test_NObservationsCurrentSeason(self): self.assertTrue(np.all(season_feature.season == season)) # And do the same for observations - check update_seasons works # Make some observations - observations = [empty_observation(), empty_observation()] + observations = [ObservationArray(), ObservationArray()] observations[0]["mjd"] = mjd_start observations[0]["ID"] = 0 observations[1]["mjd"] = mjd_start + 100 @@ -252,7 +252,7 @@ def test_NObservationsCurrentSeason(self): self.assertTrue(np.all(np.delete(season_feature.feature, indxs) == 0)) # Add an observation at a different point on the sky, but where # season should not turn oer for ra above yet. - observations.append(empty_observation()) + observations.append(ObservationArray()) observations[-1]["mjd"] = mjd_start + 10 observations[-1]["RA"] = np.radians(50) observations[-1]["dec"] = np.radians(-20) @@ -315,7 +315,7 @@ def test_NObservationsCurrentSeason(self): season_feature = features.NObservationsCurrentSeason( nside=nside, mjd_start=mjd_start, seeing_fwhm_max=1.0 ) - observation = empty_observation() + observation = ObservationArray() observation["mjd"] = mjd_start observation["RA"] = np.radians(30) observation["dec"] = np.radians(-20) diff --git a/tests/scheduler/test_surveys.py b/tests/scheduler/test_surveys.py index 01881df6..d646e225 100644 --- a/tests/scheduler/test_surveys.py +++ b/tests/scheduler/test_surveys.py @@ -8,14 +8,14 @@ import rubin_scheduler.scheduler.surveys as surveys from rubin_scheduler.scheduler.basis_functions import SimpleArrayBasisFunction from rubin_scheduler.scheduler.model_observatory import ModelObservatory -from rubin_scheduler.scheduler.utils import HpInLsstFov, empty_observation, set_default_nside +from rubin_scheduler.scheduler.utils import HpInLsstFov, ObservationArray, set_default_nside from rubin_scheduler.utils import survey_start_mjd def make_observations_list(nobs=1): observations_list = [] for i in range(0, nobs): - observation = empty_observation() + observation = ObservationArray() observation["mjd"] = survey_start_mjd() + i * 30 / 60 / 60 / 24 observation["RA"] = np.radians(30) observation["dec"] = np.radians(-20) @@ -146,7 +146,7 @@ def test_pointings_survey(self): # Make a ring of points near the equator so # some should always be visible - fields = empty_observation(n=10) + fields = ObservationArray(n=10) fields["RA"] = np.arange(0, fields.size) / fields.size * 2.0 * np.pi fields["dec"] = -0.01 fields["scheduler_note"] = ["test%i" % ind for ind in range(fields.size)] diff --git a/tests/scheduler/test_utils.py b/tests/scheduler/test_utils.py index d780c435..abfe6d77 100644 --- a/tests/scheduler/test_utils.py +++ b/tests/scheduler/test_utils.py @@ -9,8 +9,8 @@ from rubin_scheduler.scheduler.example import example_scheduler, run_sched from rubin_scheduler.scheduler.model_observatory import KinemModel, ModelObservatory from rubin_scheduler.scheduler.utils import ( + ObservationArray, SchemaConverter, - empty_observation, make_rolling_footprints, restore_scheduler, run_info_table, @@ -33,55 +33,6 @@ def test_nside(self): _ = example_scheduler(mjd_start=mjd_start, nside=64) _ = example_scheduler(mjd_start=mjd_start, nside=8) - @unittest.skipUnless( - os.path.isfile(os.path.join(get_data_dir(), "scheduler/dust_maps/dust_nside_32.npz")), - "Test data not available.", - ) - def test_example(self): - """Test the example scheduler executes all the expected surveys""" - mjd_start = survey_start_mjd() - scheduler = example_scheduler(mjd_start=mjd_start) - observatory, scheduler, observations = run_sched(scheduler, mjd_start=mjd_start, survey_length=7) - u_notes = np.unique(observations["scheduler_note"]) - - # Note that some of these may change and need to be updated if - # survey start date changes, e.g., different DDFs in season, - # or different lunar phase means different filters get picked - # for the blobs - notes_to_check = [ - "DD:COSMOS", - "blob_long, gr, a", - "blob_long, gr, b", - "greedy", - "long", - "pair_15, iz, a", - "pair_15, iz, b", - "pair_15, ri, a", - "pair_15, ri, b", - "pair_15, yy, a", - "pair_15, yy, b", - "pair_33, gr, a", - "pair_33, gr, b", - "pair_33, ri, a", - "pair_33, ug, a", - "pair_33, ug, b", - "pair_33, yy, a", - "pair_33, yy, b", - "pair_33, zy, a", - "pair_33, zy, b", - "twilight_near_sun, 0", - "twilight_near_sun, 1", - "twilight_near_sun, 2", - "twilight_near_sun, 3", - ] - - for note in notes_to_check: - assert note in u_notes - - for note in u_notes: - # If this fails, time to add something to notes_to_check - assert note in u_notes - @unittest.skipUnless( os.path.isfile(os.path.join(get_data_dir(), "scheduler/dust_maps/dust_nside_32.npz")), "Test data not available.", @@ -344,7 +295,7 @@ def test_schema_convert(self): sc = SchemaConverter() n = 20 - obs = empty_observation(n=n) + obs = ObservationArray(n=n) obs["ID"] = np.arange(n) # check that we can write observations to a database