Skip to content

Commit

Permalink
Centralize day_obs calculation in MWI constructor.
Browse files Browse the repository at this point in the history
This prevents the output run code from depending on which time library
we use to calculate day_obs; it only needs to know its value.
  • Loading branch information
kfindeisen committed Jan 26, 2024
1 parent f25ea1c commit 4f840cb
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 18 deletions.
13 changes: 6 additions & 7 deletions python/activator/middleware_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,8 +221,7 @@ def __init__(self, central_butler: Butler, image_bucket: str, visit: FannedOutVi
self.instrument = lsst.obs.base.Instrument.from_string(visit.instrument, central_butler.registry)
self.pipelines = pipelines

# Guard against a processing run starting on one day and ending the next.
self._day_obs = datetime.datetime.now(_DAY_OBS_TZ)
self._day_obs = datetime.datetime.now(_DAY_OBS_TZ).strftime("%Y-%m-%d")

self._init_local_butler(local_repo, [self.instrument.makeUmbrellaCollectionName()], None)
self._prep_collections()
Expand Down Expand Up @@ -625,15 +624,15 @@ def get_key(ref):

def _get_init_output_run(self,
pipeline_file: str,
date: datetime.date) -> str:
date: str) -> str:
"""Generate a deterministic init-output collection name that avoids
configuration conflicts.
Parameters
----------
pipeline_file : `str`
The pipeline file that the run will be used for.
date : `datetime.date`
date : `str`
Date of the processing run (not observation!).
Returns
Expand All @@ -647,15 +646,15 @@ def _get_init_output_run(self,

def _get_output_run(self,
pipeline_file: str,
date: datetime.date) -> str:
date: str) -> str:
"""Generate a deterministic collection name that avoids version or
provenance conflicts.
Parameters
----------
pipeline_file : `str`
The pipeline file that the run will be used for.
date : `datetime.date`
date : `str`
Date of the processing run (not observation!).
Returns
Expand All @@ -666,7 +665,7 @@ def _get_output_run(self,
pipeline_name, _ = os.path.splitext(os.path.basename(pipeline_file))
# Order optimized for S3 bucket -- filter out as many files as soon as possible.
return self.instrument.makeCollectionName(
"prompt", f"output-{date:%Y-%m-%d}", pipeline_name, self._deployment)
"prompt", f"output-{date}", pipeline_name, self._deployment)

def _prep_collections(self):
"""Pre-register output collections in advance of running the pipeline.
Expand Down
16 changes: 5 additions & 11 deletions tests/test_middleware_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -598,17 +598,11 @@ def test_run_pipeline_cascading_exception(self):

def test_get_output_run(self):
filename = "ApPipe.yaml"
for date in [datetime.date.today(), datetime.datetime.today()]:
out_run = self.interface._get_output_run(filename, date)
self.assertEqual(out_run,
f"{instname}/prompt/output-{date.year:04d}-{date.month:02d}-{date.day:02d}"
"/ApPipe/prompt-proto-service-042"
)
init_run = self.interface._get_init_output_run(filename, date)
self.assertEqual(init_run,
f"{instname}/prompt/output-{date.year:04d}-{date.month:02d}-{date.day:02d}"
"/ApPipe/prompt-proto-service-042"
)
date = "2023-01-22"
out_run = self.interface._get_output_run(filename, date)
self.assertEqual(out_run, f"{instname}/prompt/output-2023-01-22/ApPipe/prompt-proto-service-042")
init_run = self.interface._get_init_output_run(filename, date)
self.assertEqual(init_run, f"{instname}/prompt/output-2023-01-22/ApPipe/prompt-proto-service-042")

def _assert_in_collection(self, butler, collection, dataset_type, data_id):
# Pass iff any dataset matches the query, no need to check them all.
Expand Down

0 comments on commit 4f840cb

Please sign in to comment.