From c69c80da682b86ed16002b66c5bb03552f97f9ac Mon Sep 17 00:00:00 2001 From: Dilan Pathirana Date: Thu, 1 Feb 2024 14:33:43 +0100 Subject: [PATCH 1/6] let users supply calibration results --- pypesto/select/method.py | 20 ++++++++++++++++++-- pypesto/select/model_problem.py | 6 ++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/pypesto/select/method.py b/pypesto/select/method.py index 56e48c7ef..0c768d26c 100644 --- a/pypesto/select/method.py +++ b/pypesto/select/method.py @@ -244,6 +244,7 @@ def __init__( # TODO deprecated model_to_pypesto_problem_method: Callable[[Any], Problem] = None, model_problem_options: dict = None, + previously_calibrated_models=None, ): """Arguments are used in every `__call__`, unless overridden.""" self.petab_select_problem = petab_select_problem @@ -255,6 +256,10 @@ def __init__( self.select_first_improvement = select_first_improvement self.startpoint_latest_mle = startpoint_latest_mle + self.previously_calibrated_models = {} + if previously_calibrated_models is not None: + self.previously_calibrated_models = previously_calibrated_models + self.logger = MethodLogger() # TODO deprecated @@ -381,8 +386,19 @@ def __call__( # `self.select_first_improvement`) newly_calibrated_models = {} for candidate_model in candidate_space.models: - # autoruns calibration - self.new_model_problem(model=candidate_model) + # If the user has previously calibrated this model, + # skip calibration. + if candidate_model.get_hash() in self.previously_calibrated_models: + _model_problem = self.new_model_problem( + model=candidate_model, + autorun=False, + ) + _model_problem.set_result_from_model( + self.previously_calibrated_models[candidate_model.get_hash()] + ) + else: + self.new_model_problem(model=candidate_model) + newly_calibrated_models[ candidate_model.get_hash() ] = candidate_model diff --git a/pypesto/select/model_problem.py b/pypesto/select/model_problem.py index bdaac904b..06006462d 100644 --- a/pypesto/select/model_problem.py +++ b/pypesto/select/model_problem.py @@ -149,6 +149,12 @@ def minimize(self) -> Result: **self.minimize_options, ) + def set_result_from_model(self, model): + self.model.criteria = model.criteria + self.model.estimated_parameters = model.estimated_parameters + if self.postprocessor is not None: + self.postprocessor(self) + def set_result(self, result: Result): """Postprocess a result. From 5683f9db22e2bfe7b5216ae4a199d6663f1be23c Mon Sep 17 00:00:00 2001 From: dilpath Date: Wed, 27 Mar 2024 17:51:13 +0100 Subject: [PATCH 2/6] handle calibrated model via petab select --- pypesto/select/method.py | 36 +++++++++++++++++++-------------- pypesto/select/model_problem.py | 6 ------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/pypesto/select/method.py b/pypesto/select/method.py index a7ebea7dc..4ad6269df 100644 --- a/pypesto/select/method.py +++ b/pypesto/select/method.py @@ -213,6 +213,11 @@ class MethodCaller: Specify the predecessor (initial) model for the model selection algorithm. If ``None``, then the algorithm will generate an initial predecessor model if required. + user_calibrated_models: + Supply calibration results for models yourself, as a list of models. + If a model with the same hash is encountered in the current model + selection run, and the user-supplied calibrated model has the + `criterion` value set, the model will not be calibrated again. select_first_improvement: If ``True``, model selection will terminate as soon as a better model is found. If `False`, all candidate models will be tested. @@ -245,7 +250,7 @@ def __init__( # TODO deprecated model_to_pypesto_problem_method: Callable[[Any], Problem] = None, model_problem_options: dict = None, - previously_calibrated_models=None, + user_calibrated_models: list[Model] = None, ): """Arguments are used in every `__call__`, unless overridden.""" self.petab_select_problem = petab_select_problem @@ -257,9 +262,11 @@ def __init__( self.select_first_improvement = select_first_improvement self.startpoint_latest_mle = startpoint_latest_mle - self.previously_calibrated_models = {} - if previously_calibrated_models is not None: - self.previously_calibrated_models = previously_calibrated_models + self.user_calibrated_models = {} + if user_calibrated_models is not None: + self.user_calibrated_models = { + model.get_hash(): model for model in user_calibrated_models + } self.logger = MethodLogger() @@ -379,6 +386,7 @@ def __call__( newly_calibrated_models=newly_calibrated_models, excluded_model_hashes=self.calibrated_models.keys(), criterion=self.criterion, + user_calibrated_models=self.user_calibrated_models, ) predecessor_model = self.candidate_space.predecessor_model @@ -389,17 +397,15 @@ def __call__( # `self.select_first_improvement`) newly_calibrated_models = {} for candidate_model in candidate_space.models: - # If the user has previously calibrated this model, - # skip calibration. - if candidate_model.get_hash() in self.previously_calibrated_models: - _model_problem = self.new_model_problem( - model=candidate_model, - autorun=False, - ) - _model_problem.set_result_from_model( - self.previously_calibrated_models[candidate_model.get_hash()] - ) - else: + # PEtab Select will set the criterion if the user has provided + # a previously calibrated copy of this model, in which case + # calibration can be skipped. + none_if_uncalibrated = candidate_model.get_criterion( + criterion=self.criterion, + compute=True, + raise_on_failure=False, + ) + if none_if_uncalibrated is None: self.new_model_problem(model=candidate_model) newly_calibrated_models[ diff --git a/pypesto/select/model_problem.py b/pypesto/select/model_problem.py index 57515fc70..cef94c0e3 100644 --- a/pypesto/select/model_problem.py +++ b/pypesto/select/model_problem.py @@ -150,12 +150,6 @@ def minimize(self) -> Result: **self.minimize_options, ) - def set_result_from_model(self, model): - self.model.criteria = model.criteria - self.model.estimated_parameters = model.estimated_parameters - if self.postprocessor is not None: - self.postprocessor(self) - def set_result(self, result: Result): """Postprocess a result. From 7fe5782b9dfdd608c7dd1175db2668391760760f Mon Sep 17 00:00:00 2001 From: dilpath Date: Wed, 27 Mar 2024 18:03:50 +0100 Subject: [PATCH 3/6] custom petab select branch --- setup.cfg | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 071142c2b..4672c2bb7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -160,10 +160,12 @@ example = notebook >= 6.1.4 benchmark_models_petab @ git+https://github.com/Benchmarking-Initiative/Benchmark-Models-PEtab.git@master#subdirectory=src/python select = - # Remove when vis is moved to PEtab Select version + # Remove when vis is moved to PEtab Select networkx >= 2.5.1 # End remove - petab-select >= 0.1.12 + #petab-select >= 0.1.12 + # FIXME before merge + petab-select @ git+https://github.com/PEtab-dev/petab_select.git@user_calibrated_models test = pytest >= 5.4.3 pytest-cov >= 2.10.0 From 3b51af9bc2a641d6c9ba2dd85fe1b9a563860ca4 Mon Sep 17 00:00:00 2001 From: dilpath Date: Thu, 28 Mar 2024 23:40:03 +0100 Subject: [PATCH 4/6] functionality moved to petab-select --- pypesto/select/method.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/pypesto/select/method.py b/pypesto/select/method.py index 4ad6269df..f19d88eaf 100644 --- a/pypesto/select/method.py +++ b/pypesto/select/method.py @@ -397,15 +397,23 @@ def __call__( # `self.select_first_improvement`) newly_calibrated_models = {} for candidate_model in candidate_space.models: - # PEtab Select will set the criterion if the user has provided - # a previously calibrated copy of this model, in which case - # calibration can be skipped. - none_if_uncalibrated = candidate_model.get_criterion( - criterion=self.criterion, - compute=True, - raise_on_failure=False, - ) - if none_if_uncalibrated is None: + if ( + candidate_model.get_criterion( + criterion=self.criterion, + compute=True, + raise_on_failure=False, + ) + is not None + ): + self.logger.log( + message=( + "Unexpected calibration result already available for " + f"model: `{candidate_model.get_hash()}`. Skipping " + "calibration." + ), + level="warning", + ) + else: self.new_model_problem(model=candidate_model) newly_calibrated_models[ From 01802e2440e95dbc3e99955e8669ea2ebd007fdf Mon Sep 17 00:00:00 2001 From: dilpath Date: Mon, 7 Oct 2024 18:24:51 +0200 Subject: [PATCH 5/6] update method for next petab_select version --- pypesto/select/method.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/pypesto/select/method.py b/pypesto/select/method.py index f19d88eaf..0717d15c3 100644 --- a/pypesto/select/method.py +++ b/pypesto/select/method.py @@ -8,6 +8,9 @@ import numpy as np import petab_select from petab_select import ( + CANDIDATE_SPACE, + MODELS, + PREDECESSOR_MODEL, VIRTUAL_INITIAL_MODEL, CandidateSpace, Criterion, @@ -378,27 +381,27 @@ def __call__( # All calibrated models in this iteration (see second return value). self.logger.new_selection() - candidate_space = petab_select.ui.candidates( + iteration = petab_select.ui.start_iteration( problem=self.petab_select_problem, candidate_space=self.candidate_space, limit=self.limit, - calibrated_models=self.calibrated_models, - newly_calibrated_models=newly_calibrated_models, - excluded_model_hashes=self.calibrated_models.keys(), + # FIXME confirm old results are reproducible after this change + # calibrated_models=self.calibrated_models, + # newly_calibrated_models=newly_calibrated_models, + # excluded_model_hashes=self.calibrated_models.keys(), criterion=self.criterion, user_calibrated_models=self.user_calibrated_models, ) - predecessor_model = self.candidate_space.predecessor_model - if not candidate_space.models: + if not iteration[MODELS]: raise StopIteration("No valid models found.") # TODO parallelize calibration (maybe not sensible if # `self.select_first_improvement`) newly_calibrated_models = {} - for candidate_model in candidate_space.models: + for model in iteration[MODELS]: if ( - candidate_model.get_criterion( + model.get_criterion( criterion=self.criterion, compute=True, raise_on_failure=False, @@ -408,27 +411,30 @@ def __call__( self.logger.log( message=( "Unexpected calibration result already available for " - f"model: `{candidate_model.get_hash()}`. Skipping " + f"model: `{model.get_hash()}`. Skipping " "calibration." ), level="warning", ) else: - self.new_model_problem(model=candidate_model) + self.new_model_problem(model=model) - newly_calibrated_models[ - candidate_model.get_hash() - ] = candidate_model + newly_calibrated_models[model.get_hash()] = model method_signal = self.handle_calibrated_model( - model=candidate_model, - predecessor_model=predecessor_model, + model=model, + predecessor_model=iteration[PREDECESSOR_MODEL], ) if method_signal.proceed == MethodSignalProceed.STOP: break - self.calibrated_models.update(newly_calibrated_models) + iteration_results = petab_select.ui.end_iteration( + candidate_space=iteration[CANDIDATE_SPACE], + newly_calibrated_models=newly_calibrated_models, + ) + + self.calibrated_models.update(iteration_results[MODELS]) - return predecessor_model, newly_calibrated_models + return iteration[PREDECESSOR_MODEL], iteration_results[MODELS] def handle_calibrated_model( self, From f221448ac57ffe19385d76162eaf8b81c04a46cb Mon Sep 17 00:00:00 2001 From: dilpath Date: Mon, 7 Oct 2024 23:18:30 +0200 Subject: [PATCH 6/6] update for petab select --- pypesto/select/method.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pypesto/select/method.py b/pypesto/select/method.py index 0717d15c3..a170497f0 100644 --- a/pypesto/select/method.py +++ b/pypesto/select/method.py @@ -11,6 +11,7 @@ CANDIDATE_SPACE, MODELS, PREDECESSOR_MODEL, + UNCALIBRATED_MODELS, VIRTUAL_INITIAL_MODEL, CandidateSpace, Criterion, @@ -393,13 +394,13 @@ def __call__( user_calibrated_models=self.user_calibrated_models, ) - if not iteration[MODELS]: + if not iteration[UNCALIBRATED_MODELS]: raise StopIteration("No valid models found.") # TODO parallelize calibration (maybe not sensible if # `self.select_first_improvement`) - newly_calibrated_models = {} - for model in iteration[MODELS]: + calibrated_models = {} + for model in iteration[UNCALIBRATED_MODELS]: if ( model.get_criterion( criterion=self.criterion, @@ -419,7 +420,7 @@ def __call__( else: self.new_model_problem(model=model) - newly_calibrated_models[model.get_hash()] = model + calibrated_models[model.get_hash()] = model method_signal = self.handle_calibrated_model( model=model, predecessor_model=iteration[PREDECESSOR_MODEL], @@ -429,7 +430,7 @@ def __call__( iteration_results = petab_select.ui.end_iteration( candidate_space=iteration[CANDIDATE_SPACE], - newly_calibrated_models=newly_calibrated_models, + calibrated_models=calibrated_models, ) self.calibrated_models.update(iteration_results[MODELS])