Merge pull request #136 from BlueBrain/memodel

add MEModel

bluepyemodel/access_point/access_point.py

@@ -31,7 +31,6 @@
 from bluepyemodel.emodel_pipeline.emodel_metadata import EModelMetadata
 from bluepyemodel.emodel_pipeline.emodel_settings import EModelPipelineSettings
 from bluepyemodel.tools.utils import get_checkpoint_path
-from bluepyemodel.tools.utils import get_legacy_checkpoint_path
 from bluepyemodel.tools.utils import read_checkpoint
 
 # pylint: disable=no-member,unused-argument,assignment-from-no-return,no-value-for-parameter
@@ -237,9 +236,7 @@ def optimisation_state(self, seed=None, continue_opt=False):
 
         # no file -> target not complete
         if not pathlib.Path(checkpoint_path).is_file():
-            checkpoint_path = get_legacy_checkpoint_path(checkpoint_path)
-            if not pathlib.Path(checkpoint_path).is_file():
-                return OptimisationState.EMPTY
+            return OptimisationState.EMPTY
 
         # there is a file & continue opt is False -> target considered complete
         if not continue_opt:
@@ -326,8 +323,8 @@ def __str__(self):
             str_ += "OPTIMISATION STATUS\n"
             str_ += f"  Number of checkpoints: {len(checkpoints)}\n"
             for c in checkpoints:
-                run, run_metadata = read_checkpoint(c)
-                str_ += f"    Seed {run_metadata['seed']};"
+                run, seed = read_checkpoint(c)
+                str_ += f"    Seed {seed};"
                 str_ += f" Last generation: {run['logbook'].select('gen')[-1]};"
                 str_ += f" Best fitness: {sum(run['halloffame'][0].fitness.values)}\n"
             str_ += "\n"

bluepyemodel/emodel_pipeline/emodel.py

@@ -38,7 +38,72 @@ def format_dict_for_resource(d):
     return out
 
 
-class EModel:
+class EModelMixin:
+    """Contains functions used both in EModel and MEModel classes."""
+
+    def build_pdf_dependencies(self, seed):
+        """Find all the pdfs associated to an emodel"""
+
+        pdfs = []
+
+        opt_pdf = search_pdfs.search_figure_emodel_optimisation(self.emodel_metadata, seed)
+        if opt_pdf:
+            pdfs.append(opt_pdf)
+
+        traces_pdf = search_pdfs.search_figure_emodel_traces(self.emodel_metadata, seed)
+        if traces_pdf:
+            pdfs += [p for p in traces_pdf if p]
+
+        scores_pdf = search_pdfs.search_figure_emodel_score(self.emodel_metadata, seed)
+        if scores_pdf:
+            pdfs += [p for p in scores_pdf if p]
+
+        thumbnail_pdf = search_pdfs.search_figure_emodel_thumbnail(self.emodel_metadata, seed)
+        if thumbnail_pdf:
+            pdfs += [p for p in thumbnail_pdf if p]
+
+        parameters_pdf = search_pdfs.search_figure_emodel_parameters(self.emodel_metadata)
+        if parameters_pdf:
+            pdfs += [p for p in parameters_pdf if p]
+
+        parameters_evo_pdf = search_pdfs.search_figure_emodel_parameters_evolution(
+            self.emodel_metadata, seed
+        )
+        if parameters_evo_pdf:
+            pdfs.append(parameters_evo_pdf)
+
+        all_parameters_evo_pdf = search_pdfs.search_figure_emodel_parameters_evolution(
+            self.emodel_metadata, seed=None
+        )
+        if all_parameters_evo_pdf:
+            pdfs.append(all_parameters_evo_pdf)
+
+        currentscape_pdfs = search_pdfs.search_figure_emodel_currentscapes(
+            self.emodel_metadata, seed
+        )
+        if currentscape_pdfs:
+            pdfs += [p for p in currentscape_pdfs if p]
+
+        bAP_pdf = search_pdfs.search_figure_emodel_bAP(self.emodel_metadata, seed)
+        if bAP_pdf:
+            pdfs += [p for p in bAP_pdf if p]
+
+        EPSP_pdf = search_pdfs.search_figure_emodel_EPSP(self.emodel_metadata, seed)
+        if EPSP_pdf:
+            pdfs += [p for p in EPSP_pdf if p]
+
+        ISI_CV_pdf = search_pdfs.search_figure_emodel_ISI_CV(self.emodel_metadata, seed)
+        if ISI_CV_pdf:
+            pdfs += [p for p in ISI_CV_pdf if p]
+
+        rheobase_pdf = search_pdfs.search_figure_emodel_rheobase(self.emodel_metadata, seed)
+        if rheobase_pdf:
+            pdfs += [p for p in rheobase_pdf if p]
+
+        return pdfs
+
+
+class EModel(EModelMixin):
     """Contains all the information related to an optimized e-model, such as its parameters or
     its e-feature values and scores.
 
@@ -66,7 +131,7 @@ def __init__(
             scoreValidation (dict or Resource): scores obtained on the validation protocols.
             passedValidation (bool or None): did the model go through validation and if yes,
                 did it pass it successfully (None: no validation, True: passed, False: didn't pass)
-            seed (str): seed used during optimisation for this emodel.
+            seed (int): seed used during optimisation for this emodel.
             emodel_metadata (EModelMetadata): metadata of the model (emodel name, etype, ttype, ...)
         """
 
@@ -109,69 +174,8 @@ def __init__(
     def copy_pdf_dependencies_to_new_path(self, seed, overwrite=False):
         """Copy pdf dependencies to new path using allen notation"""
         search_pdfs.copy_emodel_pdf_dependencies_to_new_path(
-            self.emodel_metadata, seed, overwrite=overwrite
-        )
-
-    def build_pdf_dependencies(self, seed):
-        """Find all the pdfs associated to an emodel"""
-
-        pdfs = []
-
-        opt_pdf = search_pdfs.search_figure_emodel_optimisation(self.emodel_metadata, seed)
-        if opt_pdf:
-            pdfs.append(opt_pdf)
-
-        traces_pdf = search_pdfs.search_figure_emodel_traces(self.emodel_metadata, seed)
-        if traces_pdf:
-            pdfs += [p for p in traces_pdf if p]
-
-        scores_pdf = search_pdfs.search_figure_emodel_score(self.emodel_metadata, seed)
-        if scores_pdf:
-            pdfs += [p for p in scores_pdf if p]
-
-        thumbnail_pdf = search_pdfs.search_figure_emodel_thumbnail(self.emodel_metadata, seed)
-        if thumbnail_pdf:
-            pdfs += [p for p in thumbnail_pdf if p]
-
-        parameters_pdf = search_pdfs.search_figure_emodel_parameters(self.emodel_metadata)
-        if parameters_pdf:
-            pdfs += [p for p in parameters_pdf if p]
-
-        parameters_evo_pdf = search_pdfs.search_figure_emodel_parameters_evolution(
-            self.emodel_metadata, seed
-        )
-        if parameters_evo_pdf:
-            pdfs.append(parameters_evo_pdf)
-
-        all_parameters_evo_pdf = search_pdfs.search_figure_emodel_parameters_evolution(
-            self.emodel_metadata, seed=None
-        )
-        if all_parameters_evo_pdf:
-            pdfs.append(all_parameters_evo_pdf)
-
-        currentscape_pdfs = search_pdfs.search_figure_emodel_currentscapes(
-            self.emodel_metadata, seed
+            self.emodel_metadata, self.emodel_metadata, False, True, seed, overwrite=overwrite
         )
-        if currentscape_pdfs:
-            pdfs += [p for p in currentscape_pdfs if p]
-
-        bAP_pdf = search_pdfs.search_figure_emodel_bAP(self.emodel_metadata, seed)
-        if bAP_pdf:
-            pdfs += [p for p in bAP_pdf if p]
-
-        EPSP_pdf = search_pdfs.search_figure_emodel_EPSP(self.emodel_metadata, seed)
-        if EPSP_pdf:
-            pdfs += [p for p in EPSP_pdf if p]
-
-        ISI_CV_pdf = search_pdfs.search_figure_emodel_ISI_CV(self.emodel_metadata, seed)
-        if ISI_CV_pdf:
-            pdfs += [p for p in ISI_CV_pdf if p]
-
-        rheobase_pdf = search_pdfs.search_figure_emodel_rheobase(self.emodel_metadata, seed)
-        if rheobase_pdf:
-            pdfs += [p for p in rheobase_pdf if p]
-
-        return pdfs
 
     def get_related_nexus_ids(self):
         return {

bluepyemodel/emodel_pipeline/emodel_metadata.py

@@ -198,7 +198,9 @@ def as_dict(self):
         """Metadata as dict."""
         return vars(self)
 
-    def as_string(self, seed=None, use_allen_notation=True):
+    def as_string(
+        self, seed=None, use_allen_notation=True, replace_semicolons=True, replace_spaces=True
+    ):
         s = ""
 
         for k in [
@@ -222,7 +224,12 @@ def as_string(self, seed=None, use_allen_notation=True):
             s += f"seed={seed}__"
 
         # can have ':' in mtype. Replace this character.
-        s = s.replace(":", "_")
+        if replace_semicolons:
+            s = s.replace(":", "_")
+
+        # also replace spaces if any
+        if replace_spaces:
+            s = s.replace(" ", "_")
 
         return s[:-2]
 

bluepyemodel/emodel_pipeline/emodel_pipeline.py

@@ -30,7 +30,6 @@
 from bluepyemodel.optimisation import store_best_model
 from bluepyemodel.tools.multiprocessing import get_mapper
 from bluepyemodel.tools.utils import get_checkpoint_path
-from bluepyemodel.tools.utils import get_legacy_checkpoint_path
 from bluepyemodel.validation.validation import validate
 
 logger = logging.getLogger()
@@ -201,10 +200,6 @@ def store_optimisation_results(self, seed=None):
         else:
             checkpoint_path = get_checkpoint_path(self.access_point.emodel_metadata, seed=1)
             checkpoint_list = glob.glob(checkpoint_path.replace("seed=1", "*"))
-            if not checkpoint_list:
-                checkpoint_list = glob.glob(
-                    get_legacy_checkpoint_path(checkpoint_path).replace("seed=1", "*")
-                )
 
             for chkp_path in checkpoint_list:
                 file_name = pathlib.Path(chkp_path).stem
@@ -280,6 +275,7 @@ def plot(self, only_validated=False, load_from_local=False):
             plotting.evolution_parameters_density(
                 evaluator=cell_evaluator,
                 checkpoint_paths=checkpoint_paths,
+                metadata=self.access_point.emodel_metadata,
                 figures_dir=pathlib.Path("./figures")
                 / self.access_point.emodel_metadata.emodel
                 / "parameter_evolution",

bluepyemodel/emodel_pipeline/memodel.py

@@ -0,0 +1,73 @@
+"""MEModel class"""
+
+"""
+Copyright 2024, EPFL/Blue Brain Project
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from bluepyemodel.emodel_pipeline.emodel import EModelMixin
+
+
+class MEModel(EModelMixin):
+    """Combination of an EModel and a Morphology. Should contain ids of these resources,
+    as well as analysis plotting figure paths."""
+
+    def __init__(
+        self,
+        seed=None,
+        emodel_metadata=None,
+        emodel_id=None,
+        morphology_id=None,
+        validated=False,
+        status="initialized",
+    ):
+        """Init
+
+        Args:
+            seed (int): seed used during optimisation for this emodel.
+            emodel_metadata (EModelMetadata): metadata of the model (emodel name, etype, ttype, ...)
+            emodel_id (str): nexus if of the e-model used in this me-model
+            morphology_id (str): nexus id of the morphology used in this me-model
+            validated (bool): whether the MEModel has been validated by user
+            status (str): whether the analysis has run or not. Can be "initialized" or "done".
+        """
+
+        self.emodel_metadata = emodel_metadata
+        self.seed = seed
+
+        self.emodel_id = emodel_id
+        self.morphology_id = morphology_id
+
+        self.validated = validated
+        self.status = status
+
+    def get_related_nexus_ids(self):
+        uses = []
+        if self.emodel_id:
+            uses.append({"id": self.emodel_id, "type": "EModel"})
+        if self.morphology_id:
+            uses.append({"id": self.morphology_id, "type": "NeuronMorphology"})
+        return {"uses": uses}
+
+    def as_dict(self):
+        pdf_dependencies = self.build_pdf_dependencies(self.seed)
+
+        return {
+            "nexus_images": pdf_dependencies,
+            "seed": self.seed,
+            "emodel_id": self.emodel_id,
+            "morphology_id": self.morphology_id,
+            "validated": self.validated,
+            "status": self.status,
+        }