Add the transport tutorial

.github/workflows/pytest.yaml

@@ -194,7 +194,7 @@ jobs:
       #------------------------------------------------
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root --with dev,server
+        run: poetry install --no-interaction --no-root --with dev,server,tutorial
 
       #------------------------
       #  install root project

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.13.0
+  rev: v1.14.1
   hooks:
   - id: mypy
     entry: bash -c "poetry run mypy ."

ixmp4/core/iamc/data.py

@@ -1,10 +1,6 @@
 from collections.abc import Iterable
-from typing import Optional, TypeVar
 
 import pandas as pd
-import pandera as pa
-from pandera.engines import pandas_engine
-from pandera.typing import Series
 
 # TODO Import this from typing when dropping Python 3.11
 from typing_extensions import Unpack
@@ -13,67 +9,17 @@
 from ixmp4.data.abstract import Run
 from ixmp4.data.abstract.iamc.datapoint import EnumerateKwargs
 from ixmp4.data.backend import Backend
+from ixmp4.data.db.iamc.utils import (
+    AddDataPointFrameSchema,
+    RemoveDataPointFrameSchema,
+    normalize_df,
+)
 
 from ..base import BaseFacade
 from ..utils import substitute_type
 from .variable import VariableRepository
 
 
-class RemoveDataPointFrameSchema(pa.DataFrameModel):
-    type: Optional[Series[pa.String]] = pa.Field(isin=[t for t in DataPointModel.Type])
-    step_year: Optional[Series[pa.Int]] = pa.Field(coerce=True, nullable=True)
-    step_datetime: Optional[Series[pandas_engine.DateTime]] = pa.Field(
-        coerce=True, nullable=True
-    )
-    step_category: Optional[Series[pa.String]] = pa.Field(nullable=True)
-
-    region: Optional[Series[pa.String]] = pa.Field(coerce=True)
-    unit: Optional[Series[pa.String]] = pa.Field(coerce=True)
-    variable: Optional[Series[pa.String]] = pa.Field(coerce=True)
-
-
-class AddDataPointFrameSchema(RemoveDataPointFrameSchema):
-    value: Series[pa.Float] = pa.Field(coerce=True)
-
-
-MAP_STEP_COLUMN = {
-    "ANNUAL": "step_year",
-    "CATEGORICAL": "step_year",
-    "DATETIME": "step_datetime",
-}
-
-
-def convert_to_std_format(df: pd.DataFrame, join_runs: bool) -> pd.DataFrame:
-    df.rename(columns={"step_category": "subannual"}, inplace=True)
-
-    if set(df.type.unique()).issubset(["ANNUAL", "CATEGORICAL"]):
-        df.rename(columns={"step_year": "year"}, inplace=True)
-        time_col = "year"
-    else:
-        T = TypeVar("T", bool, float, int, str)
-
-        def map_step_column(df: "pd.Series[T]") -> "pd.Series[T]":
-            df["time"] = df[MAP_STEP_COLUMN[str(df.type)]]
-            return df
-
-        df = df.apply(map_step_column, axis=1)
-        time_col = "time"
-
-    columns = ["model", "scenario", "version"] if join_runs else []
-    columns += ["region", "variable", "unit"] + [time_col]
-    if "subannual" in df.columns:
-        columns += ["subannual"]
-    return df[columns + ["value"]]
-
-
-def normalize_df(df: pd.DataFrame, raw: bool, join_runs: bool) -> pd.DataFrame:
-    if not df.empty:
-        df = df.drop(columns=["time_series__id"])
-        if raw is False:
-            return convert_to_std_format(df, join_runs)
-    return df
-
-
 class RunIamcData(BaseFacade):
     """IAMC data.
 
@@ -106,29 +52,17 @@ def _get_or_create_ts(self, df: pd.DataFrame) -> pd.DataFrame:
 
         # merge on the identity columns
         return pd.merge(
-            df,
-            ts_df,
-            how="left",
-            on=id_cols,
-            suffixes=(None, "_y"),
+            df, ts_df, how="left", on=id_cols, suffixes=(None, "_y")
         )  # tada, df with 'time_series__id' added from the database.
 
-    def add(
-        self,
-        df: pd.DataFrame,
-        type: Optional[DataPointModel.Type] = None,
-    ) -> None:
+    def add(self, df: pd.DataFrame, type: DataPointModel.Type | None = None) -> None:
         df = AddDataPointFrameSchema.validate(df)  # type: ignore[assignment]
         df["run__id"] = self.run.id
         df = self._get_or_create_ts(df)
         substitute_type(df, type)
         self.backend.iamc.datapoints.bulk_upsert(df)
 
-    def remove(
-        self,
-        df: pd.DataFrame,
-        type: Optional[DataPointModel.Type] = None,
-    ) -> None:
+    def remove(self, df: pd.DataFrame, type: DataPointModel.Type | None = None) -> None:
         df = RemoveDataPointFrameSchema.validate(df)  # type: ignore[assignment]
         df["run__id"] = self.run.id
         df = self._get_or_create_ts(df)

ixmp4/core/optimization/data.py

@@ -29,3 +29,9 @@ def __init__(self, run: Run, **kwargs: Backend) -> None:
         self.scalars = ScalarRepository(_backend=self.backend, _run=run)
         self.tables = TableRepository(_backend=self.backend, _run=run)
         self.variables = VariableRepository(_backend=self.backend, _run=run)
+
+    def remove_solution(self) -> None:
+        for equation in self.equations.list():
+            equation.remove_data()
+        for variable in self.variables.list():
+            variable.remove_data()

ixmp4/core/optimization/equation.py

@@ -40,7 +40,7 @@ def data(self) -> dict[str, Any]:
         return self._model.data
 
     def add(self, data: dict[str, Any] | pd.DataFrame) -> None:
-        """Adds data to an existing Equation."""
+        """Adds data to the Equation."""
         self.backend.optimization.equations.add_data(
             equation_id=self._model.id, data=data
         )
@@ -49,7 +49,7 @@ def add(self, data: dict[str, Any] | pd.DataFrame) -> None:
         ).data
 
     def remove_data(self) -> None:
-        """Removes data from an existing Equation."""
+        """Removes all data from the Equation."""
         self.backend.optimization.equations.remove_data(equation_id=self._model.id)
         self._model.data = self.backend.optimization.equations.get(
             run_id=self._model.run__id, name=self._model.name

ixmp4/core/optimization/indexset.py

@@ -29,7 +29,7 @@ def name(self) -> str:
         return self._model.name
 
     @property
-    def data(self) -> list[float | int | str]:
+    def data(self) -> list[float] | list[int] | list[str]:
         return self._model.data
 
     def add(

ixmp4/core/optimization/parameter.py

@@ -40,7 +40,7 @@ def data(self) -> dict[str, Any]:
         return self._model.data
 
     def add(self, data: dict[str, Any] | pd.DataFrame) -> None:
-        """Adds data to an existing Parameter."""
+        """Adds data to the Parameter."""
         self.backend.optimization.parameters.add_data(
             parameter_id=self._model.id, data=data
         )

ixmp4/core/optimization/table.py

@@ -40,7 +40,7 @@ def data(self) -> dict[str, Any]:
         return self._model.data
 
     def add(self, data: dict[str, Any] | pd.DataFrame) -> None:
-        """Adds data to an existing Table."""
+        """Adds data to the Table."""
         self.backend.optimization.tables.add_data(table_id=self._model.id, data=data)
         self._model.data = self.backend.optimization.tables.get(
             run_id=self._model.run__id, name=self._model.name

ixmp4/core/optimization/variable.py

@@ -40,7 +40,7 @@ def data(self) -> dict[str, Any]:
         return self._model.data
 
     def add(self, data: dict[str, Any] | pd.DataFrame) -> None:
-        """Adds data to an existing Variable."""
+        """Adds data to the Variable."""
         self.backend.optimization.variables.add_data(
             variable_id=self._model.id, data=data
         )
@@ -49,7 +49,7 @@ def add(self, data: dict[str, Any] | pd.DataFrame) -> None:
         ).data
 
     def remove_data(self) -> None:
-        """Removes data from an existing Variable."""
+        """Removes all data from the Variable."""
         self.backend.optimization.variables.remove_data(variable_id=self._model.id)
         self._model.data = self.backend.optimization.variables.get(
             run_id=self._model.run__id, name=self._model.name

ixmp4/core/run.py

@@ -73,27 +73,35 @@ def unset_as_default(self) -> None:
         """Unsets this run as the default version."""
         self.backend.runs.unset_as_default_version(self._model.id)
 
+    def clone(
+        self,
+        model: str | None = None,
+        scenario: str | None = None,
+        keep_solution: bool = True,
+    ) -> "Run":
+        return Run(
+            _backend=self.backend,
+            _model=self.backend.runs.clone(
+                run_id=self.id,
+                model_name=model,
+                scenario_name=scenario,
+                keep_solution=keep_solution,
+            ),
+        )
+
 
 class RunRepository(BaseFacade):
-    def create(
-        self,
-        model: str,
-        scenario: str,
-    ) -> Run:
+    def create(self, model: str, scenario: str) -> Run:
         return Run(
             _backend=self.backend, _model=self.backend.runs.create(model, scenario)
         )
 
-    def get(
-        self,
-        model: str,
-        scenario: str,
-        version: int | None = None,
-    ) -> Run:
-        if version is None:
-            _model = self.backend.runs.get_default_version(model, scenario)
-        else:
-            _model = self.backend.runs.get(model, scenario, version)
+    def get(self, model: str, scenario: str, version: int | None = None) -> Run:
+        _model = (
+            self.backend.runs.get_default_version(model, scenario)
+            if version is None
+            else self.backend.runs.get(model, scenario, version)
+        )
         return Run(_backend=self.backend, _model=_model)
 
     def list(self, **kwargs: Unpack[EnumerateKwargs]) -> list[Run]:

ixmp4/data/abstract/run.py

@@ -137,6 +137,26 @@ def get_default_version(self, model_name: str, scenario_name: str) -> Run:
         """
         ...
 
+    def get_by_id(self, id: int) -> Run:
+        """Retrieves a Run by its id.
+
+        Parameters
+        ----------
+        id : int
+            Unique integer id.
+
+        Raises
+        ------
+        :class:`ixmp4.data.abstract.Run.NotFound`.
+            If the Run with `id` does not exist.
+
+        Returns
+        -------
+        :class:`ixmp4.data.abstract.Run`:
+            The retrieved Run.
+        """
+        ...
+
     def list(self, **kwargs: Unpack[EnumerateKwargs]) -> list[Run]:
         r"""Lists runs by specified criteria.
 
@@ -206,3 +226,31 @@ def unset_as_default_version(self, id: int) -> None:
 
         """
         ...
+
+    def clone(
+        self,
+        run_id: int,
+        model_name: str | None = None,
+        scenario_name: str | None = None,
+        keep_solution: bool = True,
+    ) -> Run:
+        """Clone all data from one run to a new one.
+
+        Parameters
+        ----------
+        run_id: int
+            The unique integer id of the base run.
+        model_name: str | None
+            The new name of the model used in the new run, optional.
+        scenario_name: str | None
+            The new name of the scenario used in the new run, optional.
+        keep_solution: bool
+            Whether to keep the solution data from the base run. Optional, defaults to
+            `True`.
+
+        Returns
+        -------
+        :class:`ixmp4.data.abstract.Run`:
+            The clone of the base run.
+        """
+        ...

ixmp4/data/api/base.py

@@ -359,9 +359,7 @@ def _handle_pagination(
             return [data.pop("results")] + results
 
     def _list(
-        self,
-        params: ParamType | None = None,
-        json: JsonType | None = None,
+        self, params: ParamType | None = None, json: JsonType | None = None
     ) -> list[ModelType]:
         data = self._request_enumeration(params=params, table=False, json=json)
         if isinstance(data, dict):
@@ -375,9 +373,7 @@ def _list(
         return [self.model_class(**i) for i in results]
 
     def _tabulate(
-        self,
-        params: ParamType | None = {},
-        json: JsonType | None = None,
+        self, params: ParamType | None = {}, json: JsonType | None = None
     ) -> pd.DataFrame:
         # we can assume this type on table endpoints
         data: dict[str, Any] = self._request_enumeration(
@@ -398,9 +394,7 @@ def _tabulate(
             return DataFrame(**data).to_pandas()
 
     def _create(
-        self,
-        *args: Unpack[tuple[str]],
-        **kwargs: Unpack[_RequestKwargs],
+        self, *args: Unpack[tuple[str]], **kwargs: Unpack[_RequestKwargs]
     ) -> dict[str, Any]:
         # we can assume this type on create endpoints
         return self._request("POST", *args, **kwargs)  # type: ignore[return-value]
@@ -455,10 +449,7 @@ def create(
         | float
         | None,
     ) -> ModelType:
-        res = self._create(
-            self.prefix,
-            json=kwargs,
-        )
+        res = self._create(self.prefix, json=kwargs)
         return self.model_class(**res)
 
 
@@ -539,9 +530,4 @@ def bulk_delete(
     def bulk_delete_chunk(self, df: pd.DataFrame, **kwargs: Any) -> None:
         dict_ = df_to_dict(df)
         json_ = DataFrame(**dict_).model_dump_json()
-        self._request(
-            "PATCH",
-            self.prefix + "bulk/",
-            params=kwargs,
-            content=json_,
-        )
+        self._request("PATCH", self.prefix + "bulk/", params=kwargs, content=json_)