Merge branch 'main' into allow-runtime-params-as-list

.github/workflows/all-checks.yml

@@ -26,7 +26,7 @@ jobs:
     strategy:
       matrix:
         os: [ windows-latest, ubuntu-latest ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
     uses: ./.github/workflows/unit-tests.yml
     with:
       os: ${{ matrix.os }}
@@ -36,7 +36,7 @@ jobs:
     strategy:
       matrix:
         os: [ windows-latest, ubuntu-latest ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
     uses: ./.github/workflows/e2e-tests.yml
     with:
       os: ${{ matrix.os }}
@@ -59,7 +59,7 @@ jobs:
     strategy:
       matrix:
         os: [ windows-latest, ubuntu-latest ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
     uses: ./.github/workflows/pip-compile.yml
     with:
       os: ${{ matrix.os }}

.github/workflows/docs-only-checks.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
     uses: ./.github/workflows/lint.yml
     with:
       os: ${{ matrix.os }}

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 
-default_stages: [commit, manual]
+default_stages: [pre-commit, manual]
 
 repos:
     - repo: https://github.com/astral-sh/ruff-pre-commit

README.md

@@ -6,7 +6,7 @@
   </picture>
 </p>
 
-[![Python version](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://pypi.org/project/kedro/)
+[![Python version](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://pypi.org/project/kedro/)
 [![PyPI version](https://badge.fury.io/py/kedro.svg)](https://pypi.org/project/kedro/)
 [![Conda version](https://img.shields.io/conda/vn/conda-forge/kedro.svg)](https://anaconda.org/conda-forge/kedro)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/kedro-org/kedro/blob/main/LICENSE.md)

RELEASE.md

@@ -1,6 +1,7 @@
 # Upcoming Release
 
 ## Major features and improvements
+* Dropped Python 3.8 support.
 * Implemented `KedroDataCatalog` repeating `DataCatalog` functionality with a few API enhancements:
   * Removed `_FrozenDatasets` and access datasets as properties;
   * Added get dataset by name feature;
@@ -33,6 +34,7 @@
 * [Manezki](https://github.com/Manezki)
 * [MigQ2](https://github.com/MigQ2)
 * [Felix Scherz](https://github.com/felixscherz)
+* [Yu-Sheng Li](https://github.com/kevin1kevin1k)
 
 # Release 0.19.8
 

docs/source/conf.py

@@ -70,7 +70,7 @@
 intersphinx_mapping = {
     "kedro-viz": ("https://docs.kedro.org/projects/kedro-viz/en/v6.6.1/", None),
     "kedro-datasets": ("https://docs.kedro.org/projects/kedro-datasets/en/kedro-datasets-2.0.0/", None),
-    "cpython": ("https://docs.python.org/3.8/", None),
+    "cpython": ("https://docs.python.org/3.9/", None),
     "ipython": ("https://ipython.readthedocs.io/en/8.21.0/", None),
     "mlflow": ("https://www.mlflow.org/docs/2.12.1/", None),
     "kedro-mlflow": ("https://kedro-mlflow.readthedocs.io/en/0.12.2/", None),

docs/source/deployment/aws_step_functions.md

@@ -156,7 +156,7 @@ This file acts as the handler for each Lambda function in our pipeline, receives
 ```Dockerfile
 # Define global args
 ARG FUNCTION_DIR="/home/app/"
-ARG RUNTIME_VERSION="3.8"
+ARG RUNTIME_VERSION="3.9"
 
 # Stage 1 - bundle base image + runtime
 # Grab a fresh copy of the image and install GCC

docs/source/deployment/databricks/databricks_deployment_workflow.md

@@ -10,7 +10,7 @@ Here are some typical use cases for running a packaged Kedro project as a Databr
 
 - **Data engineering pipeline**: the output of your Kedro project is a file or set of files containing cleaned and processed data.
 - **Machine learning with MLflow**: your Kedro project runs an ML model; metrics about your experiments are tracked in MLflow.
-- **Automated and scheduled runs**: your Kedro project should be [run on Databricks automatically](https://docs.databricks.com/workflows/jobs/schedule-jobs.html#add-a-job-schedule).
+- **Automated and scheduled runs**: your Kedro project should be [run on Databricks automatically](https://docs.databricks.com/en/jobs/scheduled.html#add-a-job-schedule).
 - **CI/CD integration**: you have a CI/CD pipeline that produces a packaged Kedro project.
 
 Running your packaged project as a Databricks job is very different from running it from a Databricks notebook. The Databricks job cluster has to be provisioned and started for each run, which is significantly slower than running it as a notebook on a cluster that has already been started. In addition, there is no way to change your project's code once it has been packaged. Instead, you must change your code, create a new package, and then upload it to Databricks again.

docs/source/deployment/databricks/databricks_ide_development_workflow.md

@@ -31,7 +31,7 @@ The main steps in this tutorial are as follows:
 
 - An active [Databricks deployment](https://docs.databricks.com/getting-started/index.html).
 - A [Databricks cluster](https://docs.databricks.com/clusters/configure.html) configured with a recent version (>= 11.3 is recommended) of the Databricks runtime.
-- [Conda installed](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html) on your local machine in order to create a virtual environment with a specific version of Python (>= 3.8 is required). If you have Python >= 3.8 installed, you can use other software to create a virtual environment.
+- [Conda installed](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html) on your local machine in order to create a virtual environment with a specific version of Python (>= 3.9 is required). If you have Python >= 3.9 installed, you can use other software to create a virtual environment.
 
 ## Set up your project
 

docs/source/deployment/databricks/databricks_notebooks_development_workflow.md

@@ -19,7 +19,7 @@ This tutorial introduces a Kedro project development workflow using only the Dat
 
 - An active [Databricks deployment](https://docs.databricks.com/getting-started/index.html).
 - A [Databricks cluster](https://docs.databricks.com/clusters/configure.html) configured with a recent version (>= 11.3 is recommended) of the Databricks runtime.
-- Python >= 3.8 installed.
+- Python >= 3.9 installed.
 - Git installed.
 - A [GitHub](https://github.com/) account.
 - A Python environment management system installed, [venv](https://docs.python.org/3/library/venv.html), [virtualenv](https://virtualenv.pypa.io/en/latest/) or [Conda](https://docs.conda.io/en/latest/) are popular choices.

docs/source/get_started/install.md

@@ -1,7 +1,7 @@
 # Set up Kedro
 
 ## Installation prerequisites
-* **Python**: Kedro supports macOS, Linux, and Windows and is built for Python 3.8+. You'll select a version of Python when you create a virtual environment for your Kedro project.
+* **Python**: Kedro supports macOS, Linux, and Windows and is built for Python 3.9+. You'll select a version of Python when you create a virtual environment for your Kedro project.
 
 * **Virtual environment**: You should create a new virtual environment for *each* new Kedro project you work on to isolate its Python dependencies from those of other projects.
 
@@ -55,7 +55,7 @@ deactivate
 conda create --name kedro-environment python=3.10 -y
 ```
 
-The example below uses Python 3.10, and creates a virtual environment called `kedro-environment`. You can opt for a different version of Python (any version >= 3.8 and <3.12) for your project, and you can name it anything you choose.
+The example below uses Python 3.10, and creates a virtual environment called `kedro-environment`. You can opt for a different version of Python (any version >= 3.9 and <3.12) for your project, and you can name it anything you choose.
 
 The `conda` virtual environment is not dependent on your current working directory and can be activated from any directory:
 
@@ -136,7 +136,7 @@ When migrating an existing project to a newer Kedro version, make sure you also
 ## Summary
 
 * Kedro can be used on Windows, macOS or Linux.
-* Installation prerequisites include a virtual environment manager like `conda`, Python 3.8+, and `git`.
+* Installation prerequisites include a virtual environment manager like `conda`, Python 3.9+, and `git`.
 * You should install Kedro using `pip install kedro`.
 
 If you encounter any problems as you set up Kedro, ask for help on Kedro's [Slack organisation](https://slack.kedro.org) or review the [searchable archive of Slack discussions](https://linen-slack.kedro.org/).

docs/source/index.rst

@@ -23,9 +23,9 @@ Welcome to Kedro's award-winning documentation!
     :target: https://opensource.org/license/apache2-0-php/
     :alt: License is Apache 2.0
 
-.. image:: https://img.shields.io/badge/3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11-blue.svg
+.. image:: https://img.shields.io/badge/3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg
     :target: https://pypi.org/project/kedro/
-    :alt: Python version 3.8, 3.9, 3.10, 3.11
+    :alt: Python version 3.9, 3.10, 3.11, 3.12
 
 .. image:: https://badge.fury.io/py/kedro.svg
     :target: https://pypi.org/project/kedro/

docs/source/introduction/index.md

@@ -18,6 +18,6 @@ Use the left-hand table of contents to explore the documentation available for m
 ```{note}
 We have designed the preliminary documentation and the [spaceflights tutorial](../tutorial/spaceflights_tutorial.md) for anyone new to Kedro. The more knowledge of Python you have, the easier you will find the learning curve.
 
-There are many excellent online resources for learning Python; you should choose those that reference Python 3, as Kedro is built for Python 3.8+. There are curated lists of online resources, such as the [official Python programming language website](https://www.python.org/) and this list of [free programming books and tutorials](https://github.com/EbookFoundation/free-programming-books/blob/master/books/free-programming-books-langs.md#python).
+There are many excellent online resources for learning Python; you should choose those that reference Python 3, as Kedro is built for Python 3.9+. There are curated lists of online resources, such as the [official Python programming language website](https://www.python.org/) and this list of [free programming books and tutorials](https://github.com/EbookFoundation/free-programming-books/blob/master/books/free-programming-books-langs.md#python).
 
 ```

docs/source/notebooks_and_ipython/notebook-example/add_kedro_to_a_notebook.ipynb

@@ -683,7 +683,6 @@
     "####################\n",
     "# Data processing  #\n",
     "####################\n",
-    "from typing import Dict, Tuple\n",
     "\n",
     "import pandas as pd\n",
     "from sklearn.linear_model import LinearRegression\n",
@@ -736,7 +735,7 @@
     "##################################\n",
     "\n",
     "\n",
-    "def split_data(data: pd.DataFrame, parameters: Dict) -> Tuple:\n",
+    "def split_data(data: pd.DataFrame, parameters: dict) -> tuple:\n",
     "    X = data[parameters[\"features\"]]\n",
     "    y = data[\"price\"]\n",
     "    X_train, X_test, y_train, y_test = train_test_split(\n",
@@ -796,7 +795,6 @@
    "outputs": [],
    "source": [
     "# Kedro setup for data management and configuration\n",
-    "from typing import Dict, Tuple\n",
     "\n",
     "import pandas as pd\n",
     "from sklearn.linear_model import LinearRegression\n",
@@ -873,7 +871,7 @@
     "##################################\n",
     "\n",
     "\n",
-    "def split_data(data: pd.DataFrame, parameters: Dict) -> Tuple:\n",
+    "def split_data(data: pd.DataFrame, parameters: dict) -> tuple:\n",
     "    X = data[parameters[\"features\"]]\n",
     "    y = data[\"price\"]\n",
     "    X_train, X_test, y_train, y_test = train_test_split(\n",

docs/source/tutorial/tutorial_template.md

@@ -32,7 +32,6 @@ The spaceflights project dependencies are stored in `requirements.txt`(you may f
 
 ```text
 # code quality packages
-ipython>=7.31.1, <8.0; python_version < '3.8'
 ipython~=8.10; python_version >= '3.8'
 ruff==0.1.8
 

features/environment.py

@@ -5,7 +5,6 @@
 import os
 import shutil
 import subprocess
-import sys
 import tempfile
 import venv
 from pathlib import Path
@@ -131,11 +130,6 @@ def _install_project_requirements(context):
         .splitlines()
     )
     install_reqs = [req for req in install_reqs if "{" not in req and "#" not in req]
-    # For Python versions 3.9 and above we use the new dataset dependency format introduced in `kedro-datasets` 3.0.0
-    if sys.version_info.minor > MINOR_PYTHON_38_VERSION:
-        install_reqs.append("kedro-datasets[pandas-csvdataset]")
-    # For Python 3.8 we use the older `kedro-datasets` dependency format
-    else:
-        install_reqs.append("kedro-datasets[pandas.CSVDataset]")
+    install_reqs.append("kedro-datasets[pandas-csvdataset]")
     call([context.pip, "install", *install_reqs], env=context.env)
     return context

features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt

@@ -2,5 +2,4 @@ ipython>=8.10
 jupyterlab>=3.0
 notebook
 kedro~={{ cookiecutter.kedro_version}}
-kedro-datasets[pandas-csvdataset]; python_version >= "3.9"
-kedro-datasets[pandas.CSVDataset]<2.0.0; python_version < '3.9'
+kedro-datasets[pandas-csvdataset]

features/steps/util.py

@@ -6,9 +6,10 @@
 import re
 from contextlib import contextmanager
 from time import sleep, time
-from typing import TYPE_CHECKING, Any, Callable, Iterator
+from typing import TYPE_CHECKING, Any, Callable
 
 if TYPE_CHECKING:
+    from collections.abc import Iterator
     from pathlib import Path
 
 

kedro/config/omegaconf_config.py

@@ -8,10 +8,10 @@
 import logging
 import mimetypes
 import typing
-from collections.abc import KeysView
+from collections.abc import Iterable, KeysView
 from enum import Enum, auto
 from pathlib import Path
-from typing import Any, Callable, Iterable
+from typing import Any, Callable
 
 import fsspec
 from omegaconf import DictConfig, OmegaConf

kedro/framework/cli/cli.py

@@ -10,10 +10,13 @@
 import traceback
 from collections import defaultdict
 from pathlib import Path
-from typing import Any, Sequence
+from typing import TYPE_CHECKING, Any
 
 import click
 
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
 from kedro import __version__ as version
 from kedro.framework.cli import BRIGHT_BLACK, ORANGE
 from kedro.framework.cli.hooks import get_cli_hook_manager

kedro/framework/cli/micropkg.py

@@ -12,7 +12,8 @@
 import toml
 from importlib import import_module
 from pathlib import Path
-from typing import Any, Iterable, Iterator, TYPE_CHECKING
+from typing import Any, TYPE_CHECKING
+
 
 import click
 from omegaconf import OmegaConf
@@ -42,6 +43,7 @@
 if TYPE_CHECKING:
     from kedro.framework.startup import ProjectMetadata
     from importlib_metadata import PackageMetadata
+    from collections.abc import Iterable, Iterator
 
 _PYPROJECT_TOML_TEMPLATE = """
 [build-system]

kedro/framework/cli/utils.py

@@ -16,10 +16,14 @@
 import typing
 import warnings
 from collections import defaultdict
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Sequence
 from importlib import import_module
 from itertools import chain
 from pathlib import Path
-from typing import IO, Any, Callable, Iterable, Sequence
+from typing import IO, Any, Callable
 
 import click
 import importlib_metadata
@@ -56,7 +60,7 @@ def call(cmd: list[str], **kwargs: Any) -> None:  # pragma: no cover
     Raises:
         click.exceptions.Exit: If `subprocess.run` returns non-zero code.
     """
-    click.echo(" ".join(shlex.quote(c) for c in cmd))
+    click.echo(shlex.join(cmd))
     code = subprocess.run(cmd, **kwargs).returncode  # noqa: PLW1510, S603
     if code:
         raise click.exceptions.Exit(code=code)

kedro/framework/hooks/manager.py

@@ -3,7 +3,8 @@
 """
 
 import logging
-from typing import Any, Iterable
+from collections.abc import Iterable
+from typing import Any
 
 from pluggy import PluginManager
 

kedro/framework/session/session.py

@@ -11,7 +11,7 @@
 import traceback
 from copy import deepcopy
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Iterable
+from typing import TYPE_CHECKING, Any
 
 import click
 
@@ -28,6 +28,8 @@
 from kedro.utils import _find_kedro_project
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
+
     from kedro.config import AbstractConfigLoader
     from kedro.framework.context import KedroContext
     from kedro.framework.session.store import BaseSessionStore

kedro/io/catalog_config_resolver.py

@@ -7,13 +7,13 @@
 import copy
 import logging
 import re
-from typing import Any, Dict
+from typing import Any
 
 from parse import parse
 
 from kedro.io.core import DatasetError
 
-Patterns = Dict[str, Dict[str, Any]]
+Patterns = dict[str, dict[str, Any]]
 
 CREDENTIALS_KEY = "credentials"
 

kedro/io/core.py

@@ -505,20 +505,13 @@ def parse_dataset_definition(
                 class_obj = tmp
                 break
         else:
-            hint = ""
-            if "DataSet" in dataset_type:
-                hint = (  # pragma: no cover # To remove when we drop support for python 3.8
-                    "Hint: If you are trying to use a dataset from `kedro-datasets`>=2.0.0, "
-                    "make sure that the dataset name uses the `Dataset` spelling instead of `DataSet`."
-                )
-            else:
-                hint = (
-                    "Hint: If you are trying to use a dataset from `kedro-datasets`, "
-                    "make sure that the package is installed in your current environment. "
-                    "You can do so by running `pip install kedro-datasets` or "
-                    "`pip install kedro-datasets[<dataset-group>]` to install `kedro-datasets` along with "
-                    "related dependencies for the specific dataset group."
-                )
+            hint = (
+                "Hint: If you are trying to use a dataset from `kedro-datasets`, "
+                "make sure that the package is installed in your current environment. "
+                "You can do so by running `pip install kedro-datasets` or "
+                "`pip install kedro-datasets[<dataset-group>]` to install `kedro-datasets` along with "
+                "related dependencies for the specific dataset group."
+            )
             raise DatasetError(
                 f"Class '{dataset_type}' not found, is this a typo?" f"\n{hint}"
             )

kedro/ipython/__init__.py

@@ -14,7 +14,10 @@
 import warnings
 from pathlib import Path
 from types import MappingProxyType
-from typing import Any, Callable, OrderedDict
+from typing import TYPE_CHECKING, Any, Callable
+
+if TYPE_CHECKING:
+    from collections import OrderedDict
 
 from IPython.core.getipython import get_ipython
 from IPython.core.magic import needs_local_scope, register_line_magic