diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml index 2dfb971e3d..afd9caced0 100644 --- a/.github/workflows/all-checks.yml +++ b/.github/workflows/all-checks.yml @@ -26,7 +26,7 @@ jobs: strategy: matrix: os: [ windows-latest, ubuntu-latest ] - python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] uses: ./.github/workflows/unit-tests.yml with: os: ${{ matrix.os }} @@ -36,7 +36,7 @@ jobs: strategy: matrix: os: [ windows-latest, ubuntu-latest ] - python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] uses: ./.github/workflows/e2e-tests.yml with: os: ${{ matrix.os }} @@ -59,7 +59,7 @@ jobs: strategy: matrix: os: [ windows-latest, ubuntu-latest ] - python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] uses: ./.github/workflows/pip-compile.yml with: os: ${{ matrix.os }} diff --git a/.github/workflows/docs-only-checks.yml b/.github/workflows/docs-only-checks.yml index d0cca88abd..edd95de234 100644 --- a/.github/workflows/docs-only-checks.yml +++ b/.github/workflows/docs-only-checks.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest ] - python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] uses: ./.github/workflows/lint.yml with: os: ${{ matrix.os }} diff --git a/README.md b/README.md index 5cc0bda930..5e82fcebc2 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@

-[![Python version](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://pypi.org/project/kedro/) +[![Python version](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://pypi.org/project/kedro/) [![PyPI version](https://badge.fury.io/py/kedro.svg)](https://pypi.org/project/kedro/) [![Conda version](https://img.shields.io/conda/vn/conda-forge/kedro.svg)](https://anaconda.org/conda-forge/kedro) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/kedro-org/kedro/blob/main/LICENSE.md) diff --git a/RELEASE.md b/RELEASE.md index 61560acf87..da3423fab0 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,6 +1,7 @@ # Upcoming Release ## Major features and improvements +* Dropped Python 3.8 support. * Implemented `KedroDataCatalog` repeating `DataCatalog` functionality with a few API enhancements: * Removed `_FrozenDatasets` and access datasets as properties; * Added get dataset by name feature; @@ -32,6 +33,7 @@ e `_get_config_credentials()` * [Manezki](https://github.com/Manezki) * [MigQ2](https://github.com/MigQ2) * [Felix Scherz](https://github.com/felixscherz) +* [Yu-Sheng Li](https://github.com/kevin1kevin1k) # Release 0.19.8 diff --git a/docs/source/conf.py b/docs/source/conf.py index 50b719f117..a883f76bd6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -70,7 +70,7 @@ intersphinx_mapping = { "kedro-viz": ("https://docs.kedro.org/projects/kedro-viz/en/v6.6.1/", None), "kedro-datasets": ("https://docs.kedro.org/projects/kedro-datasets/en/kedro-datasets-2.0.0/", None), - "cpython": ("https://docs.python.org/3.8/", None), + "cpython": ("https://docs.python.org/3.9/", None), "ipython": ("https://ipython.readthedocs.io/en/8.21.0/", None), "mlflow": ("https://www.mlflow.org/docs/2.12.1/", None), "kedro-mlflow": ("https://kedro-mlflow.readthedocs.io/en/0.12.2/", None), diff --git a/docs/source/deployment/aws_step_functions.md b/docs/source/deployment/aws_step_functions.md index 8a3d70da53..9802eb542b 100644 --- a/docs/source/deployment/aws_step_functions.md +++ b/docs/source/deployment/aws_step_functions.md @@ -156,7 +156,7 @@ This file acts as the handler for each Lambda function in our pipeline, receives ```Dockerfile # Define global args ARG FUNCTION_DIR="/home/app/" -ARG RUNTIME_VERSION="3.8" +ARG RUNTIME_VERSION="3.9" # Stage 1 - bundle base image + runtime # Grab a fresh copy of the image and install GCC diff --git a/docs/source/deployment/databricks/databricks_ide_development_workflow.md b/docs/source/deployment/databricks/databricks_ide_development_workflow.md index f85799272d..7146ec7927 100644 --- a/docs/source/deployment/databricks/databricks_ide_development_workflow.md +++ b/docs/source/deployment/databricks/databricks_ide_development_workflow.md @@ -31,7 +31,7 @@ The main steps in this tutorial are as follows: - An active [Databricks deployment](https://docs.databricks.com/getting-started/index.html). - A [Databricks cluster](https://docs.databricks.com/clusters/configure.html) configured with a recent version (>= 11.3 is recommended) of the Databricks runtime. -- [Conda installed](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html) on your local machine in order to create a virtual environment with a specific version of Python (>= 3.8 is required). If you have Python >= 3.8 installed, you can use other software to create a virtual environment. +- [Conda installed](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html) on your local machine in order to create a virtual environment with a specific version of Python (>= 3.9 is required). If you have Python >= 3.9 installed, you can use other software to create a virtual environment. ## Set up your project diff --git a/docs/source/deployment/databricks/databricks_notebooks_development_workflow.md b/docs/source/deployment/databricks/databricks_notebooks_development_workflow.md index 8fc4b76f4c..8a0616708b 100644 --- a/docs/source/deployment/databricks/databricks_notebooks_development_workflow.md +++ b/docs/source/deployment/databricks/databricks_notebooks_development_workflow.md @@ -19,7 +19,7 @@ This tutorial introduces a Kedro project development workflow using only the Dat - An active [Databricks deployment](https://docs.databricks.com/getting-started/index.html). - A [Databricks cluster](https://docs.databricks.com/clusters/configure.html) configured with a recent version (>= 11.3 is recommended) of the Databricks runtime. -- Python >= 3.8 installed. +- Python >= 3.9 installed. - Git installed. - A [GitHub](https://github.com/) account. - A Python environment management system installed, [venv](https://docs.python.org/3/library/venv.html), [virtualenv](https://virtualenv.pypa.io/en/latest/) or [Conda](https://docs.conda.io/en/latest/) are popular choices. diff --git a/docs/source/get_started/install.md b/docs/source/get_started/install.md index d61d084a89..fe75decda2 100644 --- a/docs/source/get_started/install.md +++ b/docs/source/get_started/install.md @@ -1,7 +1,7 @@ # Set up Kedro ## Installation prerequisites -* **Python**: Kedro supports macOS, Linux, and Windows and is built for Python 3.8+. You'll select a version of Python when you create a virtual environment for your Kedro project. +* **Python**: Kedro supports macOS, Linux, and Windows and is built for Python 3.9+. You'll select a version of Python when you create a virtual environment for your Kedro project. * **Virtual environment**: You should create a new virtual environment for *each* new Kedro project you work on to isolate its Python dependencies from those of other projects. @@ -55,7 +55,7 @@ deactivate conda create --name kedro-environment python=3.10 -y ``` -The example below uses Python 3.10, and creates a virtual environment called `kedro-environment`. You can opt for a different version of Python (any version >= 3.8 and <3.12) for your project, and you can name it anything you choose. +The example below uses Python 3.10, and creates a virtual environment called `kedro-environment`. You can opt for a different version of Python (any version >= 3.9 and <3.12) for your project, and you can name it anything you choose. The `conda` virtual environment is not dependent on your current working directory and can be activated from any directory: @@ -136,7 +136,7 @@ When migrating an existing project to a newer Kedro version, make sure you also ## Summary * Kedro can be used on Windows, macOS or Linux. -* Installation prerequisites include a virtual environment manager like `conda`, Python 3.8+, and `git`. +* Installation prerequisites include a virtual environment manager like `conda`, Python 3.9+, and `git`. * You should install Kedro using `pip install kedro`. If you encounter any problems as you set up Kedro, ask for help on Kedro's [Slack organisation](https://slack.kedro.org) or review the [searchable archive of Slack discussions](https://linen-slack.kedro.org/). diff --git a/docs/source/index.rst b/docs/source/index.rst index ce8d85a9a1..ecbdbbe381 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -23,9 +23,9 @@ Welcome to Kedro's award-winning documentation! :target: https://opensource.org/license/apache2-0-php/ :alt: License is Apache 2.0 -.. image:: https://img.shields.io/badge/3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11-blue.svg +.. image:: https://img.shields.io/badge/3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg :target: https://pypi.org/project/kedro/ - :alt: Python version 3.8, 3.9, 3.10, 3.11 + :alt: Python version 3.9, 3.10, 3.11, 3.12 .. image:: https://badge.fury.io/py/kedro.svg :target: https://pypi.org/project/kedro/ diff --git a/docs/source/introduction/index.md b/docs/source/introduction/index.md index 38dab844a3..d9de994933 100644 --- a/docs/source/introduction/index.md +++ b/docs/source/introduction/index.md @@ -18,6 +18,6 @@ Use the left-hand table of contents to explore the documentation available for m ```{note} We have designed the preliminary documentation and the [spaceflights tutorial](../tutorial/spaceflights_tutorial.md) for anyone new to Kedro. The more knowledge of Python you have, the easier you will find the learning curve. -There are many excellent online resources for learning Python; you should choose those that reference Python 3, as Kedro is built for Python 3.8+. There are curated lists of online resources, such as the [official Python programming language website](https://www.python.org/) and this list of [free programming books and tutorials](https://github.com/EbookFoundation/free-programming-books/blob/master/books/free-programming-books-langs.md#python). +There are many excellent online resources for learning Python; you should choose those that reference Python 3, as Kedro is built for Python 3.9+. There are curated lists of online resources, such as the [official Python programming language website](https://www.python.org/) and this list of [free programming books and tutorials](https://github.com/EbookFoundation/free-programming-books/blob/master/books/free-programming-books-langs.md#python). ``` diff --git a/docs/source/notebooks_and_ipython/notebook-example/add_kedro_to_a_notebook.ipynb b/docs/source/notebooks_and_ipython/notebook-example/add_kedro_to_a_notebook.ipynb index 3c036386cc..1e91ef0d29 100644 --- a/docs/source/notebooks_and_ipython/notebook-example/add_kedro_to_a_notebook.ipynb +++ b/docs/source/notebooks_and_ipython/notebook-example/add_kedro_to_a_notebook.ipynb @@ -683,7 +683,6 @@ "####################\n", "# Data processing #\n", "####################\n", - "from typing import Dict, Tuple\n", "\n", "import pandas as pd\n", "from sklearn.linear_model import LinearRegression\n", @@ -736,7 +735,7 @@ "##################################\n", "\n", "\n", - "def split_data(data: pd.DataFrame, parameters: Dict) -> Tuple:\n", + "def split_data(data: pd.DataFrame, parameters: dict) -> tuple:\n", " X = data[parameters[\"features\"]]\n", " y = data[\"price\"]\n", " X_train, X_test, y_train, y_test = train_test_split(\n", @@ -796,7 +795,6 @@ "outputs": [], "source": [ "# Kedro setup for data management and configuration\n", - "from typing import Dict, Tuple\n", "\n", "import pandas as pd\n", "from sklearn.linear_model import LinearRegression\n", @@ -873,7 +871,7 @@ "##################################\n", "\n", "\n", - "def split_data(data: pd.DataFrame, parameters: Dict) -> Tuple:\n", + "def split_data(data: pd.DataFrame, parameters: dict) -> tuple:\n", " X = data[parameters[\"features\"]]\n", " y = data[\"price\"]\n", " X_train, X_test, y_train, y_test = train_test_split(\n", diff --git a/docs/source/tutorial/tutorial_template.md b/docs/source/tutorial/tutorial_template.md index d8462f1b20..1586487707 100644 --- a/docs/source/tutorial/tutorial_template.md +++ b/docs/source/tutorial/tutorial_template.md @@ -32,7 +32,6 @@ The spaceflights project dependencies are stored in `requirements.txt`(you may f ```text # code quality packages -ipython>=7.31.1, <8.0; python_version < '3.8' ipython~=8.10; python_version >= '3.8' ruff==0.1.8 diff --git a/features/environment.py b/features/environment.py index 75bd183ed8..f1d45df926 100644 --- a/features/environment.py +++ b/features/environment.py @@ -5,7 +5,6 @@ import os import shutil import subprocess -import sys import tempfile import venv from pathlib import Path @@ -131,11 +130,6 @@ def _install_project_requirements(context): .splitlines() ) install_reqs = [req for req in install_reqs if "{" not in req and "#" not in req] - # For Python versions 3.9 and above we use the new dataset dependency format introduced in `kedro-datasets` 3.0.0 - if sys.version_info.minor > MINOR_PYTHON_38_VERSION: - install_reqs.append("kedro-datasets[pandas-csvdataset]") - # For Python 3.8 we use the older `kedro-datasets` dependency format - else: - install_reqs.append("kedro-datasets[pandas.CSVDataset]") + install_reqs.append("kedro-datasets[pandas-csvdataset]") call([context.pip, "install", *install_reqs], env=context.env) return context diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt b/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt index b07568d9da..014df14d12 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/requirements.txt @@ -2,5 +2,4 @@ ipython>=8.10 jupyterlab>=3.0 notebook kedro~={{ cookiecutter.kedro_version}} -kedro-datasets[pandas-csvdataset]; python_version >= "3.9" -kedro-datasets[pandas.CSVDataset]<2.0.0; python_version < '3.9' +kedro-datasets[pandas-csvdataset] diff --git a/features/steps/util.py b/features/steps/util.py index 437b3f6f5e..588e4530e1 100644 --- a/features/steps/util.py +++ b/features/steps/util.py @@ -6,9 +6,10 @@ import re from contextlib import contextmanager from time import sleep, time -from typing import TYPE_CHECKING, Any, Callable, Iterator +from typing import TYPE_CHECKING, Any, Callable if TYPE_CHECKING: + from collections.abc import Iterator from pathlib import Path diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index c4850159a1..8d82ebf360 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -8,10 +8,10 @@ import logging import mimetypes import typing -from collections.abc import KeysView +from collections.abc import Iterable, KeysView from enum import Enum, auto from pathlib import Path -from typing import Any, Callable, Iterable +from typing import Any, Callable import fsspec from omegaconf import DictConfig, OmegaConf diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index b22fa70d00..f5917e1b87 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -10,10 +10,13 @@ import traceback from collections import defaultdict from pathlib import Path -from typing import Any, Sequence +from typing import TYPE_CHECKING, Any import click +if TYPE_CHECKING: + from collections.abc import Sequence + from kedro import __version__ as version from kedro.framework.cli import BRIGHT_BLACK, ORANGE from kedro.framework.cli.hooks import get_cli_hook_manager diff --git a/kedro/framework/cli/micropkg.py b/kedro/framework/cli/micropkg.py index d2733ff712..a80efd172a 100644 --- a/kedro/framework/cli/micropkg.py +++ b/kedro/framework/cli/micropkg.py @@ -12,7 +12,8 @@ import toml from importlib import import_module from pathlib import Path -from typing import Any, Iterable, Iterator, TYPE_CHECKING +from typing import Any, TYPE_CHECKING + import click from omegaconf import OmegaConf @@ -42,6 +43,7 @@ if TYPE_CHECKING: from kedro.framework.startup import ProjectMetadata from importlib_metadata import PackageMetadata + from collections.abc import Iterable, Iterator _PYPROJECT_TOML_TEMPLATE = """ [build-system] diff --git a/kedro/framework/cli/utils.py b/kedro/framework/cli/utils.py index e165c15579..1b50408cc5 100644 --- a/kedro/framework/cli/utils.py +++ b/kedro/framework/cli/utils.py @@ -15,10 +15,14 @@ import typing import warnings from collections import defaultdict +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence from importlib import import_module from itertools import chain from pathlib import Path -from typing import IO, Any, Callable, Iterable, Sequence +from typing import IO, Any, Callable import click import importlib_metadata @@ -55,7 +59,7 @@ def call(cmd: list[str], **kwargs: Any) -> None: # pragma: no cover Raises: click.exceptions.Exit: If `subprocess.run` returns non-zero code. """ - click.echo(" ".join(shlex.quote(c) for c in cmd)) + click.echo(shlex.join(cmd)) code = subprocess.run(cmd, **kwargs).returncode # noqa: PLW1510, S603 if code: raise click.exceptions.Exit(code=code) diff --git a/kedro/framework/hooks/manager.py b/kedro/framework/hooks/manager.py index 5cbbcf9f27..ceec064246 100644 --- a/kedro/framework/hooks/manager.py +++ b/kedro/framework/hooks/manager.py @@ -3,7 +3,8 @@ """ import logging -from typing import Any, Iterable +from collections.abc import Iterable +from typing import Any from pluggy import PluginManager diff --git a/kedro/framework/session/session.py b/kedro/framework/session/session.py index caa3553954..ec0dc9bf4d 100644 --- a/kedro/framework/session/session.py +++ b/kedro/framework/session/session.py @@ -11,7 +11,7 @@ import traceback from copy import deepcopy from pathlib import Path -from typing import TYPE_CHECKING, Any, Iterable +from typing import TYPE_CHECKING, Any import click @@ -28,6 +28,8 @@ from kedro.utils import _find_kedro_project if TYPE_CHECKING: + from collections.abc import Iterable + from kedro.config import AbstractConfigLoader from kedro.framework.context import KedroContext from kedro.framework.session.store import BaseSessionStore diff --git a/kedro/io/catalog_config_resolver.py b/kedro/io/catalog_config_resolver.py index aeafea81e3..dc55d18b3c 100644 --- a/kedro/io/catalog_config_resolver.py +++ b/kedro/io/catalog_config_resolver.py @@ -7,13 +7,13 @@ import copy import logging import re -from typing import Any, Dict +from typing import Any from parse import parse from kedro.io.core import DatasetError -Patterns = Dict[str, Dict[str, Any]] +Patterns = dict[str, dict[str, Any]] CREDENTIALS_KEY = "credentials" diff --git a/kedro/io/core.py b/kedro/io/core.py index 036babc829..53b660835c 100644 --- a/kedro/io/core.py +++ b/kedro/io/core.py @@ -505,20 +505,13 @@ def parse_dataset_definition( class_obj = tmp break else: - hint = "" - if "DataSet" in dataset_type: - hint = ( # pragma: no cover # To remove when we drop support for python 3.8 - "Hint: If you are trying to use a dataset from `kedro-datasets`>=2.0.0, " - "make sure that the dataset name uses the `Dataset` spelling instead of `DataSet`." - ) - else: - hint = ( - "Hint: If you are trying to use a dataset from `kedro-datasets`, " - "make sure that the package is installed in your current environment. " - "You can do so by running `pip install kedro-datasets` or " - "`pip install kedro-datasets[]` to install `kedro-datasets` along with " - "related dependencies for the specific dataset group." - ) + hint = ( + "Hint: If you are trying to use a dataset from `kedro-datasets`, " + "make sure that the package is installed in your current environment. " + "You can do so by running `pip install kedro-datasets` or " + "`pip install kedro-datasets[]` to install `kedro-datasets` along with " + "related dependencies for the specific dataset group." + ) raise DatasetError( f"Class '{dataset_type}' not found, is this a typo?" f"\n{hint}" ) diff --git a/kedro/ipython/__init__.py b/kedro/ipython/__init__.py index 7cdbf92138..0e479e8a68 100644 --- a/kedro/ipython/__init__.py +++ b/kedro/ipython/__init__.py @@ -14,7 +14,10 @@ import warnings from pathlib import Path from types import MappingProxyType -from typing import Any, Callable, OrderedDict +from typing import TYPE_CHECKING, Any, Callable + +if TYPE_CHECKING: + from collections import OrderedDict from IPython.core.getipython import get_ipython from IPython.core.magic import needs_local_scope, register_line_magic diff --git a/kedro/pipeline/modular_pipeline.py b/kedro/pipeline/modular_pipeline.py index 172a5f9593..c6372f37a4 100644 --- a/kedro/pipeline/modular_pipeline.py +++ b/kedro/pipeline/modular_pipeline.py @@ -4,13 +4,15 @@ import copy import difflib -from typing import TYPE_CHECKING, AbstractSet, Iterable +from typing import TYPE_CHECKING from kedro.pipeline.pipeline import Pipeline from .transcoding import TRANSCODING_SEPARATOR, _strip_transcoding, _transcode_split if TYPE_CHECKING: + from collections.abc import Iterable, Set + from kedro.pipeline.node import Node @@ -35,7 +37,7 @@ def _is_parameter(name: str) -> bool: def _validate_inputs_outputs( - inputs: AbstractSet[str], outputs: AbstractSet[str], pipe: Pipeline + inputs: Set[str], outputs: Set[str], pipe: Pipeline ) -> None: """Safeguards to ensure that: - parameters are not specified under inputs @@ -64,9 +66,9 @@ def _validate_inputs_outputs( def _validate_datasets_exist( - inputs: AbstractSet[str], - outputs: AbstractSet[str], - parameters: AbstractSet[str], + inputs: Set[str], + outputs: Set[str], + parameters: Set[str], pipe: Pipeline, ) -> None: """Validate that inputs, parameters and outputs map correctly onto the provided nodes.""" diff --git a/kedro/pipeline/node.py b/kedro/pipeline/node.py index 09a83410aa..b382bee8cf 100644 --- a/kedro/pipeline/node.py +++ b/kedro/pipeline/node.py @@ -9,13 +9,16 @@ import logging import re from collections import Counter -from typing import Any, Callable, Iterable +from typing import TYPE_CHECKING, Any, Callable from warnings import warn from more_itertools import spy, unzip from .transcoding import _strip_transcoding +if TYPE_CHECKING: + from collections.abc import Iterable + class Node: """``Node`` is an auxiliary class facilitating the operations required to diff --git a/kedro/pipeline/pipeline.py b/kedro/pipeline/pipeline.py index 7810a98049..ab7365a154 100644 --- a/kedro/pipeline/pipeline.py +++ b/kedro/pipeline/pipeline.py @@ -8,16 +8,18 @@ import json from collections import Counter, defaultdict -from itertools import chain -from typing import Any, Iterable - from graphlib import CycleError, TopologicalSorter +from itertools import chain +from typing import TYPE_CHECKING, Any import kedro from kedro.pipeline.node import Node, _to_list from .transcoding import _strip_transcoding +if TYPE_CHECKING: + from collections.abc import Iterable + def __getattr__(name: str) -> Any: if name == "TRANSCODING_SEPARATOR": diff --git a/kedro/pipeline/transcoding.py b/kedro/pipeline/transcoding.py index 71f0dac342..eae9a10cf7 100644 --- a/kedro/pipeline/transcoding.py +++ b/kedro/pipeline/transcoding.py @@ -1,9 +1,7 @@ -from typing import Tuple - TRANSCODING_SEPARATOR = "@" -def _transcode_split(element: str) -> Tuple[str, str]: +def _transcode_split(element: str) -> tuple[str, str]: """Split the name by the transcoding separator. If the transcoding part is missing, empty string will be put in. diff --git a/kedro/runner/parallel_runner.py b/kedro/runner/parallel_runner.py index d09601ff7e..7626bf8679 100644 --- a/kedro/runner/parallel_runner.py +++ b/kedro/runner/parallel_runner.py @@ -13,7 +13,7 @@ from multiprocessing.managers import BaseProxy, SyncManager from multiprocessing.reduction import ForkingPickler from pickle import PicklingError -from typing import TYPE_CHECKING, Any, Iterable +from typing import TYPE_CHECKING, Any from kedro.framework.hooks.manager import ( _create_hook_manager, @@ -30,6 +30,8 @@ from kedro.runner.runner import AbstractRunner, run_node if TYPE_CHECKING: + from collections.abc import Iterable + from pluggy import PluginManager from kedro.pipeline import Pipeline diff --git a/kedro/runner/runner.py b/kedro/runner/runner.py index f3a0889909..f6716e070f 100644 --- a/kedro/runner/runner.py +++ b/kedro/runner/runner.py @@ -9,6 +9,7 @@ import logging from abc import ABC, abstractmethod from collections import deque +from collections.abc import Iterator from concurrent.futures import ( ALL_COMPLETED, Future, @@ -16,7 +17,7 @@ as_completed, wait, ) -from typing import TYPE_CHECKING, Any, Collection, Iterable, Iterator +from typing import TYPE_CHECKING, Any from more_itertools import interleave @@ -25,6 +26,8 @@ from kedro.pipeline import Pipeline if TYPE_CHECKING: + from collections.abc import Collection, Iterable + from pluggy import PluginManager from kedro.pipeline.node import Node @@ -369,7 +372,7 @@ def _find_initial_node_group(pipeline: Pipeline, nodes: Iterable[Node]) -> list[ A list of initial ``Node``s to run given inputs (in topological order). """ - node_names = set(n.name for n in nodes) + node_names = {n.name for n in nodes} if len(node_names) == 0: return [] sub_pipeline = pipeline.only_nodes(*node_names) diff --git a/kedro/templates/project/hooks/utils.py b/kedro/templates/project/hooks/utils.py index c112d72bbb..595884248e 100644 --- a/kedro/templates/project/hooks/utils.py +++ b/kedro/templates/project/hooks/utils.py @@ -36,7 +36,7 @@ def _remove_from_file(file_path: Path, content_to_remove: str) -> None: file_path (Path): The path of the file from which to remove content. content_to_remove (str): The content to be removed from the file. """ - with open(file_path, "r") as file: + with open(file_path) as file: lines = file.readlines() # Split the content to remove into lines and remove trailing whitespaces/newlines @@ -86,7 +86,7 @@ def _remove_from_toml(file_path: Path, sections_to_remove: list) -> None: sections_to_remove (list): A list of section keys to remove from the TOML file. """ # Load the TOML file - with open(file_path, "r") as file: + with open(file_path) as file: data = toml.load(file) # Remove the specified sections @@ -162,7 +162,7 @@ def _remove_extras_from_kedro_datasets(file_path: Path) -> None: Args: file_path (Path): The path of the requirements file. """ - with open(file_path, "r") as file: + with open(file_path) as file: lines = file.readlines() for i, line in enumerate(lines): diff --git a/pyproject.toml b/pyproject.toml index d9ebbfd70b..23b60e9a61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ {name = "Kedro"} ] description = "Kedro helps you build production-ready data and analytics pipelines" -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "attrs>=21.3", "build>=0.7.0", @@ -33,7 +33,6 @@ dependencies = [ "rope>=0.21,<2.0", # subject to LGPLv3 license "toml>=0.10.0", "typing_extensions>=4.0", - "graphlib_backport>=1.0.0; python_version < '3.9'", ] keywords = [ "pipelines", @@ -45,10 +44,10 @@ keywords = [ license = {text = "Apache Software License (Apache 2.0)"} classifiers = [ "Development Status :: 4 - Beta", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dynamic = ["readme", "version"] @@ -58,13 +57,11 @@ test = [ "coverage[toml]", "import-linter==2.0", "ipylab>=1.0.0", - "ipython>=7.31.1, <8.0; python_version < '3.8'", - "ipython~=8.10; python_version >= '3.8'", + "ipython~=8.10", "jupyterlab_server>=2.11.1", "jupyterlab>=3,<5", "jupyter~=1.0", - "kedro-datasets; python_version >= '3.9'", - "kedro-datasets<2.0.0; python_version < '3.9'", + "kedro-datasets", "mypy~=1.0", "pandas~=2.0", "pluggy>=1.0", diff --git a/tests/framework/cli/pipeline/test_pipeline.py b/tests/framework/cli/pipeline/test_pipeline.py index df01d685a6..099790221e 100644 --- a/tests/framework/cli/pipeline/test_pipeline.py +++ b/tests/framework/cli/pipeline/test_pipeline.py @@ -127,11 +127,11 @@ def test_create_pipeline_template_command_line_override( assert not (pipelines_dir / PIPELINE_NAME).exists() # Rename the local template dir to something else so we know the command line flag is taking precedence - try: - # Can skip if already there but copytree has a dirs_exist_ok flag in >python 3.8 only - shutil.copytree(fake_local_template_dir, fake_repo_path / "local_templates") - except FileExistsError: - pass + shutil.copytree( + fake_local_template_dir, + fake_repo_path / "local_templates", + dirs_exist_ok=True, + ) cmd = ["pipeline", "create", PIPELINE_NAME] cmd += ["-t", str(fake_repo_path / "local_templates/pipeline")] diff --git a/tests/framework/session/test_session.py b/tests/framework/session/test_session.py index 086d581045..1c67824d8d 100644 --- a/tests/framework/session/test_session.py +++ b/tests/framework/session/test_session.py @@ -5,7 +5,7 @@ import textwrap from collections.abc import Mapping from pathlib import Path -from typing import Any, Type +from typing import Any from unittest.mock import create_autospec import pytest @@ -50,7 +50,7 @@ class BadConfigLoader: NEW_TYPING = sys.version_info[:3] >= (3, 7, 0) # PEP 560 -def create_attrs_autospec(spec: Type, spec_set: bool = True) -> Any: +def create_attrs_autospec(spec: type, spec_set: bool = True) -> Any: """Creates a mock of an attr class (creates mocks recursively on all attributes). https://github.com/python-attrs/attrs/issues/462#issuecomment-1134656377 diff --git a/tests/io/test_data_catalog.py b/tests/io/test_data_catalog.py index 39c0f77307..54cbdf340d 100644 --- a/tests/io/test_data_catalog.py +++ b/tests/io/test_data_catalog.py @@ -1,6 +1,5 @@ import logging import re -import sys from copy import deepcopy from datetime import datetime, timezone from pathlib import Path @@ -463,23 +462,6 @@ def test_config_missing_class(self, correct_config): with pytest.raises(DatasetError, match=re.escape(pattern)): DataCatalog.from_config(**correct_config) - @pytest.mark.skipif( - sys.version_info < (3, 9), - reason="for python 3.8 kedro-datasets version 1.8 is used which has the old spelling", - ) - def test_config_incorrect_spelling(self, correct_config): - """Check hint if the type uses the old DataSet spelling""" - correct_config["catalog"]["boats"]["type"] = "pandas.CSVDataSet" - - pattern = ( - "An exception occurred when parsing config for dataset 'boats':\n" - "Class 'pandas.CSVDataSet' not found, is this a typo?" - "\nHint: If you are trying to use a dataset from `kedro-datasets`>=2.0.0," - " make sure that the dataset name uses the `Dataset` spelling instead of `DataSet`." - ) - with pytest.raises(DatasetError, match=re.escape(pattern)): - DataCatalog.from_config(**correct_config) - def test_config_invalid_dataset(self, correct_config): """Check the error if the type points to invalid class""" correct_config["catalog"]["boats"]["type"] = "DataCatalog" diff --git a/tests/io/test_kedro_data_catalog.py b/tests/io/test_kedro_data_catalog.py index b98e8fae83..5e0c463e7d 100644 --- a/tests/io/test_kedro_data_catalog.py +++ b/tests/io/test_kedro_data_catalog.py @@ -1,6 +1,5 @@ import logging import re -import sys from copy import deepcopy from datetime import datetime, timezone from pathlib import Path @@ -347,23 +346,6 @@ def test_config_missing_class(self, correct_config): with pytest.raises(DatasetError, match=re.escape(pattern)): KedroDataCatalog.from_config(**correct_config) - @pytest.mark.skipif( - sys.version_info < (3, 9), - reason="for python 3.8 kedro-datasets version 1.8 is used which has the old spelling", - ) - def test_config_incorrect_spelling(self, correct_config): - """Check hint if the type uses the old DataSet spelling""" - correct_config["catalog"]["boats"]["type"] = "pandas.CSVDataSet" - - pattern = ( - "An exception occurred when parsing config for dataset 'boats':\n" - "Class 'pandas.CSVDataSet' not found, is this a typo?" - "\nHint: If you are trying to use a dataset from `kedro-datasets`>=2.0.0," - " make sure that the dataset name uses the `Dataset` spelling instead of `DataSet`." - ) - with pytest.raises(DatasetError, match=re.escape(pattern)): - KedroDataCatalog.from_config(**correct_config) - def test_config_invalid_dataset(self, correct_config): """Check the error if the type points to invalid class""" correct_config["catalog"]["boats"]["type"] = "KedroDataCatalog" diff --git a/tests/runner/test_resume_logic.py b/tests/runner/test_resume_logic.py index bd1f8e8acb..c733c504c5 100644 --- a/tests/runner/test_resume_logic.py +++ b/tests/runner/test_resume_logic.py @@ -153,6 +153,6 @@ def test_suggestion_consistency( test_pipeline, remaining_nodes, persistent_dataset_catalog ) - assert set(n.name for n in required_nodes) == set( + assert {n.name for n in required_nodes} == { n.name for n in test_pipeline.from_nodes(*resume_node_names).nodes - ) + }