Make kedro commands work from inside subdirectories in project (#3683)

* Move _find_kedro_project to utils

Signed-off-by: Ankita Katiyar <[email protected]>

* Refactor fn

Signed-off-by: Ankita Katiyar <[email protected]>

* Fix _find_kedro_project

Signed-off-by: Ankita Katiyar <[email protected]>

* Make kedro commands available inside subdirectories

Signed-off-by: Ankita Katiyar <[email protected]>

* Add e2e test and update release notes

Signed-off-by: Ankita Katiyar <[email protected]>

* Fix e2e test

Signed-off-by: Ankita Katiyar <[email protected]>

* Resolve paths for session and update release notes

Signed-off-by: Ankita Katiyar <[email protected]>

---------

Signed-off-by: Ankita Katiyar <[email protected]>
Signed-off-by: Ankita Katiyar <[email protected]>

RELEASE.md

@@ -1,12 +1,14 @@
 # Upcoming Release 0.19.4
 
 ## Major features and improvements
+* Kedro commands now work from any subdirectory within a Kedro project.
 * Kedro CLI now provides a better error message when project commands are run outside of a project i.e. `kedro run`
 
 ## Bug fixes and other changes
 * Updated `kedro pipeline create` and `kedro pipeline delete` to read the base environment from the project settings.
 
 ## Breaking changes to the API
+* Methods `_is_project` and `_find_kedro_project` have been moved to `kedro.utils`. We recommend not using private methods in your code, but if you do, please update your code to use the new location.
 
 ## Documentation changes
 

features/run.feature

@@ -60,3 +60,11 @@ Feature: Run Project
     When I execute the kedro command "run --params extra1=1,extra2=value2"
     Then I should get a successful exit code
     And the logs should show that 4 nodes were run
+
+  Scenario: Run kedro run from within a sub-directory
+    Given I have prepared a config file
+    And I have run a non-interactive kedro new with starter "default"
+    And I have changed the current working directory to "data"
+    When I execute the kedro command "run"
+    Then I should get a successful exit code
+    And the logs should show that 4 nodes were run

features/steps/cli_steps.py

@@ -732,3 +732,9 @@ def add_micropkg_to_pyproject_toml(context: behave.runner.Context):
     )
     with pyproject_toml_path.open(mode="a") as file:
         file.write(project_toml_str)
+
+
+@given('I have changed the current working directory to "{dir}"')
+def change_dir(context, dir):
+    """Execute Kedro target."""
+    util.chdir(dir)

kedro/framework/cli/cli.py

@@ -32,7 +32,8 @@
     load_entry_points,
 )
 from kedro.framework.project import LOGGING  # noqa: F401
-from kedro.framework.startup import _is_project, bootstrap_project
+from kedro.framework.startup import bootstrap_project
+from kedro.utils import _find_kedro_project, _is_project
 
 LOGO = rf"""
  _            _
@@ -226,5 +227,7 @@ def main() -> None:  # pragma: no cover
     commands to `kedro`'s before invoking the CLI.
     """
     _init_plugins()
-    cli_collection = KedroCLI(project_path=Path.cwd())
+    cli_collection = KedroCLI(
+        project_path=_find_kedro_project(Path.cwd()) or Path.cwd()
+    )
     cli_collection()

kedro/framework/session/session.py

@@ -27,6 +27,7 @@
 from kedro.framework.session.store import BaseSessionStore
 from kedro.io.core import generate_timestamp
 from kedro.runner import AbstractRunner, SequentialRunner
+from kedro.utils import _find_kedro_project
 
 
 def _describe_git(project_path: Path) -> dict[str, dict[str, Any]]:
@@ -104,7 +105,9 @@ def __init__(  # noqa: PLR0913
         save_on_close: bool = False,
         conf_source: str | None = None,
     ):
-        self._project_path = Path(project_path or Path.cwd()).resolve()
+        self._project_path = Path(
+            project_path or _find_kedro_project(Path.cwd()) or Path.cwd()
+        ).resolve()
         self.session_id = session_id
         self.save_on_close = save_on_close
         self._package_name = package_name

kedro/framework/startup.py

@@ -34,17 +34,6 @@ def _version_mismatch_error(kedro_init_version: str) -> str:
     )
 
 
-def _is_project(project_path: Union[str, Path]) -> bool:
-    metadata_file = Path(project_path).expanduser().resolve() / _PYPROJECT
-    if not metadata_file.is_file():
-        return False
-
-    try:
-        return "[tool.kedro]" in metadata_file.read_text(encoding="utf-8")
-    except Exception:  # noqa: broad-except
-        return False
-
-
 def _get_project_metadata(project_path: Union[str, Path]) -> ProjectMetadata:
     """Read project metadata from `<project_root>/pyproject.toml` config file,
     under the `[tool.kedro]` section.

kedro/ipython/__init__.py

@@ -31,9 +31,9 @@
     pipelines,
 )
 from kedro.framework.session import KedroSession
-from kedro.framework.startup import _is_project, bootstrap_project
+from kedro.framework.startup import bootstrap_project
 from kedro.pipeline.node import Node
-from kedro.utils import _is_databricks
+from kedro.utils import _find_kedro_project, _is_databricks
 
 logger = logging.getLogger(__name__)
 
@@ -186,15 +186,6 @@ def _remove_cached_modules(package_name: str) -> None:  # pragma: no cover
         del sys.modules[module]
 
 
-def _find_kedro_project(current_dir: Path) -> Any:  # pragma: no cover
-    while current_dir != current_dir.parent:
-        if _is_project(current_dir):
-            return current_dir
-        current_dir = current_dir.parent
-
-    return None
-
-
 def _guess_run_environment() -> str:  # pragma: no cover
     """Best effort to guess the IPython/Jupyter environment"""
     # https://github.com/microsoft/vscode-jupyter/issues/7380

kedro/utils.py

@@ -3,7 +3,10 @@
 """
 import importlib
 import os
-from typing import Any
+from pathlib import Path
+from typing import Any, Union
+
+_PYPROJECT = "pyproject.toml"
 
 
 def load_obj(obj_path: str, default_obj_path: str = "") -> Any:
@@ -29,3 +32,22 @@ def load_obj(obj_path: str, default_obj_path: str = "") -> Any:
 
 def _is_databricks() -> bool:
     return "DATABRICKS_RUNTIME_VERSION" in os.environ
+
+
+def _is_project(project_path: Union[str, Path]) -> bool:
+    metadata_file = Path(project_path).expanduser().resolve() / _PYPROJECT
+    if not metadata_file.is_file():
+        return False
+
+    try:
+        return "[tool.kedro]" in metadata_file.read_text(encoding="utf-8")
+    except Exception:  # noqa: broad-except
+        return False
+
+
+def _find_kedro_project(current_dir: Path) -> Any:  # pragma: no cover
+    paths_to_check = [current_dir] + list(current_dir.parents)
+    for parent_dir in paths_to_check:
+        if _is_project(parent_dir):
+            return parent_dir
+    return None

tests/framework/test_startup.py

@@ -10,10 +10,10 @@
 from kedro.framework.startup import (
     ProjectMetadata,
     _get_project_metadata,
-    _is_project,
     _validate_source_path,
     bootstrap_project,
 )
+from kedro.utils import _is_project
 
 
 class TestIsProject: