From 47a29b6576bf5e8f296e2d9550aca8e4587c9c24 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 29 Jan 2025 16:28:34 -0500 Subject: [PATCH 1/3] fix: allow pypy- to be used (matching GHA) (#913) * fix: allow pypy- to be used (matching GHA) Signed-off-by: Henry Schreiner * ci: add Linux ARM * Update noxfile.py * Remove added ARM image * tests: add unit test for pypy behavior Signed-off-by: Henry Schreiner --------- Signed-off-by: Henry Schreiner --- nox/virtualenv.py | 5 +++++ noxfile.py | 5 +++-- tests/test_virtualenv.py | 10 ++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 2a8d8ba3..4836827e 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -453,6 +453,7 @@ class VirtualEnv(ProcessEnv): be ``py -3.6-32`` * ``X.Y.Z``, e.g. ``3.4.9`` * ``pythonX.Y``, e.g. ``python2.7`` + * ``pypyX.Y``, e.g. ``pypy3.10`` (also ``pypy-3.10`` allowed) * A path in the filesystem to a Python executable If not specified, this will use the currently running Python. @@ -472,6 +473,10 @@ def __init__( venv_backend: str = "virtualenv", venv_params: Sequence[str] = (), ): + # "pypy-" -> "pypy" + if interpreter and interpreter.startswith("pypy-"): + interpreter = interpreter[:4] + interpreter[5:] + self.location_name = location self.location = os.path.abspath(location) self.interpreter = interpreter diff --git a/noxfile.py b/noxfile.py index 75319631..81dda9ea 100644 --- a/noxfile.py +++ b/noxfile.py @@ -148,7 +148,8 @@ def docs(session: nox.Session) -> None: # The following sessions are only to be run in CI to check the nox GHA action def _check_python_version(session: nox.Session) -> None: if session.python.startswith("pypy"): - python_version = session.python[4:] + # Drop starting "pypy" and maybe "-" + python_version = session.python.lstrip("py-") implementation = "pypy" else: python_version = session.python @@ -170,7 +171,7 @@ def _check_python_version(session: nox.Session) -> None: @nox.session( python=[ *ALL_PYTHONS, - "pypy3.10", + "pypy-3.10", ], default=False, ) diff --git a/tests/test_virtualenv.py b/tests/test_virtualenv.py index 9f1a6e78..dcc620a1 100644 --- a/tests/test_virtualenv.py +++ b/tests/test_virtualenv.py @@ -322,6 +322,16 @@ def test_constructor_defaults( assert venv.venv_backend == "virtualenv" +def test_constructor_pypy_dash( + make_one: Callable[..., tuple[VirtualEnv, Path]], +) -> None: + venv, _ = make_one(interpreter="pypy-3.10") + assert venv.location + assert venv.interpreter == "pypy3.10" + assert venv.reuse_existing is False + assert venv.venv_backend == "virtualenv" + + @pytest.mark.skipif(IS_WINDOWS, reason="Not testing multiple interpreters on Windows.") def test_constructor_explicit( make_one: Callable[..., tuple[VirtualEnv, Path]], From 73534d2856253ef490f57a1a52a4a54926765c79 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 30 Jan 2025 18:15:43 -0500 Subject: [PATCH 2/3] fix: don't trigger a background update process for virtualenv (#918) Signed-off-by: Henry Schreiner --- nox/virtualenv.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nox/virtualenv.py b/nox/virtualenv.py index 4836827e..e416ed58 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -668,7 +668,13 @@ def create(self) -> bool: return False if self.venv_backend == "virtualenv": - cmd = [sys.executable, "-m", "virtualenv", self.location] + cmd = [ + sys.executable, + "-m", + "virtualenv", + self.location, + "--no-periodic-update", + ] if self.interpreter: cmd.extend(["-p", self._resolved_interpreter]) elif self.venv_backend == "uv": From beab6c32519c4aa1f18c53c3f4a9b75665e5258c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 31 Jan 2025 13:46:52 -0500 Subject: [PATCH 3/3] feat: support PEP 723 noxfiles (#881) * feat: support PEP 723 directly fix: version based on pipx fix: subprocess on Windows fix: uv or virtualenv, test fix: windows fix: ignore errors in rmtree (3.8+ should be fine on Windows now) fix: resolve nox path on Windows chore: update for recent linting/typing additions Signed-off-by: Henry Schreiner * feat: support setting the script backend Signed-off-by: Henry Schreiner --------- Signed-off-by: Henry Schreiner --- docs/tutorial.rst | 13 ++- nox/_cli.py | 126 ++++++++++++++++++++++++- nox/_options.py | 12 +++ nox/project.py | 13 ++- nox/virtualenv.py | 7 +- tests/resources/noxfile_script_mode.py | 12 +++ tests/test__cli.py | 101 ++++++++++++++++++++ tests/test_main.py | 45 +++++++++ 8 files changed, 319 insertions(+), 10 deletions(-) create mode 100644 tests/resources/noxfile_script_mode.py create mode 100644 tests/test__cli.py diff --git a/docs/tutorial.rst b/docs/tutorial.rst index bb4e20e3..c8f33ab8 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -597,8 +597,8 @@ the tags, so all three sessions: * flake8 -Running without the nox command -------------------------------- +Running without the nox command or adding dependencies +------------------------------------------------------ With a few small additions to your noxfile, you can support running using only a generalized Python runner, such as ``pipx run noxfile.py``, ``uv run @@ -618,6 +618,15 @@ And the following block of code: if __name__ == "__main__": nox.main() +If this comment block is present, nox will also read it, and run a custom +environment (``_nox_script_mode``) if the dependencies are not met in the +current environment. This allows you to specify dependencies for your noxfile +or a minimum version of nox here (``requires-python`` version setting not +supported yet, but planned). You can control this with +``--script-mode``/``NOX_SCRIPT_MODE``; ``none`` will deactivate it, and +``fresh`` will rebuild it; the default is ``reuse``. You can also set +``--script-venv-backend``/``tool.nox.script-venv-backend``/``NOX_SCRIPT_VENV_BACKEND`` +to control the backend used; the default is ``"uv|virtualenv"``. Next steps ---------- diff --git a/nox/_cli.py b/nox/_cli.py index bc69fc39..f1807c59 100644 --- a/nox/_cli.py +++ b/nox/_cli.py @@ -16,12 +16,26 @@ from __future__ import annotations +import importlib.metadata +import os +import shutil +import subprocess import sys -from typing import Any +from pathlib import Path +from typing import TYPE_CHECKING, Any, NoReturn +import packaging.requirements +import packaging.utils + +import nox.command +import nox.virtualenv from nox import _options, tasks, workflow from nox._version import get_nox_version from nox.logger import setup_logging +from nox.project import load_toml + +if TYPE_CHECKING: + from collections.abc import Generator __all__ = ["execute_workflow", "main"] @@ -51,6 +65,88 @@ def execute_workflow(args: Any) -> int: ) +def get_dependencies( + req: packaging.requirements.Requirement, +) -> Generator[packaging.requirements.Requirement, None, None]: + """ + Gets all dependencies. Raises ModuleNotFoundError if a package is not installed. + """ + info = importlib.metadata.metadata(req.name) + yield req + + dist_list = info.get_all("requires-dist") or [] + extra_list = [packaging.requirements.Requirement(mk) for mk in dist_list] + for extra in req.extras: + for ireq in extra_list: + if ireq.marker and not ireq.marker.evaluate({"extra": extra}): + continue + yield from get_dependencies(ireq) + + +def check_dependencies(dependencies: list[str]) -> bool: + """ + Checks to see if a list of dependencies is currently installed. + """ + itr_deps = (packaging.requirements.Requirement(d) for d in dependencies) + deps = [d for d in itr_deps if not d.marker or d.marker.evaluate()] + + # Select the one nox dependency (required) + nox_dep = [d for d in deps if packaging.utils.canonicalize_name(d.name) == "nox"] + if not nox_dep: + msg = "Must have a nox dependency in TOML script dependencies" + raise ValueError(msg) + + try: + expanded_deps = {d for req in deps for d in get_dependencies(req)} + except ModuleNotFoundError: + return False + + for dep in expanded_deps: + if dep.specifier: + version = importlib.metadata.version(dep.name) + if not dep.specifier.contains(version): + return False + + return True + + +def run_script_mode( + envdir: Path, *, reuse: bool, dependencies: list[str], venv_backend: str +) -> NoReturn: + envdir.mkdir(exist_ok=True) + noxenv = envdir.joinpath("_nox_script_mode") + venv = nox.virtualenv.get_virtualenv( + *venv_backend.split("|"), + reuse_existing=reuse, + envdir=str(noxenv), + ) + venv.create() + env = {k: v for k, v in venv._get_env({}).items() if v is not None} + env["NOX_SCRIPT_MODE"] = "none" + cmd = ( + [nox.virtualenv.UV, "pip", "install"] + if venv.venv_backend == "uv" + else ["pip", "install"] + ) + subprocess.run([*cmd, *dependencies], env=env, check=True) + nox_cmd = shutil.which("nox", path=env["PATH"]) + assert nox_cmd is not None, "Nox must be discoverable when installed" + # The os.exec functions don't work properly on Windows + if sys.platform.startswith("win"): + raise SystemExit( + subprocess.run( + [nox_cmd, *sys.argv[1:]], + env=env, + stdout=None, + stderr=None, + encoding="utf-8", + text=True, + check=False, + ).returncode + ) + os.execle(nox_cmd, nox_cmd, *sys.argv[1:], env) # pragma: nocover # noqa: S606 + + def main() -> None: args = _options.options.parse_args() @@ -65,6 +161,34 @@ def main() -> None: setup_logging( color=args.color, verbose=args.verbose, add_timestamp=args.add_timestamp ) + nox_script_mode = os.environ.get("NOX_SCRIPT_MODE", "") or args.script_mode + if nox_script_mode not in {"none", "reuse", "fresh"}: + msg = f"Invalid NOX_SCRIPT_MODE: {nox_script_mode!r}, must be one of 'none', 'reuse', or 'fresh'" + raise SystemExit(msg) + if nox_script_mode != "none": + toml_config = load_toml(os.path.expandvars(args.noxfile), missing_ok=True) + dependencies = toml_config.get("dependencies") + if dependencies is not None: + valid_env = check_dependencies(dependencies) + # Coverage misses this, but it's covered via subprocess call + if not valid_env: # pragma: nocover + venv_backend = ( + os.environ.get("NOX_SCRIPT_VENV_BACKEND") + or args.script_venv_backend + or ( + toml_config.get("tool", {}) + .get("nox", {}) + .get("script-venv-backend", "uv|virtualenv") + ) + ) + + envdir = Path(args.envdir or ".nox") + run_script_mode( + envdir, + reuse=nox_script_mode == "reuse", + dependencies=dependencies, + venv_backend=venv_backend, + ) exit_code = execute_workflow(args) diff --git a/nox/_options.py b/nox/_options.py index 10c9515e..adb0e2a0 100644 --- a/nox/_options.py +++ b/nox/_options.py @@ -338,6 +338,18 @@ def _tag_completer( action="store_true", help="Show the Nox version and exit.", ), + _option_set.Option( + "script_mode", + "--script-mode", + group=options.groups["general"], + choices=["none", "fresh", "reuse"], + default="reuse", + ), + _option_set.Option( + "script_venv_backend", + "--script-venv-backend", + group=options.groups["general"], + ), _option_set.Option( "list_sessions", "-l", diff --git a/nox/project.py b/nox/project.py index d808c9a9..627d1e6a 100644 --- a/nox/project.py +++ b/nox/project.py @@ -34,7 +34,9 @@ def __dir__() -> list[str]: ) -def load_toml(filename: os.PathLike[str] | str) -> dict[str, Any]: +def load_toml( + filename: os.PathLike[str] | str, *, missing_ok: bool = False +) -> dict[str, Any]: """ Load a toml file or a script with a PEP 723 script block. @@ -42,6 +44,9 @@ def load_toml(filename: os.PathLike[str] | str) -> dict[str, Any]: ``.py`` extension / no extension to be considered a script. Other file extensions are not valid in this function. + If ``missing_ok``, this will return an empty dict if a script block was not + found, otherwise it will raise a error. + Example: .. code-block:: python @@ -55,7 +60,7 @@ def myscript(session): if filepath.suffix == ".toml": return _load_toml_file(filepath) if filepath.suffix in {".py", ""}: - return _load_script_block(filepath) + return _load_script_block(filepath, missing_ok=missing_ok) msg = f"Extension must be .py or .toml, got {filepath.suffix}" raise ValueError(msg) @@ -65,12 +70,14 @@ def _load_toml_file(filepath: Path) -> dict[str, Any]: return tomllib.load(f) -def _load_script_block(filepath: Path) -> dict[str, Any]: +def _load_script_block(filepath: Path, *, missing_ok: bool) -> dict[str, Any]: name = "script" script = filepath.read_text(encoding="utf-8") matches = list(filter(lambda m: m.group("type") == name, REGEX.finditer(script))) if not matches: + if missing_ok: + return {} msg = f"No {name} block found in {filepath}" raise ValueError(msg) if len(matches) > 1: diff --git a/nox/virtualenv.py b/nox/virtualenv.py index e416ed58..66f81ea6 100644 --- a/nox/virtualenv.py +++ b/nox/virtualenv.py @@ -353,7 +353,7 @@ def _clean_location(self) -> bool: if self.reuse_existing and is_conda: return False if not is_conda: - shutil.rmtree(self.location) + shutil.rmtree(self.location, ignore_errors=True) else: cmd = [ self.conda_cmd, @@ -365,8 +365,7 @@ def _clean_location(self) -> bool: ] nox.command.run(cmd, silent=True, log=False) # Make sure that location is clean - with contextlib.suppress(FileNotFoundError): - shutil.rmtree(self.location) + shutil.rmtree(self.location, ignore_errors=True) return True @@ -498,7 +497,7 @@ def _clean_location(self) -> bool: and self._check_reused_environment_interpreter() ): return False - shutil.rmtree(self.location) + shutil.rmtree(self.location, ignore_errors=True) return True def _read_pyvenv_cfg(self) -> dict[str, str] | None: diff --git a/tests/resources/noxfile_script_mode.py b/tests/resources/noxfile_script_mode.py new file mode 100644 index 00000000..d95d0bb2 --- /dev/null +++ b/tests/resources/noxfile_script_mode.py @@ -0,0 +1,12 @@ +# /// script +# dependencies = ["nox", "cowsay"] +# /// + +import cowsay + +import nox + + +@nox.session +def example(session: nox.Session) -> None: + print(cowsay.cow("hello_world")) diff --git a/tests/test__cli.py b/tests/test__cli.py new file mode 100644 index 00000000..e210862c --- /dev/null +++ b/tests/test__cli.py @@ -0,0 +1,101 @@ +import importlib.metadata +import importlib.util +import sys +from pathlib import Path + +import packaging.requirements +import packaging.version +import pytest + +import nox._cli + + +def test_get_dependencies() -> None: + if importlib.util.find_spec("tox") is None: + with pytest.raises(ModuleNotFoundError): + list( + nox._cli.get_dependencies( + packaging.requirements.Requirement("nox[tox_to_nox]") + ) + ) + else: + deps = nox._cli.get_dependencies( + packaging.requirements.Requirement("nox[tox_to_nox]") + ) + dep_list = { + "argcomplete", + "attrs", + "colorlog", + "dependency-groups", + "jinja2", + "nox", + "packaging", + "tox", + "virtualenv", + } + if sys.version_info < (3, 9): + dep_list.add("importlib-resources") + if sys.version_info < (3, 11): + dep_list.add("tomli") + assert {d.name for d in deps} == dep_list + + +def test_version_check() -> None: + current_version = packaging.version.Version(importlib.metadata.version("nox")) + + assert nox._cli.check_dependencies([f"nox>={current_version}"]) + assert not nox._cli.check_dependencies([f"nox>{current_version}"]) + + plus_one = packaging.version.Version( + f"{current_version.major}.{current_version.minor}.{current_version.micro + 1}" + ) + assert not nox._cli.check_dependencies([f"nox>={plus_one}"]) + + +def test_nox_check() -> None: + with pytest.raises(ValueError, match="Must have a nox"): + nox._cli.check_dependencies(["packaging"]) + + with pytest.raises(ValueError, match="Must have a nox"): + nox._cli.check_dependencies([]) + + +def test_unmatched_specifier() -> None: + assert not nox._cli.check_dependencies(["packaging<1", "nox"]) + + +def test_invalid_mode(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NOX_SCRIPT_MODE", "invalid") + monkeypatch.setattr(sys, "argv", ["nox"]) + + with pytest.raises(SystemExit, match="Invalid NOX_SCRIPT_MODE"): + nox._cli.main() + + +def test_invalid_backend_envvar( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("NOX_SCRIPT_VENV_BACKEND", "invalid") + monkeypatch.setattr(sys, "argv", ["nox"]) + monkeypatch.chdir(tmp_path) + tmp_path.joinpath("noxfile.py").write_text( + "# /// script\n# dependencies=['nox', 'invalid']\n# ///", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="Expected venv_backend one of"): + nox._cli.main() + + +def test_invalid_backend_inline( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(sys, "argv", ["nox"]) + monkeypatch.chdir(tmp_path) + tmp_path.joinpath("noxfile.py").write_text( + "# /// script\n# dependencies=['nox', 'invalid']\n# tool.nox.script-venv-backend = 'invalid'\n# ///", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="Expected venv_backend one of"): + nox._cli.main() diff --git a/tests/test_main.py b/tests/test_main.py index f1bda816..16258f44 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1020,3 +1020,48 @@ def test_symlink_sym_not(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(Path(RESOURCES) / "sym_dir") res = subprocess.run([sys.executable, "-m", "nox", "-s", "orig"], check=False) assert res.returncode == 1 + + +def test_noxfile_script_mode() -> None: + job = subprocess.run( + [ + sys.executable, + "-m", + "nox", + "-f", + Path(RESOURCES) / "noxfile_script_mode.py", + "-s", + "example", + ], + check=False, + capture_output=True, + text=True, + encoding="utf-8", + ) + print(job.stdout) + print(job.stderr) + assert job.returncode == 0 + assert "hello_world" in job.stdout + + +def test_noxfile_no_script_mode() -> None: + env = os.environ.copy() + env["NOX_SCRIPT_MODE"] = "none" + job = subprocess.run( + [ + sys.executable, + "-m", + "nox", + "-f", + Path(RESOURCES) / "noxfile_script_mode.py", + "-s", + "example", + ], + env=env, + check=False, + capture_output=True, + text=True, + encoding="utf-8", + ) + assert job.returncode == 1 + assert "No module named 'cowsay'" in job.stderr