diff --git a/.github/workflows/integtests.yaml b/.github/workflows/integtests.yaml index af5361f..e2a1d0b 100644 --- a/.github/workflows/integtests.yaml +++ b/.github/workflows/integtests.yaml @@ -29,7 +29,6 @@ jobs: bin/fades -v -d pytest -x pytest --version - name: Using a different Python run: | - pacman -S --noconfirm python-virtualenv export TEST_PYTHON_VERSION=3.9 python bin/fades -v --python=python3.9 -d pytest -x pytest -v tests/integtest.py @@ -53,19 +52,55 @@ jobs: bin/fades -v -d pytest -x pytest --version - name: Using a different Python run: | - # XXX Facundo 2024-02-29 - remove virtualenv in this install as part of issue #411 work - yum install --assumeyes python3.9 python3-virtualenv + yum install --assumeyes python3.9 cd /fades export TEST_PYTHON_VERSION=3.9 python3.12 bin/fades -v --python=python3.9 -d pytest -x pytest -v tests/integtest.py - native: + native-windows: + strategy: + matrix: + # just a selection otherwise it's too much + # - latest OS (left here even if it's only one to simplify upgrading later) + # - oldest and newest Python + os: [windows-2022] + python-version: [3.8, "3.12"] + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} + uses: actions/setup-python@v5 + id: matrixpy + with: + python-version: ${{ matrix.python-version }} + + - name: Also set up Python 3.10 for cross-Python test + uses: actions/setup-python@v5 + id: otherpy + with: + python-version: "3.10" + + - name: Install dependencies + run: | + ${{ steps.matrixpy.outputs.python-path }} -m pip install -U packaging + - name: Simple fades run + run: | + ${{ steps.matrixpy.outputs.python-path }} bin/fades -v -d pytest -x pytest --version + + - name: Using a different Python + run: | + set TEST_PYTHON_VERSION=3.10 + ${{ steps.matrixpy.outputs.python-path }} bin/fades -v --python=${{ steps.otherpy.outputs.python-path }} -d pytest -x pytest -v tests/integtest.py + + native-generic: strategy: matrix: # just a selection otherwise it's too much # - latest OSes # - oldest and newest Python - os: [ubuntu-22.04, macos-12, windows-2022] + os: [ubuntu-22.04, macos-12] python-version: [3.8, "3.12"] runs-on: ${{ matrix.os }} @@ -78,11 +113,10 @@ jobs: with: python-version: ${{ matrix.python-version }} - # XXX Facundo 2024-03-02 - disabled until we enable the cross test below - # - name: Also set up Python 3.10 for cross-Python test - # uses: actions/setup-python@v4 - # with: - # python-version: "3.10" + - name: Also set up Python 3.10 for cross-Python test + uses: actions/setup-python@v5 + with: + python-version: "3.10" - name: Install dependencies run: | @@ -91,9 +125,7 @@ jobs: run: | ${{ steps.matrixpy.outputs.python-path }} bin/fades -v -d pytest -x pytest --version - # XXX Facundo 2024-03-02 - commented out as until we finish issue #411 work we need 'virtualenv' and it's a - # hassle to install it in a multiplatform way -- this should be enabled while working on that issue - # - name: Using a different Python - # run: | - # export TEST_PYTHON_VERSION=3.10 - # ${{ steps.matrixpy.outputs.python-path }} bin/fades -v --python=python3.10 -d pytest -x pytest -v tests/integtest.py + - name: Using a different Python + run: | + export TEST_PYTHON_VERSION=3.10 + ${{ steps.matrixpy.outputs.python-path }} bin/fades -v --python=python3.10 -d pytest -x pytest -v tests/integtest.py diff --git a/README.rst b/README.rst index 3985fc6..4e44b78 100644 --- a/README.rst +++ b/README.rst @@ -380,13 +380,11 @@ pinned, so you will get exactly what you was expecting. Under the hood options ---------------------- -For particular use cases you can send specifics arguments to ``virtualenv``, ``pip`` and ``python``. using the -``--virtuaenv-options``, ``--pip-options`` and ``--python-options`` respectively. You have to use that argument for each argument -sent. +For particular use cases you can send specifics arguments to the ``venv`` module, ``pip`` and ``python`` itself, using the ``--venv-options``, ``--pip-options`` and ``--python-options`` modifiers respectively. You have to use that argument for each argument sent. Examples: -``fades -d requests --virtualenv-options="--always-copy" --virtualenv-options="--extra-search-dir=/tmp"`` +``fades -d requests --venv-options="--symlinks"`` ``fades -d requests --pip-options="--index-url='http://example.com'"`` @@ -420,7 +418,7 @@ options with a dash, it has to be replaced with a underscore.:: dependency=requests;django>=1.8 # separated by semicolon There is a little difference in how fades handle these settings: "dependency", "pip-options" and -"virtualenv-options". In these cases you have to use a semicolon separated list. +"venv-options". In these cases you have to use a semicolon separated list. The most important thing is that these options will be merged. So if you configure in `/etc/fades/fades.ini` "dependency=requests" you will have requests in all the virtual environments @@ -436,7 +434,7 @@ dependencies. There are cases however when you'll want to do some clean up to re unnecessary virtual environments from disk. By running *fades* with the ``--rm`` argument, *fades* will remove the -virtual environment matching the provided UUID if such a environment exists (one easy +virtual environment matching the provided UUID if such environment exists (one easy way to find out the environment's UUID is calling *fades* with the ``--where`` option). @@ -537,23 +535,23 @@ This means that if run fades with a Python version and then run it again with a different Python version, it may need to create a new virtual environment. Let's see some examples. Let's say you run fades with ``python``, which -is a symlink in your ``/usr/bin/`` to ``python3.4`` (running it directly +is a symlink in your ``/usr/bin/`` to ``python3.6`` (running it directly by hand or because fades is installed to use that Python version). -If you have Python 3.4.2 installed in your system, and it's upgraded to -Python 3.4.3, fades will keep reusing the already created virtualenvs, as +If you have Python 3.6.2 installed in your system, and it's upgraded to +Python 3.6.3, fades will keep reusing the already created virtual environments, as only the micro version changed, not minor or major. -But if Python 3.5 is installed in your system, and the default ``python`` +But if Python 3.7 is installed in your system, and the default ``python`` is pointed to this new one, fades will start creating all the -virtualenvs again, with this new version. +virtual environments again, with this new version. This is a good thing, because you want that the dependencies installed with one specific Python in the virtual environment are kept being used by the same Python version. However, if you want to avoid this behaviour, be sure to always call fades -with the specific Python version (``/usr/bin/python3.4`` or ``python3.4``, +with the specific Python version (``/usr/bin/python3.6`` or ``python3.6``, for example), so it won't matter if a new version is available in the system. @@ -597,7 +595,7 @@ Else, keep reading to know how to install the dependencies first, and Dependencies ------------ -Besides needing Python 3.3 or greater, fades depends on the ``python-xdg`` package. This package should be installed on any GNU/Linux OS wiht a freedesktop.org GUI. However it is an **optional** dependency. +Besides needing Python 3.6 or greater, fades depends on the ``python-xdg`` package. This package should be installed on any GNU/Linux OS wiht a freedesktop.org GUI. However it is an **optional** dependency. You can install it in Ubuntu/Debian with:: @@ -607,10 +605,6 @@ And on Arch Linux with:: pacman -S python-xdg -Fades also needs the `virtualenv `_ -package to support different Python versions for child execution. (see the -``--python`` option.) - For others debian and ubuntu ---------------------------- diff --git a/fades/envbuilder.py b/fades/envbuilder.py index 7383073..1df1d3f 100644 --- a/fades/envbuilder.py +++ b/fades/envbuilder.py @@ -14,17 +14,11 @@ # # For further info, check https://github.com/PyAr/fades -"""Extended class from EnvBuilder to create a venv using a uuid4 id. - -NOTE: this class only work in the same python version that Fades is -running. So, you don't need to have installed a virtualenv tool. For -other python versions Fades needs a virtualenv tool installed. -""" +"""Tools to create, destroy and handle usage of virtual environments.""" import logging import os import shutil -import sys from datetime import datetime from venv import EnvBuilder @@ -46,12 +40,9 @@ class _FadesEnvBuilder(EnvBuilder): and ``destroy_env``. """ - def __init__(self, env_path=None): - """Init.""" + def __init__(self): basedir = helpers.get_basedir() - if env_path is None: - env_path = os.path.join(basedir, str(uuid4())) - self.env_path = env_path + self.env_path = os.path.join(basedir, str(uuid4())) self.env_bin_path = '' logger.debug("Env will be created at: %s", self.env_path) @@ -60,9 +51,8 @@ def __init__(self, env_path=None): # because it doesn't work properly (it does a special magic to run the script # and ends up mixing external and internal pips) self.pip_installed = False - super().__init__(with_pip=False, symlinks=True) - elif sys.version_info >= (3, 4): + else: # try to install pip using default machinery (which will work in a lot # of systems, noticeably it won't in some debians or ubuntus, like # Trusty; in that cases mark it to install manually later) @@ -72,46 +62,40 @@ def __init__(self, env_path=None): except ImportError: self.pip_installed = False - super().__init__(with_pip=self.pip_installed, symlinks=True) - - else: - # old Python doesn't have integrated pip - self.pip_installed = False - super().__init__(symlinks=True) + super().__init__(with_pip=self.pip_installed, symlinks=True) - def create_with_virtualenv(self, interpreter, virtualenv_options): - """Create a virtual environment using the virtualenv lib.""" - args = ['virtualenv', '--python', interpreter, self.env_path] - args.extend(virtualenv_options) + def create_with_external_venv(self, interpreter, options): + """Create a virtual environment using the venv module externally.""" + args = [interpreter, "-m", "venv", self.env_path] + args.extend(options) if not self.pip_installed: - args.insert(3, '--no-pip') + args.insert(3, '--without-pip') + try: helpers.logged_exec(args) - self.env_bin_path = os.path.join(self.env_path, 'bin') - except FileNotFoundError as error: - logger.error('Virtualenv is not installed. It is needed to create a virtualenv with ' - 'a different python version than fades (got {})'.format(error)) - raise FadesError('virtualenv not found') except helpers.ExecutionError as error: error.dump_to_log(logger) - raise FadesError('virtualenv could not be run') + raise FadesError("Failed to run venv module externally") except Exception as error: - logger.exception("Error creating virtualenv: %s", error) - raise FadesError('General error while running virtualenv') + logger.exception("Error creating virtual environment: %s", error) + raise FadesError("General error while running external venv") + + self.env_bin_path = os.path.join(self.env_path, 'bin') def create_env(self, interpreter, is_current, options): """Create the virtual environment and return its info.""" + venv_options = options['venv_options'] if is_current: - # apply pyvenv options - pyvenv_options = options['pyvenv_options'] - if "--system-site-packages" in pyvenv_options: - self.system_site_packages = True - logger.debug("Creating virtual environment with pyvenv. options=%s", pyvenv_options) + # apply venv options + logger.debug("Creating virtual environment internally; options=%s", venv_options) + for option in venv_options: + attrname = option[2:].replace("-", "_") # '--system-packgs' -> 'system_packgs' + setattr(self, attrname, True) self.create(self.env_path) else: - virtualenv_options = options['virtualenv_options'] - logger.debug("Creating virtual environment with virtualenv") - self.create_with_virtualenv(interpreter, virtualenv_options) + logger.debug( + "Creating virtual environment with external venv; options=%s", venv_options) + self.create_with_external_venv(interpreter, venv_options) logger.debug("env_bin_path: %s", self.env_bin_path) # Re check if pip was installed (supporting both binary and .exe for Windows) diff --git a/fades/file_options.py b/fades/file_options.py index 83c2688..2f5ea65 100644 --- a/fades/file_options.py +++ b/fades/file_options.py @@ -1,4 +1,4 @@ -# Copyright 2016 Facundo Batista, Nicolás Demarchi +# Copyright 2016-2024 Facundo Batista, Nicolás Demarchi # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License version 3, as published @@ -27,7 +27,7 @@ CONFIG_FILES = ("/etc/fades/fades.ini", os.path.join(get_confdir(), 'fades.ini'), ".fades.ini") -MERGEABLE_CONFIGS = ("dependency", "pip_options", "virtualenv-options") +MERGEABLE_CONFIGS = ("dependency", "pip_options", "venv-options") def options_from_file(args): diff --git a/fades/main.py b/fades/main.py index d80a1f6..55aaf2f 100644 --- a/fades/main.py +++ b/fades/main.py @@ -1,4 +1,4 @@ -# Copyright 2014-2020 Facundo Batista, Nicolás Demarchi +# Copyright 2014-2024 Facundo Batista, Nicolás Demarchi # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General @@ -245,8 +245,9 @@ def go(): '--system-site-packages', action='store_true', default=False, help="give the virtual environment access to the system site-packages dir.") parser.add_argument( - '--virtualenv-options', action='append', default=[], - help="extra options to be supplied to virtualenv (this option can be used multiple times)") + '--venv-options', action='append', default=[], + help="extra options to be supplied to the venv module " + "(this option can be used multiple times)") parser.add_argument( '-U', '--check-updates', action='store_true', help="check for packages updates") parser.add_argument( @@ -395,11 +396,9 @@ def go(): pip_options = args.pip_options # pip_options mustn't store. python_options = args.python_options options = {} - options['pyvenv_options'] = [] - options['virtualenv_options'] = args.virtualenv_options + options['venv_options'] = args.venv_options if args.system_site_packages: - options['virtualenv_options'].append("--system-site-packages") - options['pyvenv_options'] = ["--system-site-packages"] + options['venv_options'].append("--system-site-packages") create_venv = False venv_data = venvscache.get_venv(indicated_deps, interpreter, uuid, options) diff --git a/man/fades.1 b/man/fades.1 index 867c495..410b6d3 100644 --- a/man/fades.1 +++ b/man/fades.1 @@ -16,7 +16,7 @@ fades - A system that automatically handles the virtualenvs in the cases normall [\fB-p\fR \fIversion\fR][\fB--python\fR=\fIversion\fR] [\fB--rm\fR=\fIUUID\fR] [\fB--system-site-packages\fR] -[\fB--virtualenv-options\fR=\fIoptions\fR] +[\fB--venv-options\fR=\fIoptions\fR] [\fB--pip-options\fR=\fIoptions\fR] [\fB--python-options\fR=\fIoptions\fR] [\fB-U\fR][\fB--check-updates\fR] @@ -99,8 +99,8 @@ Remove a virtual environment by UUID. See \fB--get-venv-dir\fR option to easily Give the virtual environment access to thesystem site-packages dir .TP -.BR --virtualenv-options=\fIVIRTUALENV_OPTION\fR -Extra options to be supplied to virtualenv. (this option can be used multiple times) +.BR --venv-options=\fIVIRTUALENV_OPTION\fR +Extra options to be supplied to the venv module (this option can be used multiple times) .TP .BR --pip-options=\fIPIP_OPTION\fR diff --git a/pkg/debian/control b/pkg/debian/control index deee0fd..56eb0ed 100644 --- a/pkg/debian/control +++ b/pkg/debian/control @@ -5,14 +5,14 @@ Build-Depends: debhelper (>= 9), dh-python, dh-translations | dh-python, python3-packaging, - python3-all (>= 3.4), + python3-all (>= 3.6), python3-xdg Maintainer: Facundo Batista Uploaders: Debian Python Modules Team Homepage: https://github.com/PyAr/fades Standards-Version: 3.9.7 -X-Python3-Version: >= 3.4 +X-Python3-Version: >= 3.6 Package: fades Architecture: all diff --git a/pkg/debian/copyright b/pkg/debian/copyright index 5985318..de4b916 100644 --- a/pkg/debian/copyright +++ b/pkg/debian/copyright @@ -4,7 +4,7 @@ Upstream-Contact: Facundo Batista Source: https://github.com/PyAr/fades/ Files: * -Copyright: (C) 2014-2020 +Copyright: (C) 2014-2024 Facundo Batista Nicolás Demarchi License: GPL-3 diff --git a/setup.py b/setup.py index bd85897..ba1125e 100755 --- a/setup.py +++ b/setup.py @@ -121,10 +121,9 @@ def finalize_options(self): install_requires=['setuptools'], tests_require=['logassert', 'pyxdg', 'pyuca', 'pytest', 'flake8', 'pep257', 'rst2html5'], # what unittests require - python_requires='>=3.3', # Minimum Python version supported. + python_requires='>=3.6', # Minimum Python version supported. extras_require={ 'pyxdg': 'pyxdg', - 'virtualenv': 'virtualenv', 'packaging': 'packaging', }, @@ -156,6 +155,8 @@ def finalize_options(self): 'Programming Language :: Python', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', diff --git a/tests/test_envbuilder.py b/tests/test_envbuilder.py index f76d848..0aa64a6 100644 --- a/tests/test_envbuilder.py +++ b/tests/test_envbuilder.py @@ -66,9 +66,7 @@ def test_create_simple(self): interpreter = 'python3' is_current = True avoid_pip_upgrade = False - options = {"virtualenv_options": [], - "pyvenv_options": [], - } + options = {"venv_options": []} pip_options = [] with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create: with patch.object(envbuilder, 'PipManager') as mock_mgr_c: @@ -101,7 +99,7 @@ def test_create_vcs(self): interpreter = 'python3' is_current = True avoid_pip_upgrade = False - options = {"virtualenv_options": [], "pyvenv_options": []} + options = {"venv_options": []} pip_options = [] with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create: with patch.object(envbuilder, 'PipManager') as mock_mgr_c: @@ -124,9 +122,7 @@ def test_unknown_repo(self): interpreter = 'python3' is_current = True avoid_pip_upgrade = False - options = {"virtualenv_options": [], - "pyvenv_options": [], - } + options = {"venv_options": []} pip_options = [] with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create: with patch.object(envbuilder, 'PipManager') as mock_mgr_c: @@ -144,9 +140,7 @@ def test_non_existing_dep(self): interpreter = 'python3' is_current = True avoid_pip_upgrade = False - options = {'virtualenv_options': [], - 'pyvenv_options': [], - } + options = {'venv_options': []} pip_options = [] with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create: @@ -170,9 +164,7 @@ def test_different_versions(self): interpreter = 'python3' is_current = True avoid_pip_upgrade = False - options = {"virtualenv_options": [], - "pyvenv_options": [], - } + options = {"venv_options": []} pip_options = [] with patch.object(envbuilder._FadesEnvBuilder, 'create_env') as mock_create: with patch.object(envbuilder, 'PipManager') as mock_mgr_c: @@ -189,13 +181,11 @@ def test_different_versions(self): } }) - def test_create_system_site_pkgs_pyvenv(self): + def test_create_system_site_pkgs_venv(self): env_builder = envbuilder._FadesEnvBuilder() interpreter = 'python3' is_current = True - options = {"virtualenv_options": [], - "pyvenv_options": ['--system-site-packages'], - } + options = {"venv_options": ['--system-site-packages']} with patch.object(EnvBuilder, 'create') as mock_create: env_builder.create_env(interpreter, is_current, options) self.assertTrue(env_builder.system_site_packages) @@ -205,39 +195,20 @@ def test_create_pyvenv(self): env_builder = envbuilder._FadesEnvBuilder() interpreter = 'python3' is_current = True - options = {"virtualenv_options": [], - "pyvenv_options": [], - } + options = {"venv_options": []} with patch.object(EnvBuilder, 'create') as mock_create: env_builder.create_env(interpreter, is_current, options) self.assertFalse(env_builder.system_site_packages) self.assertTrue(mock_create.called) - def test_create_system_site_pkgs_virtualenv(self): + def test_create_virtual_environment(self): env_builder = envbuilder._FadesEnvBuilder() interpreter = 'pythonX.Y' is_current = False - options = {"virtualenv_options": ['--system-site-packages'], - "pyvenv_options": [], - } - with patch.object(envbuilder._FadesEnvBuilder, 'create_with_virtualenv') as mock_create: + options = {"venv_options": []} + with patch.object(envbuilder._FadesEnvBuilder, 'create_with_external_venv') as mock_create: env_builder.create_env(interpreter, is_current, options) - mock_create.assert_called_with(interpreter, options['virtualenv_options']) - - def test_create_virtualenv(self): - env_builder = envbuilder._FadesEnvBuilder() - interpreter = 'pythonX.Y' - is_current = False - options = {"virtualenv_options": [], - "pyvenv_options": [], - } - with patch.object(envbuilder._FadesEnvBuilder, 'create_with_virtualenv') as mock_create: - env_builder.create_env(interpreter, is_current, options) - mock_create.assert_called_with(interpreter, options['virtualenv_options']) - - def test_custom_env_path(self): - builder = envbuilder._FadesEnvBuilder('some-path') - self.assertEqual(builder.env_path, 'some-path') + mock_create.assert_called_with(interpreter, options['venv_options']) class EnvDestructionTestCase(unittest.TestCase): @@ -245,10 +216,7 @@ class EnvDestructionTestCase(unittest.TestCase): def test_destroy_venv(self): builder = envbuilder._FadesEnvBuilder() # make sure the virtualenv exists on disk - options = {"virtualenv_options": [], - "pyvenv_options": ['--system-site-packages'], - "pip-options": [], - } + options = {"venv_options": [], "pip-options": []} def fake_create(*_): """Fake venv create. @@ -260,7 +228,7 @@ def fake_create(*_): builder.env_path = fake_venv_path fake_venv_path = tempfile.TemporaryDirectory().name - builder.create_with_virtualenv = fake_create + builder.create_with_external_venv = fake_create builder.create_env('python', False, options=options) assert os.path.exists(builder.env_path) @@ -360,45 +328,27 @@ def test_usage_file_is_compacted_when_though_no_venv_is_removed(self): else: self.assertEqual(old_date, d, msg="Others envs have old date") - def test_filenotfound_exception(self): - env_builder = envbuilder._FadesEnvBuilder() - interpreter = 'python3' - is_current = False - options = {"virtualenv_options": [], - "pyvenv_options": ['--system-site-packages'], - } - with patch('fades.envbuilder.helpers.logged_exec') as mock_lexec: - # mock_lexec.side_effect = envbuilder.helpers.ExecutionError('matanga!') - mock_lexec.side_effect = FileNotFoundError('matanga!') - with self.assertRaises(FadesError) as cm: - env_builder.create_env(interpreter, is_current, options) - self.assertEqual(str(cm.exception), 'virtualenv not found') - def test_executionerror_exception(self): env_builder = envbuilder._FadesEnvBuilder() interpreter = 'python3' is_current = False - options = {"virtualenv_options": [], - "pyvenv_options": ['--system-site-packages'], - } + options = {"venv_options": []} with patch('fades.envbuilder.helpers.logged_exec') as mock_lexec: mock_lexec.side_effect = envbuilder.helpers.ExecutionError(1, 'cmd', ['stdout']) with self.assertRaises(FadesError) as cm: env_builder.create_env(interpreter, is_current, options) - self.assertEqual(str(cm.exception), 'virtualenv could not be run') + self.assertEqual(str(cm.exception), "Failed to run venv module externally") def test_general_error_exception(self): env_builder = envbuilder._FadesEnvBuilder() interpreter = 'python3' is_current = False - options = {"virtualenv_options": [], - "pyvenv_options": ['--system-site-packages'], - } + options = {"venv_options": []} with patch('fades.envbuilder.helpers.logged_exec') as mock_lexec: mock_lexec.side_effect = Exception() with self.assertRaises(FadesError) as cm: env_builder.create_env(interpreter, is_current, options) - self.assertEqual(str(cm.exception), 'General error while running virtualenv') + self.assertEqual(str(cm.exception), "General error while running external venv") def test_when_a_venv_is_removed_it_is_removed_from_everywhere(self): old_date = datetime.utcnow()