From 9fcabaf8a0f582c57f9c4447cdece88a1f7a3bce Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Wed, 20 Dec 2023 03:19:24 +0100 Subject: [PATCH] Generate `constructor` artifacts Fixes #657. --- .../conda_store_server/action/__init__.py | 3 + .../action/generate_constructor_artifacts.py | 98 +++++++++++++++++++ conda-store-server/tests/test_actions.py | 40 ++++++++ 3 files changed, 141 insertions(+) create mode 100644 conda-store-server/conda_store_server/action/generate_constructor_artifacts.py diff --git a/conda-store-server/conda_store_server/action/__init__.py b/conda-store-server/conda_store_server/action/__init__.py index a4e04c754..36eb571e9 100644 --- a/conda-store-server/conda_store_server/action/__init__.py +++ b/conda-store-server/conda_store_server/action/__init__.py @@ -32,3 +32,6 @@ from conda_store_server.action.add_lockfile_packages import ( action_add_lockfile_packages, # noqa ) +from conda_store_server.action.generate_constructor_artifacts import ( + action_generate_constructor_artifacts, # noqa +) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py b/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py new file mode 100644 index 000000000..4d6dbacad --- /dev/null +++ b/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py @@ -0,0 +1,98 @@ +import pathlib +import subprocess +import sys +import tempfile + +import yaml +from conda_store_server import action, schema + + +@action.action +def action_generate_constructor_artifacts( + context, + conda_command: str, + specification: schema.CondaSpecification, + installer_dir: str, +): + # Helpers + def print_cmd(cmd): + context.log.info(f"Running command: {' '.join(cmd)}") + context.log.info( + subprocess.check_output(cmd, stderr=subprocess.STDOUT, encoding="utf-8") + ) + + def write_file(filename, s): + with open(filename, "w") as f: + context.log.info(f"{filename}:\n{s}") + f.write(s) + + # pip dependencies are not directly supported by constructor, they will be + # installed via the post_install script: + # https://github.com/conda/constructor/issues/515 + dependencies = [] + pip_dependencies = [] + for d in specification.dependencies: + if type(d) is schema.CondaSpecificationPip: + pip_dependencies.extend(d.pip) + else: + dependencies.append(d) + + # Creates the construct.yaml file and post_install script + ext = ".exe" if sys.platform == "win32" else ".sh" + installer_dir = pathlib.Path(installer_dir) + installer_filename = (installer_dir / specification.name).with_suffix(ext) + + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_dir = pathlib.Path(tmp_dir) + construct_file = tmp_dir / "construct.yaml" + post_install_file = tmp_dir / "post-install.sh" + env_dir = tmp_dir / "env" + + construct = { + "installer_filename": str(installer_filename), + "post_install": str(post_install_file), + "name": specification.name, + "channels": specification.channels, + "specs": dependencies, + # XXX: This is required: use the env hash and datetime? + "version": 1, + } + + # XXX: Support Windows + post_install = """\ +#!/usr/bin/env bash +set -euxo pipefail +""" + if pip_dependencies: + post_install += f""" +conda run -p "$PREFIX" pip install {' '.join(pip_dependencies)} +""" + + # Writes files to disk + write_file(construct_file, yaml.dump(construct)) + write_file(post_install_file, post_install) + + # Installs constructor + command = [ + conda_command, + "create", + "-y", + "-p", + str(env_dir), + "constructor", + ] + print_cmd(command) + + # Calls constructor + command = [ + conda_command, + "run", + "-p", + str(env_dir), + "--no-capture-output", + "constructor", + str(tmp_dir), + ] + print_cmd(command) + + return installer_filename diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index 8d79e3ea4..6d0a63ec6 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -1,8 +1,11 @@ import asyncio import datetime +import os import pathlib import re +import subprocess import sys +import tempfile import pytest import yarl @@ -110,6 +113,43 @@ def test_solve_lockfile_multiple_platforms(conda_store, specification, request): assert len(context.result["package"]) != 0 +@pytest.mark.parametrize( + "specification_name", + [ + "simple_specification", + "simple_specification_with_pip", + ], +) +def test_generate_constructor_artifacts(conda_store, specification_name, request): + specification = request.getfixturevalue(specification_name) + with tempfile.TemporaryDirectory() as installer_dir: + # Creates the installer + context = action.action_generate_constructor_artifacts( + conda_command=conda_store.conda_command, + specification=specification, + installer_dir=installer_dir, + ) + + # Checks that the installer was created + installer = context.result + assert installer.exists() + + with tempfile.TemporaryDirectory() as tmp_dir: + # Runs the installer + out_dir = pathlib.Path(tmp_dir) / 'out' + subprocess.check_output([installer, '-b', '-p', str(out_dir)]) + + # Checks the output directory + assert out_dir.exists() + lib_dir = out_dir / 'lib' + if specification_name == 'simple_specification': + assert any(str(x).endswith('libz.so') for x in lib_dir.iterdir()) + else: + # Uses rglob to not depend on the version of the python + # directory, which is where site-packages is located + assert any(str(x).endswith('site-packages/flask') for x in lib_dir.rglob('*')) + + def test_fetch_and_extract_conda_packages(tmp_path, simple_conda_lock): context = action.action_fetch_and_extract_conda_packages( conda_lock_spec=simple_conda_lock,