From 41ad99a1d42c37e048d8cba1cb72a1bef8abb24d Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 12 Sep 2024 12:14:16 +0200 Subject: [PATCH 01/54] Squashed commit of the following: commit c14bec5c6c559671b2730effef55b692779aae46 Author: czoido Date: Thu Sep 12 12:11:00 2024 +0200 wip commit 6ccc91362cb079a08250ab610fb5f2d7c436d126 Author: czoido Date: Thu Sep 12 12:08:33 2024 +0200 wip commit 384879d80d8e693d8f1eda5a0cb0ce1a042269ea Author: czoido Date: Thu Sep 12 11:14:52 2024 +0200 wip commit 5be6b5e507e1c5df52537590843b75f47c2060ef Author: czoido Date: Thu Sep 12 11:08:07 2024 +0200 wip commit 1c34d892c1640a1be4d54fca6032a69570d6e87c Author: czoido Date: Thu Sep 12 10:39:14 2024 +0200 wip commit 8111fbd2ce264c7bbad157e34e72824127d9f36f Author: czoido Date: Thu Sep 12 10:29:16 2024 +0200 wip commit 8897500654234db8750004e3e83fc8fcfa281d6e Author: czoido Date: Thu Sep 12 10:06:17 2024 +0200 wip commit 1cdd522372b5ce9ab246101cbc3bcd2292149cba Author: czoido Date: Thu Sep 12 10:02:23 2024 +0200 wip commit 2243173a0c69a6f500aba4fcc2303d1a91228704 Author: czoido Date: Thu Sep 12 09:53:27 2024 +0200 wip commit a621eb9f7b003ce44a4abc2adf88a7962e212d5e Author: czoido Date: Thu Sep 12 09:52:32 2024 +0200 wip commit 11be27a0b1ded802d843dd52eaf05ed6bdae150d Author: czoido Date: Thu Sep 12 09:50:50 2024 +0200 wip commit c75ad927a15dfb1501db142a5fafed8fbb2b164a Author: czoido Date: Thu Sep 12 09:40:26 2024 +0200 wip commit 2507615664aafa4161772a07832d1ee24cb76731 Author: czoido Date: Thu Sep 12 09:33:42 2024 +0200 wip commit 2b196891e0786d74dc0d3b367f8fd89ef38cb1d3 Author: czoido Date: Thu Sep 12 09:28:18 2024 +0200 wip commit b5830854f9c316b71624d02a2d30a56bdd70520f Author: czoido Date: Thu Sep 12 09:25:33 2024 +0200 wip commit de5868bbc4797fa4957b52274e72c4bfcaba5128 Author: czoido Date: Thu Sep 12 09:18:43 2024 +0200 wip commit 9cf49c39d8442a739cb5d52f4e9d9f41a35cd342 Author: czoido Date: Thu Sep 12 09:16:49 2024 +0200 wip commit ef33f1de360527d439dba017aa8b6599106f0ca9 Author: czoido Date: Thu Sep 12 09:15:12 2024 +0200 wip commit 7ce43404c7a5b706c8a470826d8a0051f3e1f252 Author: czoido Date: Thu Sep 12 09:13:20 2024 +0200 use runner user commit b2ebc7e5eecba7d1828d3e5242822abc2eb75fc4 Author: czoido Date: Mon Sep 9 16:58:36 2024 +0200 wip commit 993111164e79c8e1597be4ac61830b09030033e5 Author: czoido Date: Mon Sep 9 16:57:18 2024 +0200 wip commit 76508e6816e928fae89295e8e03b76414a305793 Author: czoido Date: Mon Sep 9 16:55:02 2024 +0200 wip commit 89599587b3b3c0944b426f0f5a27d3c605e9af0e Author: czoido Date: Mon Sep 9 16:49:35 2024 +0200 wip commit a4e7ab898175ee5e6fbf340d777c00806d9ad400 Author: czoido Date: Mon Sep 9 16:32:38 2024 +0200 wip commit 1289a25295c91c692595bb2ba8b12a4143a6bf82 Author: czoido Date: Mon Sep 9 16:23:39 2024 +0200 wip commit 7aa09c8800def6794ed7f915144e00802c1a5b9e Author: czoido Date: Mon Sep 9 16:05:39 2024 +0200 wip commit 87b4f7a25dc65bd9d14f0ea7bc969b71cc5abc12 Author: czoido Date: Mon Sep 9 16:04:16 2024 +0200 wip commit f33915a23188c91c04c452aff07697c08e31c4eb Author: czoido Date: Mon Sep 9 16:02:21 2024 +0200 wip commit ed55445ea3e47aeac1ec3b5c9f6ebdcd32f8bdce Author: czoido Date: Mon Sep 9 15:58:08 2024 +0200 wip commit 3ff073f013fa48e3410c5c49a18f728cecebbb52 Author: czoido Date: Mon Sep 9 12:41:49 2024 +0200 wip commit 525db4c0b36289b283d1aa6cc06c9b65a9d165a8 Author: czoido Date: Mon Sep 9 12:41:10 2024 +0200 wip commit ad9024e827ad0b22cff839d3d9164cd19cfae786 Author: czoido Date: Mon Sep 9 12:38:01 2024 +0200 wip commit 81667cff9f4ecf0f25a335dec1839b6b6b3a32b6 Author: czoido Date: Mon Sep 9 12:35:34 2024 +0200 wip commit 29056141033f2d9b31596f2edb24afb33a6c547a Author: czoido Date: Mon Sep 9 12:32:50 2024 +0200 wip commit 560e8ab3503efa41d94172b2da4c431be9f34c57 Author: czoido Date: Mon Sep 9 12:32:25 2024 +0200 wip commit cfba8218c0ac397dd6c5d71cf1c5ee416dfedb0c Author: czoido Date: Mon Sep 9 12:29:42 2024 +0200 wip commit 8ebd462e8c48a6b139f24e194d77c51edf2a7afe Author: czoido Date: Mon Sep 9 12:26:53 2024 +0200 just linux commit 46ae7b22fa4185091834765a902c4d6a21a9a72b Author: czoido Date: Mon Sep 9 12:26:23 2024 +0200 permissions changes commit 76c7497e63907d593ec9b37d32964fe3eb84d917 Author: czoido Date: Thu Sep 5 15:37:28 2024 +0200 wip commit 68c05e073696a0c6625b0726b53303ff08f83d50 Author: czoido Date: Thu Sep 5 15:36:13 2024 +0200 wip commit ce6c82924220cac8b20e2773945c5ea7c63749d5 Author: czoido Date: Thu Sep 5 15:24:19 2024 +0200 wip --- .ci/__init__.py | 0 .ci/bump_dev_version.py | 22 ---- .ci/jenkins/testsv2.jenkins | 184 ------------------------------ .github/workflows/ci-pipeline.yml | 173 ++++++++++++++++++++++++++++ ci/Dockerfile | 51 +++++++++ 5 files changed, 224 insertions(+), 206 deletions(-) delete mode 100644 .ci/__init__.py delete mode 100644 .ci/bump_dev_version.py delete mode 100644 .ci/jenkins/testsv2.jenkins create mode 100644 .github/workflows/ci-pipeline.yml create mode 100644 ci/Dockerfile diff --git a/.ci/__init__.py b/.ci/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/.ci/bump_dev_version.py b/.ci/bump_dev_version.py deleted file mode 100644 index e3f802833ea..00000000000 --- a/.ci/bump_dev_version.py +++ /dev/null @@ -1,22 +0,0 @@ -import os -import time - - -def replace_in_file(file_path, search, replace): - with open(file_path, "r") as handle: - content = handle.read() - if search not in content: - raise Exception("Incorrect development version in conans/__init__.py") - content = content.replace(search, replace) - content = content.encode("utf-8") - with open(file_path, "wb") as handle: - handle.write(content) - -def bump_dev(): - vfile = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../conans/__init__.py") - snapshot = "%s" % int(time.time()) - replace_in_file(vfile, "-dev'", "-dev%s'" % snapshot) - - -if __name__ == "__main__": - bump_dev() diff --git a/.ci/jenkins/testsv2.jenkins b/.ci/jenkins/testsv2.jenkins deleted file mode 100644 index 01bc871790c..00000000000 --- a/.ci/jenkins/testsv2.jenkins +++ /dev/null @@ -1,184 +0,0 @@ -def branchName = env.BRANCH_NAME -def jobName = env.JOB_NAME - -void cancelPrevious() { - stage("Cancelling previous") { - def buildNumber = env.BUILD_NUMBER as int - if (buildNumber > 1) milestone(buildNumber - 1) - milestone(buildNumber) - } -} - -String getTempFolder(String nodeName) { - if (nodeName == "Windows") { return "C:/J/t_v2/" } - return "/tmp/" -} - -List getConfigurations(String moduleName, String branchName, String jobName) { - def configs = [] - - String LinuxNode - if (moduleName == "test/unittests" || moduleName == "test/integration") { - LinuxNode = "LinuxUnittests" - } else if (moduleName == "test/functional") { - LinuxNode = "LinuxFunctional" - } - - configs.add([node: LinuxNode, pyvers: ["PY312"]]) - - if (branchName =~ /(^PR-.*)/) { - configs.add([node: LinuxNode, pyvers: ["PY38"]]) - configs.add([node: "Windows", pyvers: ["PY38"]]) - configs.add([node: "M2Macos", pyvers: ["PY38"]]) - } - else if (jobName == "ConanNightlyv2" || branchName =~ /^release.*/) { - configs.add([node: LinuxNode, pyvers: ["PY36", "PY38"]]) - configs.add([node: "Windows", pyvers: ["PY36", "PY38"]]) - configs.add([node: "M2Macos", pyvers: ["PY36", "PY38"]]) - } - else if (branchName == "develop2") { - configs.add([node: LinuxNode, pyvers: ["PY36", "PY38"]]) - configs.add([node: "Windows", pyvers: ["PY36", "PY38"]]) - configs.add([node: "M2Macos", pyvers: ["PY36", "PY38"]]) - } - return configs -} - -boolean shouldTestServer(String moduleName) { - // run conan_server --help after running functional tests - if (moduleName == "test/functional") { - return true - } - return false -} - -private Closure runTests(String nodeName, String pythonVersion, String module) { - def ret = { - node(nodeName) { - stage("${nodeName} - ${pythonVersion}") { - def scmVars = checkout scm - def commit = scmVars["GIT_COMMIT"].substring(0, 4) - def workDir = getTempFolder(nodeName) + "${commit}/${pythonVersion}/" - def venvName = "env_${env.BUILD_NUMBER}_${pythonVersion}_${commit}" - def pipArguments = "--no-cache-dir --timeout 30" - def pipInstalls = "python -m pip install -r conans/requirements.txt ${pipArguments} && " + - "python -m pip install -r conans/requirements_server.txt ${pipArguments} && " + - "python -m pip install -r conans/requirements_dev.txt ${pipArguments} && " + - "python -m pip install meson ${pipArguments} && " - def conanToolInstall = "python -m pip install . && conan --version && conan --help && python --version && " - def launchTests = "python -m pytest ${module} -n=4 --durations=20 -rs" - if (shouldTestServer(module)) { - def rm = (nodeName == "Windows") ? "del" : "rm" - def mv = (nodeName == "Windows") ? "move" : "mv" - def conanServerToolInstall = "${rm} setup.py && ${mv} setup_server.py setup.py && python -m pip install . && conan_server --help" - launchTests = launchTests + ' && ' + conanServerToolInstall - } - if (nodeName.indexOf("Linux") >= 0) { - try { - // we only test scons in Linux - pipInstalls = pipInstalls + "python -m pip install scons ${pipArguments} && " - def sourcedir = "/home/conan/" - def pyenvdir = "${sourcedir}.pyenv" - def launchCommand = "export PYENV_ROOT=${pyenvdir} && " + - "export PATH=\"${pyenvdir}/versions/\$${pythonVersion}/bin:${pyenvdir}/bin:\$PATH\" && " + - "pyenv global \$${pythonVersion} && " + - pipInstalls + - conanToolInstall + - launchTests - sh(script: launchCommand) - } finally {} - } else if (nodeName == "M2Macos") { - def localDir = "/Users/jenkins" - withEnv(["CONAN_TEST_FOLDER=${workDir}", "PY27=2.7.18", "PY36=3.6.15", "PY38=3.8.12", "PY39=3.9.10", - "PYENV_ROOT=${localDir}/.pyenv", "PATH+EXTRA=${localDir}/.pyenv/shims:/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"]) { - try { - sh(script: "mkdir -p ${workDir}") - def pythonLocation = "${localDir}/.pyenv/versions/\$${pythonVersion}/bin/python" - def configVenv = "${pythonLocation} -m virtualenv --python ${pythonLocation} ${workDir}${venvName} && " + - "source ${workDir}${venvName}/bin/activate && python --version && " - def launchCommand = configVenv + pipInstalls + conanToolInstall + launchTests - sh(script: launchCommand) - } finally { - sh(script: "rm -rf ${workDir}") - } - } - } else if (nodeName == "Windows") { - withEnv(["CONAN_TEST_FOLDER=${workDir}", "CONAN_BASH_PATH=c:/tools/msys64/usr/bin/bash", - "PY27=Python27", "PY36=Python36", "PY38=Python38-64"]) { - try { - bat(script: "if not exist \"${workDir}\" mkdir \"${workDir}\"") - def pythonLocation = "C:/%${pythonVersion}%/python.exe" - def configVenv = "virtualenv --python ${pythonLocation} ${workDir}${venvName} && " + - "${workDir}${venvName}/Scripts/activate && python --version && " - def launchCommand = configVenv + pipInstalls + conanToolInstall + launchTests - bat(script: launchCommand) - } finally { - bat(script: "rd /s /q \"${workDir}\"") - } - } - } - cleanWs() - } - } - } - return ret -} - -void runTestsModule(String moduleName, String branchName, String jobName) { - def configs = getConfigurations(moduleName, branchName, jobName) - def paralellRuns = [:] - configs.each { config -> - def testKind = moduleName.split("/").last() - config["pyvers"].each { pyver -> - paralellRuns["${testKind} - ${config['node']} - ${pyver}"] = runTests(config["node"], pyver, moduleName) - } - } - parallel(paralellRuns) -} - -def testModules = ["test/unittests", "test/integration", "test/functional"] - -void deployToPypiTest() { - node("LinuxFunctional") { - try { - def scmVars = checkout scm - checkout scm - stage("Deploy conan to pypitesting") { - withCredentials([usernamePassword(credentialsId: 'PYPITEST_CONAN_CREDENTIALS', usernameVariable: 'TWINE_USERNAME', passwordVariable: 'TWINE_PASSWORD')]) { - def deployConan = "python -m pip install twine && " + - "python .ci/bump_dev_version.py && " + - "rm -rf dist/ && python setup.py sdist && " + - "python -m twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/*" - sh(script: deployConan) - } - } - stage("Deploy conan-server to pypitesting") { - withCredentials([usernamePassword(credentialsId: 'PYPITEST_CONAN_SERVER_CREDENTIALS', usernameVariable: 'TWINE_USERNAME', passwordVariable: 'TWINE_PASSWORD')]) { - def deployServer = "rm -rf dist/ rm setup.py && mv setup_server.py setup.py && python setup.py sdist && " + - "python -m twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/*" - sh(script: deployServer) - } - } - } finally {} - } -} - -try { - cancelPrevious() - - testModules.each { moduleName -> - runTestsModule(moduleName, branchName, jobName) - } - - if (branchName == "develop2") { - deployToPypiTest() - } -} -catch(e){ - if (branchName == "develop2") { - def subject = "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'" - def summary = "${subject} (${env.BUILD_URL}), Conan Branch: ${branchName}" - slackSend (color: '#FF0000', message: summary) - } - throw e -} diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml new file mode 100644 index 00000000000..ab0bdb10ce9 --- /dev/null +++ b/.github/workflows/ci-pipeline.yml @@ -0,0 +1,173 @@ +name: CI Pipeline + +on: + push: + branches: + - '*' + pull_request: + branches: + - '*' + +jobs: + build_container: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Log in to GitHub Container Registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Calculate Dockerfile checksum + id: dockerfile_hash + run: | + DOCKERFILE_HASH=$(find ./ci/Dockerfile -type f -exec sha256sum {} \; | sha256sum | cut -d' ' -f1) + echo "Dockerfile hash: $DOCKERFILE_HASH" + echo "::set-output name=hash::$DOCKERFILE_HASH" + + - name: Cache Docker image + id: cache_docker_image + uses: actions/cache@v3 + with: + path: docker_image_cache + key: ${{ runner.os }}-docker-${{ steps.dockerfile_hash.outputs.hash }} + + - name: Pull Docker image (if exists in cache or registry) + id: pull_image + run: | + docker pull ghcr.io/${{ github.repository_owner }}/my-custom-image:latest || echo "Image not found" + + - name: Build Docker image (if not pulled or cache miss) + if: steps.pull_image.outcome == 'failure' || steps.cache_docker_image.outputs.cache-hit != 'true' + run: | + docker build -t ghcr.io/${{ github.repository_owner }}/my-custom-image:latest -f ./ci/Dockerfile . + mkdir -p docker_image_cache && docker save ghcr.io/${{ github.repository_owner }}/my-custom-image:latest -o docker_image_cache/my-custom-image.tar + + - name: Push Docker image to GHCR + if: steps.pull_image.outcome == 'failure' || steps.cache_docker_image.outputs.cache-hit != 'true' + run: | + docker push ghcr.io/${{ github.repository_owner }}/my-custom-image:latest + + linux_unittests: + needs: build_container + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository_owner }}/my-custom-image:latest + options: --user runner + strategy: + matrix: + python-version: [3.8] + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + run: | + pyenv global 3.8.6 + /home/runner/.pyenv/shims/python -m pip install --upgrade pip + /home/runner/.pyenv/shims/python -m pip install -r conans/requirements.txt + /home/runner/.pyenv/shims/python -m pip install -r conans/requirements_dev.txt + /home/runner/.pyenv/shims/python -m pip install meson + + - name: Run tests + run: | + /home/runner/.pyenv/shims/python -m pytest test/unittests --durations=20 -n 4 + + # linux_functional: + # needs: cancel_previous + # runs-on: ubuntu-latest + # container: + # image: conanio/ci-functional + # strategy: + # matrix: + # python-version: [3.8, 3.9, 3.12] + # steps: + # - name: Checkout code + # uses: actions/checkout@v2 + + # - name: Set up Python ${{ matrix.python-version }} + # run: | + # pip install --upgrade pip + # pip install -r conans/requirements.txt + # pip install -r conans/requirements_dev.txt + # pip install meson + + # - name: Run tests + # run: | + # pytest test/functional --durations=20 -n 4 + + # windows_tests: + # needs: cancel_previous + # runs-on: windows-latest + # strategy: + # matrix: + # python-version: [3.8, 3.9, 3.12] + # steps: + # - name: Checkout code + # uses: actions/checkout@v2 + + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v2 + # with: + # python-version: ${{ matrix.python-version }} + + # - name: Install dependencies + # run: | + # python -m pip install --upgrade pip + # pip install -r conans/requirements.txt + # pip install -r conans/requirements_dev.txt + # pip install meson + + # - name: Run tests + # run: | + # pytest test/unittests --durations=20 -n 4 + + # macos_tests: + # needs: cancel_previous + # runs-on: macos-latest + # strategy: + # matrix: + # python-version: [3.8, 3.9, 3.12] + # steps: + # - name: Checkout code + # uses: actions/checkout@v2 + + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v2 + # with: + # python-version: ${{ matrix.python-version }} + + # - name: Install dependencies + # run: | + # python -m pip install --upgrade pip + # pip install -r conans/requirements.txt + # pip install -r conans/requirements_dev.txt + # pip install meson + + # - name: Run tests + # run: | + # pytest test/unittests --durations=20 -n 4 + + deploy_pypitest: + if: github.ref == 'refs/heads/develop2' + runs-on: ubuntu-latest + container: + image: conanio/ci-functional + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Install dependencies for deployment + run: | + python -m pip install --upgrade pip + python -m pip install twine + python .ci/bump_dev_version.py + + - name: Build and deploy to TestPyPI +# env: +# TWINE_USERNAME: ${{ secrets.PYPITEST_CONAN_CREDENTIALS_USERNAME }} +# TWINE_PASSWORD: ${{ secrets.PYPITEST_CONAN_CREDENTIALS_PASSWORD }} + run: | + rm -rf dist/ + python setup.py sdist +# twine upload --repository-url https://test.pypi.org/legacy/ dist/* diff --git a/ci/Dockerfile b/ci/Dockerfile new file mode 100644 index 00000000000..a9bccc505a2 --- /dev/null +++ b/ci/Dockerfile @@ -0,0 +1,51 @@ +FROM ubuntu:latest + +LABEL maintainer="Conan.io " + +ARG DEBIAN_FRONTEND=noninteractive +ENV PY38 3.8.6 +ENV PYENV_ROOT="/home/runner/.pyenv" +ENV PATH="$PYENV_ROOT/bin:$PATH" + +# Install essential packages and dependencies for pyenv and other tools +RUN apt-get -qq update \ + && apt-get -qq install -y --no-install-recommends \ + sudo ca-certificates \ + make build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ + xz-utils tk-dev libffi-dev liblzma-dev git \ + adduser \ + && rm -rf /var/lib/apt/lists/* + + +# Create the 'runner' user +RUN adduser --disabled-password --gecos "" --uid 1001 runner \ + && usermod -aG sudo runner + +# Set up pyenv environment and install Python version +RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash \ + && export PATH="$PYENV_ROOT/bin:$PATH" \ + && eval "$(pyenv init --path)" \ + && pyenv install $PY38 \ + && pyenv global $PY38 \ + && python -m ensurepip --upgrade \ + && chown -R runner:runner /home/runner/.pyenv + + + + +# Clean up unnecessary packages +RUN apt-get -qq remove make build-essential gcc \ + && rm -rf /var/lib/apt/lists/* + +# Set up the work directory for the runner user +WORKDIR /home/runner + +# Switch to the runner user +USER runner + +# Ensure pyenv is available for the runner user +RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> /home/runner/.bashrc \ + && echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> /home/runner/.bashrc \ + && echo 'eval "$(pyenv init --path)"' >> /home/runner/.bashrc \ + && echo 'eval "$(pyenv init -)"' >> /home/runner/.bashrc From 3a39b6898278a3de8f5dc985afe2f8fddad47c7e Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 12 Sep 2024 16:09:10 +0200 Subject: [PATCH 02/54] wip --- .github/workflows/ci-pipeline.yml | 12 ++--- ci/Dockerfile | 83 ++++++++++++++----------------- 2 files changed, 42 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index ab0bdb10ce9..92826f26ed2 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/${{ github.repository_owner }}/my-custom-image:latest - options: --user runner + options: --user conan strategy: matrix: python-version: [3.8] @@ -64,14 +64,14 @@ jobs: - name: Set up Python ${{ matrix.python-version }} run: | pyenv global 3.8.6 - /home/runner/.pyenv/shims/python -m pip install --upgrade pip - /home/runner/.pyenv/shims/python -m pip install -r conans/requirements.txt - /home/runner/.pyenv/shims/python -m pip install -r conans/requirements_dev.txt - /home/runner/.pyenv/shims/python -m pip install meson + python -m pip install --upgrade pip + python -m pip install -r conans/requirements.txt + python -m pip install -r conans/requirements_dev.txt + python -m pip install meson - name: Run tests run: | - /home/runner/.pyenv/shims/python -m pytest test/unittests --durations=20 -n 4 + python -m pytest test/unittests --durations=20 -n 4 # linux_functional: # needs: cancel_previous diff --git a/ci/Dockerfile b/ci/Dockerfile index a9bccc505a2..77260431927 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -2,50 +2,39 @@ FROM ubuntu:latest LABEL maintainer="Conan.io " -ARG DEBIAN_FRONTEND=noninteractive -ENV PY38 3.8.6 -ENV PYENV_ROOT="/home/runner/.pyenv" -ENV PATH="$PYENV_ROOT/bin:$PATH" - -# Install essential packages and dependencies for pyenv and other tools -RUN apt-get -qq update \ - && apt-get -qq install -y --no-install-recommends \ - sudo ca-certificates \ - make build-essential libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev git \ - adduser \ - && rm -rf /var/lib/apt/lists/* - - -# Create the 'runner' user -RUN adduser --disabled-password --gecos "" --uid 1001 runner \ - && usermod -aG sudo runner - -# Set up pyenv environment and install Python version -RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash \ - && export PATH="$PYENV_ROOT/bin:$PATH" \ - && eval "$(pyenv init --path)" \ - && pyenv install $PY38 \ - && pyenv global $PY38 \ - && python -m ensurepip --upgrade \ - && chown -R runner:runner /home/runner/.pyenv - - - - -# Clean up unnecessary packages -RUN apt-get -qq remove make build-essential gcc \ - && rm -rf /var/lib/apt/lists/* - -# Set up the work directory for the runner user -WORKDIR /home/runner - -# Switch to the runner user -USER runner - -# Ensure pyenv is available for the runner user -RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> /home/runner/.bashrc \ - && echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> /home/runner/.bashrc \ - && echo 'eval "$(pyenv init --path)"' >> /home/runner/.bashrc \ - && echo 'eval "$(pyenv init -)"' >> /home/runner/.bashrc +RUN apt-get update && apt-get install -y \ + git \ + curl \ + make \ + build-essential \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + wget \ + llvm \ + libncurses5-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libffi-dev \ + liblzma-dev \ + python3-openssl \ + ca-certificates \ + sudo + +RUN useradd -m -s /bin/bash conan && \ + echo 'conan ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER conan +WORKDIR /home/conan +ENV PATH /home/conan/.pyenv/shims:/home/conan/.pyenv/bin:$PATH + +RUN curl https://pyenv.run | bash \ + && pyenv init - \ + && pyenv virtualenv-init - \ + && pyenv update \ + && pyenv install 3.8.6 + +CMD ["/bin/bash"] From 212901de70a8b5e97a2d2b995449cafb0595b396 Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 12 Sep 2024 16:38:17 +0200 Subject: [PATCH 03/54] wip --- .github/workflows/ci-pipeline.yml | 6 ++++-- ci/Dockerfile | 13 ++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 92826f26ed2..06e971137ae 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -56,14 +56,16 @@ jobs: options: --user conan strategy: matrix: - python-version: [3.8] + python-version: [3.8.6] steps: - name: Checkout code uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} run: | - pyenv global 3.8.6 + eval "$(pyenv init --path)" + eval "$(pyenv init -)" + pyenv global ${{ matrix.python-version }} python -m pip install --upgrade pip python -m pip install -r conans/requirements.txt python -m pip install -r conans/requirements_dev.txt diff --git a/ci/Dockerfile b/ci/Dockerfile index 77260431927..8761ad17d64 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -31,10 +31,13 @@ USER conan WORKDIR /home/conan ENV PATH /home/conan/.pyenv/shims:/home/conan/.pyenv/bin:$PATH -RUN curl https://pyenv.run | bash \ - && pyenv init - \ - && pyenv virtualenv-init - \ - && pyenv update \ - && pyenv install 3.8.6 +RUN curl https://pyenv.run | bash + +RUN echo 'export PATH="/home/conan/.pyenv/bin:$PATH"' >> /home/conan/.bashrc && \ + echo 'eval "$(pyenv init --path)"' >> /home/conan/.bashrc && \ + echo 'eval "$(pyenv init -)"' >> /home/conan/.bashrc && \ + echo 'eval "$(pyenv virtualenv-init -)"' >> /home/conan/.bashrc + +RUN /bin/bash -c "source /home/conan/.bashrc && pyenv install 3.8.6 && pyenv global 3.8.6" CMD ["/bin/bash"] From fa04da7af2a9c764d5c52d8f24c43bfc66c6dc82 Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 13 Sep 2024 07:08:40 +0200 Subject: [PATCH 04/54] wip --- .github/workflows/ci-pipeline.yml | 12 ++++++------ ci/{Dockerfile => conan-unittests} | 0 2 files changed, 6 insertions(+), 6 deletions(-) rename ci/{Dockerfile => conan-unittests} (100%) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 06e971137ae..4f0aad6e10f 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -21,7 +21,7 @@ jobs: - name: Calculate Dockerfile checksum id: dockerfile_hash run: | - DOCKERFILE_HASH=$(find ./ci/Dockerfile -type f -exec sha256sum {} \; | sha256sum | cut -d' ' -f1) + DOCKERFILE_HASH=$(find ./ci/conan-unittests -type f -exec sha256sum {} \; | sha256sum | cut -d' ' -f1) echo "Dockerfile hash: $DOCKERFILE_HASH" echo "::set-output name=hash::$DOCKERFILE_HASH" @@ -35,24 +35,24 @@ jobs: - name: Pull Docker image (if exists in cache or registry) id: pull_image run: | - docker pull ghcr.io/${{ github.repository_owner }}/my-custom-image:latest || echo "Image not found" + docker pull ghcr.io/${{ github.repository_owner }}/conan-unittests:latest || echo "Image not found" - name: Build Docker image (if not pulled or cache miss) if: steps.pull_image.outcome == 'failure' || steps.cache_docker_image.outputs.cache-hit != 'true' run: | - docker build -t ghcr.io/${{ github.repository_owner }}/my-custom-image:latest -f ./ci/Dockerfile . - mkdir -p docker_image_cache && docker save ghcr.io/${{ github.repository_owner }}/my-custom-image:latest -o docker_image_cache/my-custom-image.tar + docker build -t ghcr.io/${{ github.repository_owner }}/conan-unittests:latest -f ./ci/conan-unittests . + mkdir -p docker_image_cache && docker save ghcr.io/${{ github.repository_owner }}/conan-unittests:latest -o docker_image_cache/conan-unittests.tar - name: Push Docker image to GHCR if: steps.pull_image.outcome == 'failure' || steps.cache_docker_image.outputs.cache-hit != 'true' run: | - docker push ghcr.io/${{ github.repository_owner }}/my-custom-image:latest + docker push ghcr.io/${{ github.repository_owner }}/conan-unittests:latest linux_unittests: needs: build_container runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository_owner }}/my-custom-image:latest + image: ghcr.io/${{ github.repository_owner }}/conan-unittests:latest options: --user conan strategy: matrix: diff --git a/ci/Dockerfile b/ci/conan-unittests similarity index 100% rename from ci/Dockerfile rename to ci/conan-unittests From 1d7946085d9201594a54d695de7276a895760bfc Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 13 Sep 2024 07:09:06 +0200 Subject: [PATCH 05/54] wip --- ci/conan-unittests | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ci/conan-unittests b/ci/conan-unittests index 8761ad17d64..2dc98252a81 100644 --- a/ci/conan-unittests +++ b/ci/conan-unittests @@ -1,7 +1,9 @@ +# Dockerfile basado en Ubuntu latest FROM ubuntu:latest LABEL maintainer="Conan.io " +# Actualiza el sistema e instala las dependencias necesarias RUN apt-get update && apt-get install -y \ git \ curl \ @@ -24,6 +26,7 @@ RUN apt-get update && apt-get install -y \ ca-certificates \ sudo +# Crear usuario 'conan' y otorgarle permisos sudo RUN useradd -m -s /bin/bash conan && \ echo 'conan ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers @@ -31,13 +34,18 @@ USER conan WORKDIR /home/conan ENV PATH /home/conan/.pyenv/shims:/home/conan/.pyenv/bin:$PATH +# Instalar pyenv RUN curl https://pyenv.run | bash -RUN echo 'export PATH="/home/conan/.pyenv/bin:$PATH"' >> /home/conan/.bashrc && \ - echo 'eval "$(pyenv init --path)"' >> /home/conan/.bashrc && \ - echo 'eval "$(pyenv init -)"' >> /home/conan/.bashrc && \ - echo 'eval "$(pyenv virtualenv-init -)"' >> /home/conan/.bashrc +# Configuración global de pyenv en /etc/profile +RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> /etc/profile && \ + echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> /etc/profile && \ + echo 'eval "$(pyenv init --path)"' >> /etc/profile && \ + echo 'eval "$(pyenv init -)"' >> /etc/profile && \ + echo 'eval "$(pyenv virtualenv-init -)"' >> /etc/profile -RUN /bin/bash -c "source /home/conan/.bashrc && pyenv install 3.8.6 && pyenv global 3.8.6" +# Instalar la versión de Python 3.8.6 y configurarla globalmente +RUN /bin/bash -c "source /etc/profile && pyenv install 3.8.6 && pyenv global 3.8.6" +# Comando por defecto CMD ["/bin/bash"] From 6a3424128c4096ba592192e92c7d81d277432742 Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 13 Sep 2024 08:06:13 +0200 Subject: [PATCH 06/54] wip --- ci/conan-unittests | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/ci/conan-unittests b/ci/conan-unittests index 2dc98252a81..cf226d3c6ba 100644 --- a/ci/conan-unittests +++ b/ci/conan-unittests @@ -1,9 +1,9 @@ -# Dockerfile basado en Ubuntu latest +# Base image is Ubuntu latest FROM ubuntu:latest LABEL maintainer="Conan.io " -# Actualiza el sistema e instala las dependencias necesarias +# Update system and install necessary dependencies RUN apt-get update && apt-get install -y \ git \ curl \ @@ -26,26 +26,36 @@ RUN apt-get update && apt-get install -y \ ca-certificates \ sudo -# Crear usuario 'conan' y otorgarle permisos sudo +# Create 'conan' user and grant sudo permissions RUN useradd -m -s /bin/bash conan && \ echo 'conan ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +# Switch to root to configure pyenv globally in /etc/profile +USER root + +# Configure pyenv globally via /etc/profile +RUN echo 'export PYENV_ROOT="/home/conan/.pyenv"' >> /etc/profile && \ + echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> /etc/profile && \ + echo 'eval "$(pyenv init --path)"' >> /etc/profile && \ + echo 'eval "$(pyenv init -)"' >> /etc/profile && \ + echo 'eval "$(pyenv virtualenv-init -)"' >> /etc/profile + +# Switch back to the 'conan' user USER conan WORKDIR /home/conan +ENV HOME /home/conan ENV PATH /home/conan/.pyenv/shims:/home/conan/.pyenv/bin:$PATH -# Instalar pyenv +# Install pyenv RUN curl https://pyenv.run | bash -# Configuración global de pyenv en /etc/profile -RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> /etc/profile && \ - echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> /etc/profile && \ - echo 'eval "$(pyenv init --path)"' >> /etc/profile && \ - echo 'eval "$(pyenv init -)"' >> /etc/profile && \ - echo 'eval "$(pyenv virtualenv-init -)"' >> /etc/profile +RUN echo 'export PATH="/home/conan/.pyenv/bin:$PATH"' >> /home/conan/.bashrc && \ + echo 'eval "$(pyenv init --path)"' >> /home/conan/.bashrc && \ + echo 'eval "$(pyenv init -)"' >> /home/conan/.bashrc && \ + echo 'eval "$(pyenv virtualenv-init -)"' >> /home/conan/.bashrc -# Instalar la versión de Python 3.8.6 y configurarla globalmente -RUN /bin/bash -c "source /etc/profile && pyenv install 3.8.6 && pyenv global 3.8.6" +# Install Python 3.8.6 using pyenv and set it as the global version +RUN /bin/bash -l -c "pyenv install 3.8.6 && pyenv global 3.8.6" -# Comando por defecto +# Default command CMD ["/bin/bash"] From cf9633eda8698f34f7c340cdc1d64f30b05c4564 Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 13 Sep 2024 08:30:06 +0200 Subject: [PATCH 07/54] wip --- .github/workflows/ci-pipeline.yml | 139 ++++++------------------------ ci/conan-unittests | 49 ++++++----- 2 files changed, 53 insertions(+), 135 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 4f0aad6e10f..c43549db9bd 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -23,7 +23,7 @@ jobs: run: | DOCKERFILE_HASH=$(find ./ci/conan-unittests -type f -exec sha256sum {} \; | sha256sum | cut -d' ' -f1) echo "Dockerfile hash: $DOCKERFILE_HASH" - echo "::set-output name=hash::$DOCKERFILE_HASH" + echo "hash=$DOCKERFILE_HASH" >> $GITHUB_OUTPUT - name: Cache Docker image id: cache_docker_image @@ -32,19 +32,20 @@ jobs: path: docker_image_cache key: ${{ runner.os }}-docker-${{ steps.dockerfile_hash.outputs.hash }} - - name: Pull Docker image (if exists in cache or registry) - id: pull_image + - name: Load Docker image from cache + if: steps.cache_docker_image.outputs.cache-hit == 'true' run: | - docker pull ghcr.io/${{ github.repository_owner }}/conan-unittests:latest || echo "Image not found" + docker load -i docker_image_cache/conan-unittests.tar - - name: Build Docker image (if not pulled or cache miss) - if: steps.pull_image.outcome == 'failure' || steps.cache_docker_image.outputs.cache-hit != 'true' + - name: Build Docker image (if not cached) + if: steps.cache_docker_image.outputs.cache-hit != 'true' run: | docker build -t ghcr.io/${{ github.repository_owner }}/conan-unittests:latest -f ./ci/conan-unittests . - mkdir -p docker_image_cache && docker save ghcr.io/${{ github.repository_owner }}/conan-unittests:latest -o docker_image_cache/conan-unittests.tar + mkdir -p docker_image_cache + docker save ghcr.io/${{ github.repository_owner }}/conan-unittests:latest -o docker_image_cache/conan-unittests.tar - name: Push Docker image to GHCR - if: steps.pull_image.outcome == 'failure' || steps.cache_docker_image.outputs.cache-hit != 'true' + if: steps.cache_docker_image.outputs.cache-hit != 'true' run: | docker push ghcr.io/${{ github.repository_owner }}/conan-unittests:latest @@ -56,120 +57,32 @@ jobs: options: --user conan strategy: matrix: - python-version: [3.8.6] + python-version: ['3.8.6'] + env: + PYENV_ROOT: /home/conan/.pyenv + PATH: /home/conan/.pyenv/bin:/home/conan/.pyenv/shims:$PATH steps: - name: Checkout code uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} run: | - eval "$(pyenv init --path)" - eval "$(pyenv init -)" pyenv global ${{ matrix.python-version }} - python -m pip install --upgrade pip - python -m pip install -r conans/requirements.txt - python -m pip install -r conans/requirements_dev.txt - python -m pip install meson + python --version - - name: Run tests - run: | - python -m pytest test/unittests --durations=20 -n 4 - - # linux_functional: - # needs: cancel_previous - # runs-on: ubuntu-latest - # container: - # image: conanio/ci-functional - # strategy: - # matrix: - # python-version: [3.8, 3.9, 3.12] - # steps: - # - name: Checkout code - # uses: actions/checkout@v2 - - # - name: Set up Python ${{ matrix.python-version }} - # run: | - # pip install --upgrade pip - # pip install -r conans/requirements.txt - # pip install -r conans/requirements_dev.txt - # pip install meson - - # - name: Run tests - # run: | - # pytest test/functional --durations=20 -n 4 - - # windows_tests: - # needs: cancel_previous - # runs-on: windows-latest - # strategy: - # matrix: - # python-version: [3.8, 3.9, 3.12] - # steps: - # - name: Checkout code - # uses: actions/checkout@v2 - - # - name: Set up Python ${{ matrix.python-version }} - # uses: actions/setup-python@v2 - # with: - # python-version: ${{ matrix.python-version }} - - # - name: Install dependencies - # run: | - # python -m pip install --upgrade pip - # pip install -r conans/requirements.txt - # pip install -r conans/requirements_dev.txt - # pip install meson - - # - name: Run tests - # run: | - # pytest test/unittests --durations=20 -n 4 - - # macos_tests: - # needs: cancel_previous - # runs-on: macos-latest - # strategy: - # matrix: - # python-version: [3.8, 3.9, 3.12] - # steps: - # - name: Checkout code - # uses: actions/checkout@v2 - - # - name: Set up Python ${{ matrix.python-version }} - # uses: actions/setup-python@v2 - # with: - # python-version: ${{ matrix.python-version }} - - # - name: Install dependencies - # run: | - # python -m pip install --upgrade pip - # pip install -r conans/requirements.txt - # pip install -r conans/requirements_dev.txt - # pip install meson - - # - name: Run tests - # run: | - # pytest test/unittests --durations=20 -n 4 - - deploy_pypitest: - if: github.ref == 'refs/heads/develop2' - runs-on: ubuntu-latest - container: - image: conanio/ci-functional - steps: - - name: Checkout code - uses: actions/checkout@v2 + - name: Cache pip + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} - - name: Install dependencies for deployment + - name: Install dependencies run: | - python -m pip install --upgrade pip - python -m pip install twine - python .ci/bump_dev_version.py + pip install --upgrade pip + pip install -r conans/requirements.txt + pip install -r conans/requirements_dev.txt + pip install meson - - name: Build and deploy to TestPyPI -# env: -# TWINE_USERNAME: ${{ secrets.PYPITEST_CONAN_CREDENTIALS_USERNAME }} -# TWINE_PASSWORD: ${{ secrets.PYPITEST_CONAN_CREDENTIALS_PASSWORD }} + - name: Run tests run: | - rm -rf dist/ - python setup.py sdist -# twine upload --repository-url https://test.pypi.org/legacy/ dist/* + pytest test/unittests --durations=20 -n 4 diff --git a/ci/conan-unittests b/ci/conan-unittests index cf226d3c6ba..90710eaeaaa 100644 --- a/ci/conan-unittests +++ b/ci/conan-unittests @@ -4,7 +4,7 @@ FROM ubuntu:latest LABEL maintainer="Conan.io " # Update system and install necessary dependencies -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends \ git \ curl \ make \ @@ -24,38 +24,43 @@ RUN apt-get update && apt-get install -y \ liblzma-dev \ python3-openssl \ ca-certificates \ - sudo + sudo \ + && rm -rf /var/lib/apt/lists/* # Create 'conan' user and grant sudo permissions RUN useradd -m -s /bin/bash conan && \ echo 'conan ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -# Switch to root to configure pyenv globally in /etc/profile -USER root - -# Configure pyenv globally via /etc/profile -RUN echo 'export PYENV_ROOT="/home/conan/.pyenv"' >> /etc/profile && \ - echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> /etc/profile && \ - echo 'eval "$(pyenv init --path)"' >> /etc/profile && \ - echo 'eval "$(pyenv init -)"' >> /etc/profile && \ - echo 'eval "$(pyenv virtualenv-init -)"' >> /etc/profile +# Install pyenv dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + libffi-dev \ + && rm -rf /var/lib/apt/lists/* -# Switch back to the 'conan' user -USER conan -WORKDIR /home/conan +# Set environment variables ENV HOME /home/conan -ENV PATH /home/conan/.pyenv/shims:/home/conan/.pyenv/bin:$PATH +ENV PYENV_ROOT $HOME/.pyenv +ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH # Install pyenv -RUN curl https://pyenv.run | bash +RUN git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT + +# Install pyenv-virtualenv (opcional, si lo necesitas) +RUN git clone https://github.com/pyenv/pyenv-virtualenv.git $PYENV_ROOT/plugins/pyenv-virtualenv -RUN echo 'export PATH="/home/conan/.pyenv/bin:$PATH"' >> /home/conan/.bashrc && \ - echo 'eval "$(pyenv init --path)"' >> /home/conan/.bashrc && \ - echo 'eval "$(pyenv init -)"' >> /home/conan/.bashrc && \ - echo 'eval "$(pyenv virtualenv-init -)"' >> /home/conan/.bashrc +# Install Python versions using pyenv +RUN /bin/bash -c "pyenv install 3.8.6 && pyenv global 3.8.6" -# Install Python 3.8.6 using pyenv and set it as the global version -RUN /bin/bash -l -c "pyenv install 3.8.6 && pyenv global 3.8.6" +# Install pip and upgrade +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py && \ + rm get-pip.py + +# Change ownership of the home directory to 'conan' user +RUN chown -R conan:conan $HOME + +# Switch to 'conan' user +USER conan +WORKDIR $HOME # Default command CMD ["/bin/bash"] From f36c22f06786e0a13fdf30983fe3aa10ac21cd5b Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 13 Sep 2024 08:45:49 +0200 Subject: [PATCH 08/54] wip --- ci/conan-unittests | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/ci/conan-unittests b/ci/conan-unittests index 90710eaeaaa..21bce423f4c 100644 --- a/ci/conan-unittests +++ b/ci/conan-unittests @@ -1,9 +1,7 @@ -# Base image is Ubuntu latest FROM ubuntu:latest LABEL maintainer="Conan.io " -# Update system and install necessary dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ git \ curl \ @@ -25,42 +23,33 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ python3-openssl \ ca-certificates \ sudo \ + tar \ && rm -rf /var/lib/apt/lists/* -# Create 'conan' user and grant sudo permissions RUN useradd -m -s /bin/bash conan && \ echo 'conan ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -# Install pyenv dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ libffi-dev \ && rm -rf /var/lib/apt/lists/* -# Set environment variables ENV HOME /home/conan ENV PYENV_ROOT $HOME/.pyenv ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH -# Install pyenv RUN git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT -# Install pyenv-virtualenv (opcional, si lo necesitas) RUN git clone https://github.com/pyenv/pyenv-virtualenv.git $PYENV_ROOT/plugins/pyenv-virtualenv -# Install Python versions using pyenv RUN /bin/bash -c "pyenv install 3.8.6 && pyenv global 3.8.6" -# Install pip and upgrade RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python get-pip.py && \ rm get-pip.py -# Change ownership of the home directory to 'conan' user RUN chown -R conan:conan $HOME -# Switch to 'conan' user USER conan WORKDIR $HOME -# Default command CMD ["/bin/bash"] From b9d119f8db5e411fd3e8742b09ec3cdb035d8abc Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 13 Sep 2024 10:01:34 +0200 Subject: [PATCH 09/54] wip --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index c43549db9bd..fe3fe50f213 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -60,7 +60,7 @@ jobs: python-version: ['3.8.6'] env: PYENV_ROOT: /home/conan/.pyenv - PATH: /home/conan/.pyenv/bin:/home/conan/.pyenv/shims:$PATH + PATH: /home/conan/.pyenv/bin:/home/conan/.pyenv/shims:/usr/bin:/bin:$PATH steps: - name: Checkout code uses: actions/checkout@v3 From 58e9aa5fd1292d0aa1c801c60465c0e958fb7cb7 Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 13 Sep 2024 10:06:14 +0200 Subject: [PATCH 10/54] wip --- .github/workflows/ci-pipeline.yml | 3 --- ci/conan-unittests | 3 +-- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index fe3fe50f213..eed49be882d 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -58,9 +58,6 @@ jobs: strategy: matrix: python-version: ['3.8.6'] - env: - PYENV_ROOT: /home/conan/.pyenv - PATH: /home/conan/.pyenv/bin:/home/conan/.pyenv/shims:/usr/bin:/bin:$PATH steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/ci/conan-unittests b/ci/conan-unittests index 21bce423f4c..ac34d067e3f 100644 --- a/ci/conan-unittests +++ b/ci/conan-unittests @@ -35,10 +35,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ENV HOME /home/conan ENV PYENV_ROOT $HOME/.pyenv -ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH +ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:/usr/bin:/bin:$PATH RUN git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT - RUN git clone https://github.com/pyenv/pyenv-virtualenv.git $PYENV_ROOT/plugins/pyenv-virtualenv RUN /bin/bash -c "pyenv install 3.8.6 && pyenv global 3.8.6" From d202c8ac7c1fa94d8e88944150ab4df0c8789332 Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 20 Sep 2024 07:44:04 +0200 Subject: [PATCH 11/54] wip --- .github/workflows/ci-pipeline.yml | 10 ++----- ci/conan-unittests | 47 +++++++++++-------------------- 2 files changed, 19 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index eed49be882d..5006b2ff73c 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -22,7 +22,6 @@ jobs: id: dockerfile_hash run: | DOCKERFILE_HASH=$(find ./ci/conan-unittests -type f -exec sha256sum {} \; | sha256sum | cut -d' ' -f1) - echo "Dockerfile hash: $DOCKERFILE_HASH" echo "hash=$DOCKERFILE_HASH" >> $GITHUB_OUTPUT - name: Cache Docker image @@ -34,8 +33,7 @@ jobs: - name: Load Docker image from cache if: steps.cache_docker_image.outputs.cache-hit == 'true' - run: | - docker load -i docker_image_cache/conan-unittests.tar + run: docker load -i docker_image_cache/conan-unittests.tar - name: Build Docker image (if not cached) if: steps.cache_docker_image.outputs.cache-hit != 'true' @@ -46,8 +44,7 @@ jobs: - name: Push Docker image to GHCR if: steps.cache_docker_image.outputs.cache-hit != 'true' - run: | - docker push ghcr.io/${{ github.repository_owner }}/conan-unittests:latest + run: docker push ghcr.io/${{ github.repository_owner }}/conan-unittests:latest linux_unittests: needs: build_container @@ -81,5 +78,4 @@ jobs: pip install meson - name: Run tests - run: | - pytest test/unittests --durations=20 -n 4 + run: pytest test/unittests --durations=20 -n 4 diff --git a/ci/conan-unittests b/ci/conan-unittests index ac34d067e3f..e3695b3d641 100644 --- a/ci/conan-unittests +++ b/ci/conan-unittests @@ -1,51 +1,36 @@ -FROM ubuntu:latest +FROM ubuntu:22.04 LABEL maintainer="Conan.io " +# Instalar dependencias necesarias RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - curl \ - make \ - build-essential \ - libssl-dev \ - zlib1g-dev \ - libbz2-dev \ - libreadline-dev \ - libsqlite3-dev \ - wget \ - llvm \ - libncurses5-dev \ - libncursesw5-dev \ - xz-utils \ - tk-dev \ - libffi-dev \ - liblzma-dev \ - python3-openssl \ - ca-certificates \ - sudo \ - tar \ - && rm -rf /var/lib/apt/lists/* + git curl make build-essential libssl-dev zlib1g-dev \ + libbz2-dev libreadline-dev libsqlite3-dev wget llvm \ + libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev \ + liblzma-dev python3-openssl ca-certificates sudo tar && \ + rm -rf /var/lib/apt/lists/* +# Crear usuario 'conan' RUN useradd -m -s /bin/bash conan && \ echo 'conan ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -RUN apt-get update && apt-get install -y --no-install-recommends \ - libffi-dev \ - && rm -rf /var/lib/apt/lists/* - +# Configurar variables de entorno para pyenv ENV HOME /home/conan ENV PYENV_ROOT $HOME/.pyenv -ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:/usr/bin:/bin:$PATH +ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH -RUN git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT -RUN git clone https://github.com/pyenv/pyenv-virtualenv.git $PYENV_ROOT/plugins/pyenv-virtualenv +# Instalar pyenv usando el instalador oficial +RUN curl https://pyenv.run | bash -RUN /bin/bash -c "pyenv install 3.8.6 && pyenv global 3.8.6" +# Instalar Python y establecer versión global +RUN pyenv install 3.8.6 && pyenv global 3.8.6 +# Instalar pip RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python get-pip.py && \ rm get-pip.py +# Cambiar propiedad al usuario 'conan' RUN chown -R conan:conan $HOME USER conan From 1ddd0fcc1dfd979884275a2bed0f79266624b6cb Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 20 Sep 2024 07:45:42 +0200 Subject: [PATCH 12/54] remove comments --- ci/conan-unittests | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ci/conan-unittests b/ci/conan-unittests index e3695b3d641..c3b60075cec 100644 --- a/ci/conan-unittests +++ b/ci/conan-unittests @@ -2,7 +2,6 @@ FROM ubuntu:22.04 LABEL maintainer="Conan.io " -# Instalar dependencias necesarias RUN apt-get update && apt-get install -y --no-install-recommends \ git curl make build-essential libssl-dev zlib1g-dev \ libbz2-dev libreadline-dev libsqlite3-dev wget llvm \ @@ -10,27 +9,21 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ liblzma-dev python3-openssl ca-certificates sudo tar && \ rm -rf /var/lib/apt/lists/* -# Crear usuario 'conan' RUN useradd -m -s /bin/bash conan && \ echo 'conan ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -# Configurar variables de entorno para pyenv ENV HOME /home/conan ENV PYENV_ROOT $HOME/.pyenv ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH -# Instalar pyenv usando el instalador oficial RUN curl https://pyenv.run | bash -# Instalar Python y establecer versión global RUN pyenv install 3.8.6 && pyenv global 3.8.6 -# Instalar pip RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python get-pip.py && \ rm get-pip.py -# Cambiar propiedad al usuario 'conan' RUN chown -R conan:conan $HOME USER conan From 007d6f428757f3c288bd7f527a250d3ff7ceb996 Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 20 Sep 2024 07:48:46 +0200 Subject: [PATCH 13/54] add noninteractive --- ci/conan-unittests | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/conan-unittests b/ci/conan-unittests index c3b60075cec..54a394b6ef4 100644 --- a/ci/conan-unittests +++ b/ci/conan-unittests @@ -2,6 +2,8 @@ FROM ubuntu:22.04 LABEL maintainer="Conan.io " +ENV DEBIAN_FRONTEND=noninteractive + RUN apt-get update && apt-get install -y --no-install-recommends \ git curl make build-essential libssl-dev zlib1g-dev \ libbz2-dev libreadline-dev libsqlite3-dev wget llvm \ From 859d0cdc5d7b4743786190d2e804418c6ca29ea3 Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 20 Sep 2024 07:57:34 +0200 Subject: [PATCH 14/54] wip --- ci/conan-unittests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/conan-unittests b/ci/conan-unittests index 54a394b6ef4..f5d82deebf1 100644 --- a/ci/conan-unittests +++ b/ci/conan-unittests @@ -16,7 +16,7 @@ RUN useradd -m -s /bin/bash conan && \ ENV HOME /home/conan ENV PYENV_ROOT $HOME/.pyenv -ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH +ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:/usr/bin:/bin:$PATH RUN curl https://pyenv.run | bash From 855f6b70e563a88a02250adcb055fc36fe9e90cb Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 20 Sep 2024 08:06:46 +0200 Subject: [PATCH 15/54] use 24.04 --- ci/conan-unittests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/conan-unittests b/ci/conan-unittests index f5d82deebf1..f6796535fdf 100644 --- a/ci/conan-unittests +++ b/ci/conan-unittests @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 LABEL maintainer="Conan.io " From da38fdd4ed3262b8c2460ee4835f7436504eabd5 Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 20 Sep 2024 08:19:03 +0200 Subject: [PATCH 16/54] minor changes --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 5006b2ff73c..6ee565356bd 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -78,4 +78,4 @@ jobs: pip install meson - name: Run tests - run: pytest test/unittests --durations=20 -n 4 + run: pytest test/unittests --durations=20 -n 2 From 469778b2706e62d46cd2b4929ab470250b6edc73 Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 20 Sep 2024 08:22:09 +0200 Subject: [PATCH 17/54] use login-action --- .github/workflows/ci-pipeline.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 6ee565356bd..c6e5f5e221e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -16,7 +16,11 @@ jobs: uses: actions/checkout@v3 - name: Log in to GitHub Container Registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Calculate Dockerfile checksum id: dockerfile_hash From 4d75f86d271c51be29255a355b8d3b42b1cf20de Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 20 Sep 2024 08:22:56 +0200 Subject: [PATCH 18/54] use 4 --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index c6e5f5e221e..b94081e139e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -82,4 +82,4 @@ jobs: pip install meson - name: Run tests - run: pytest test/unittests --durations=20 -n 2 + run: pytest test/unittests --durations=20 -n 4 From 745914ec0641bfb65b2ab0773cbfbf0b62d526ac Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 23 Sep 2024 15:22:56 +0200 Subject: [PATCH 19/54] wip --- .github/workflows/ci-pipeline.yml | 16 +++++++++++++--- ci/conan-unittests | 6 +++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b94081e139e..839084c83d7 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -3,7 +3,8 @@ name: CI Pipeline on: push: branches: - - '*' + - develop2 + - release/* pull_request: branches: - '*' @@ -58,7 +59,8 @@ jobs: options: --user conan strategy: matrix: - python-version: ['3.8.6'] + python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] + test-type: ['unittests', 'integration', 'functional'] steps: - name: Checkout code uses: actions/checkout@v3 @@ -79,7 +81,15 @@ jobs: pip install --upgrade pip pip install -r conans/requirements.txt pip install -r conans/requirements_dev.txt + pip install -r conans/requirements_server.txt pip install meson - name: Run tests - run: pytest test/unittests --durations=20 -n 4 + run: | + if [ "${{ matrix.test-type }}" == "unittests" ]; then + pytest test/unittests --durations=20 -n 4 + elif [ "${{ matrix.test-type }}" == "integration" ]; then + pytest test/integration --durations=20 -n 4 + elif [ "${{ matrix.test-type }}" == "functional" ]; then + pytest test/functional --durations=20 -n 4 + fi diff --git a/ci/conan-unittests b/ci/conan-unittests index f6796535fdf..973dd156e8f 100644 --- a/ci/conan-unittests +++ b/ci/conan-unittests @@ -20,7 +20,11 @@ ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:/usr/bin:/bin:$PATH RUN curl https://pyenv.run | bash -RUN pyenv install 3.8.6 && pyenv global 3.8.6 +RUN pyenv install 3.6.15 && \ + pyenv install 3.8.6 && \ + pyenv install 3.9.2 && \ + pyenv install 3.12.3 && \ + pyenv global 3.9.2 RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python get-pip.py && \ From 45e88c598a5799d094c1f664910df50f4b7a51c3 Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 23 Sep 2024 15:25:00 +0200 Subject: [PATCH 20/54] change image name, just use 1 image for all tests --- .github/workflows/ci-pipeline.yml | 12 ++++++------ ci/{conan-unittests => conan-tests} | 0 2 files changed, 6 insertions(+), 6 deletions(-) rename ci/{conan-unittests => conan-tests} (100%) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 839084c83d7..60ba0abbb2b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -26,7 +26,7 @@ jobs: - name: Calculate Dockerfile checksum id: dockerfile_hash run: | - DOCKERFILE_HASH=$(find ./ci/conan-unittests -type f -exec sha256sum {} \; | sha256sum | cut -d' ' -f1) + DOCKERFILE_HASH=$(find ./ci/conan-tests -type f -exec sha256sum {} \; | sha256sum | cut -d' ' -f1) echo "hash=$DOCKERFILE_HASH" >> $GITHUB_OUTPUT - name: Cache Docker image @@ -38,24 +38,24 @@ jobs: - name: Load Docker image from cache if: steps.cache_docker_image.outputs.cache-hit == 'true' - run: docker load -i docker_image_cache/conan-unittests.tar + run: docker load -i docker_image_cache/conan-tests.tar - name: Build Docker image (if not cached) if: steps.cache_docker_image.outputs.cache-hit != 'true' run: | - docker build -t ghcr.io/${{ github.repository_owner }}/conan-unittests:latest -f ./ci/conan-unittests . + docker build -t ghcr.io/${{ github.repository_owner }}/conan-tests:latest -f ./ci/conan-tests . mkdir -p docker_image_cache - docker save ghcr.io/${{ github.repository_owner }}/conan-unittests:latest -o docker_image_cache/conan-unittests.tar + docker save ghcr.io/${{ github.repository_owner }}/conan-tests:latest -o docker_image_cache/conan-tests.tar - name: Push Docker image to GHCR if: steps.cache_docker_image.outputs.cache-hit != 'true' - run: docker push ghcr.io/${{ github.repository_owner }}/conan-unittests:latest + run: docker push ghcr.io/${{ github.repository_owner }}/conan-tests:latest linux_unittests: needs: build_container runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository_owner }}/conan-unittests:latest + image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest options: --user conan strategy: matrix: diff --git a/ci/conan-unittests b/ci/conan-tests similarity index 100% rename from ci/conan-unittests rename to ci/conan-tests From 7e2374664d61c092a2c527385375b7436d2fc7f1 Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 23 Sep 2024 16:19:45 +0200 Subject: [PATCH 21/54] wip --- ci/conan-tests | 111 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 3 deletions(-) diff --git a/ci/conan-tests b/ci/conan-tests index 973dd156e8f..614f5cdeaa7 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -4,12 +4,35 @@ LABEL maintainer="Conan.io " ENV DEBIAN_FRONTEND=noninteractive +ENV PY36=3.6.15 \ + PY38=3.8.6 \ + PY39=3.9.2 \ + PY312=3.12.3 \ + CMAKE_3_15_7=/usr/share/cmake-3.15.7/bin/cmake \ + CMAKE_3_16_9=/usr/share/cmake-3.16.9/bin/cmake \ + CMAKE_3_17_5=/usr/share/cmake-3.17.5/bin/cmake \ + CMAKE_3_19_7=/usr/share/cmake-3.19.7/bin/cmake \ + CMAKE_3_23_5=/usr/share/cmake-3.23.5/bin/cmake \ + GCC_7=/usr/bin/gcc-7 \ + GXX_7=/usr/bin/g++-7 \ + GCC_9=/usr/bin/gcc-9 \ + GXX_9=/usr/bin/g++-9 \ + GCC_11=/usr/bin/gcc-11 \ + GXX_11=/usr/bin/g++-11 \ + CLANG_7=/usr/bin/clang-7 \ + CLANGXX_7=/usr/bin/clang++-7 \ + CLANG_9=/usr/bin/clang-9 \ + CLANGXX_9=/usr/bin/clang++-9 \ + BAZEL_6_3_2=6.3.2 \ + BAZEL_7_1_2=7.1.2 + RUN apt-get update && apt-get install -y --no-install-recommends \ git curl make build-essential libssl-dev zlib1g-dev \ libbz2-dev libreadline-dev libsqlite3-dev wget llvm \ libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev \ - liblzma-dev python3-openssl ca-certificates sudo tar && \ - rm -rf /var/lib/apt/lists/* + liblzma-dev python3-openssl ca-certificates sudo tar \ + software-properties-common \ + && rm -rf /var/lib/apt/lists/* RUN useradd -m -s /bin/bash conan && \ echo 'conan ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers @@ -17,7 +40,7 @@ RUN useradd -m -s /bin/bash conan && \ ENV HOME /home/conan ENV PYENV_ROOT $HOME/.pyenv ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:/usr/bin:/bin:$PATH - + RUN curl https://pyenv.run | bash RUN pyenv install 3.6.15 && \ @@ -32,6 +55,88 @@ RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ RUN chown -R conan:conan $HOME +USER root + +RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + add-apt-repository -y ppa:fish-shell/release-3 && \ + apt-get update + +RUN apt-get install -y --no-install-recommends fish && \ + rm -rf /var/lib/apt/lists/* + +RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.7/cmake-3.15.7-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.15.7-Linux-x86_64.tar.gz && \ + mv cmake-3.15.7-Linux-x86_64 /usr/share/cmake-3.15.7 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.16.9/cmake-3.16.9-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.16.9-Linux-x86_64.tar.gz && \ + mv cmake-3.16.9-Linux-x86_64 /usr/share/cmake-3.16.9 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.17.5-Linux-x86_64.tar.gz && \ + mv cmake-3.17.5-Linux-x86_64 /usr/share/cmake-3.17.5 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.19.7/cmake-3.19.7-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.19.7-Linux-x86_64.tar.gz && \ + mv cmake-3.19.7-Linux-x86_64 /usr/share/cmake-3.19.7 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.23.5/cmake-3.23.5-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.23.5-Linux-x86_64.tar.gz && \ + mv cmake-3.23.5-Linux-x86_64 /usr/share/cmake-3.23.5 && \ + update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.15.7/bin/cmake 10 && \ + update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.16.9/bin/cmake 20 && \ + update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.17.5/bin/cmake 30 && \ + update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.19.7/bin/cmake 40 && \ + update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.23.5/bin/cmake 50 && \ + update-alternatives --set cmake /usr/share/cmake-3.15.7/bin/cmake && \ + rm cmake-*.tar.gz + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc-7 g++-7 \ + gcc-9 g++-9 \ + gcc-11 g++-11 \ + clang-7 clang++-7 \ + clang-9 clang++-9 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 20 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 10 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 30 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 20 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 10 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 30 && \ + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-7 20 && \ + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-9 10 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-7 20 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-9 10 && \ + update-alternatives --set gcc /usr/bin/gcc-9 && \ + update-alternatives --set g++ /usr/bin/g++-9 && \ + update-alternatives --set clang /usr/bin/clang-9 && \ + update-alternatives --set clang++ /usr/bin/clang++-9 && \ + rm -rf /var/lib/apt/lists/* + +RUN wget https://dl.google.com/android/repository/android-ndk-r23c-linux.zip && \ + unzip android-ndk-r23c-linux.zip -d /opt/ && \ + rm android-ndk-r23c-linux.zip + +RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ + wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_6_3_2}/bazel-${BAZEL_6_3_2}-linux-x86_64 && \ + chmod +x bazel-${BAZEL_6_3_2}-linux-x86_64 && \ + mv bazel-${BAZEL_6_3_2}-linux-x86_64 /usr/share/bazel-$BAZEL_6_3_2/bin/bazel && \ + mkdir -p /usr/share/bazel-$BAZEL_7_1_2/bin && \ + wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_7_1_2}/bazel-${BAZEL_7_1_2}-linux-x86_64 && \ + chmod +x bazel-${BAZEL_7_1_2}-linux-x86_64 && \ + mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel && \ + rm bazel-*.linux-x86_64 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + software-properties-common && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable" && \ + apt-get update && \ + apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io && \ + rm -rf /var/lib/apt/lists/* + +RUN usermod -aG docker conan + USER conan WORKDIR $HOME From 8dd1c1a306723e96d7a7a45da1319f2bd81ad855 Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 23 Sep 2024 17:13:22 +0200 Subject: [PATCH 22/54] fix folder name --- ci/conan-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/conan-tests b/ci/conan-tests index 614f5cdeaa7..7d56690c6ce 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -78,7 +78,7 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.7/cmake-3.15.7 mv cmake-3.19.7-Linux-x86_64 /usr/share/cmake-3.19.7 && \ wget https://github.com/Kitware/CMake/releases/download/v3.23.5/cmake-3.23.5-Linux-x86_64.tar.gz && \ tar -xvzf cmake-3.23.5-Linux-x86_64.tar.gz && \ - mv cmake-3.23.5-Linux-x86_64 /usr/share/cmake-3.23.5 && \ + mv cmake-3.23.5-linux-x86_64 /usr/share/cmake-3.23.5 && \ update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.15.7/bin/cmake 10 && \ update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.16.9/bin/cmake 20 && \ update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.17.5/bin/cmake 30 && \ From 179c9154ff6546dca0b88623a3abf3ba64c31b6d Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 23 Sep 2024 17:52:43 +0200 Subject: [PATCH 23/54] update versions --- ci/conan-tests | 46 +++++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/ci/conan-tests b/ci/conan-tests index 7d56690c6ce..3137e6f8475 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -13,16 +13,12 @@ ENV PY36=3.6.15 \ CMAKE_3_17_5=/usr/share/cmake-3.17.5/bin/cmake \ CMAKE_3_19_7=/usr/share/cmake-3.19.7/bin/cmake \ CMAKE_3_23_5=/usr/share/cmake-3.23.5/bin/cmake \ - GCC_7=/usr/bin/gcc-7 \ - GXX_7=/usr/bin/g++-7 \ GCC_9=/usr/bin/gcc-9 \ GXX_9=/usr/bin/g++-9 \ GCC_11=/usr/bin/gcc-11 \ GXX_11=/usr/bin/g++-11 \ - CLANG_7=/usr/bin/clang-7 \ - CLANGXX_7=/usr/bin/clang++-7 \ - CLANG_9=/usr/bin/clang-9 \ - CLANGXX_9=/usr/bin/clang++-9 \ + CLANG_14=/usr/bin/clang-14 \ + CLANGXX_14=/usr/bin/clang++-14 \ BAZEL_6_3_2=6.3.2 \ BAZEL_7_1_2=7.1.2 @@ -40,7 +36,7 @@ RUN useradd -m -s /bin/bash conan && \ ENV HOME /home/conan ENV PYENV_ROOT $HOME/.pyenv ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:/usr/bin:/bin:$PATH - + RUN curl https://pyenv.run | bash RUN pyenv install 3.6.15 && \ @@ -65,19 +61,19 @@ RUN apt-get install -y --no-install-recommends fish && \ rm -rf /var/lib/apt/lists/* RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.7/cmake-3.15.7-Linux-x86_64.tar.gz && \ - tar -xvzf cmake-3.15.7-Linux-x86_64.tar.gz && \ - mv cmake-3.15.7-Linux-x86_64 /usr/share/cmake-3.15.7 && \ + tar --transform='s/cmake-3.15.7-Linux-x86_64/cmake-3.15.7-linux-x86_64/' -xvzf cmake-3.15.7-Linux-x86_64.tar.gz && \ + mv cmake-3.15.7-linux-x86_64 /usr/share/cmake-3.15.7 && \ wget https://github.com/Kitware/CMake/releases/download/v3.16.9/cmake-3.16.9-Linux-x86_64.tar.gz && \ - tar -xvzf cmake-3.16.9-Linux-x86_64.tar.gz && \ - mv cmake-3.16.9-Linux-x86_64 /usr/share/cmake-3.16.9 && \ + tar --transform='s/cmake-3.16.9-Linux-x86_64/cmake-3.16.9-linux-x86_64/' -xvzf cmake-3.16.9-Linux-x86_64.tar.gz && \ + mv cmake-3.16.9-linux-x86_64 /usr/share/cmake-3.16.9 && \ wget https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz && \ - tar -xvzf cmake-3.17.5-Linux-x86_64.tar.gz && \ - mv cmake-3.17.5-Linux-x86_64 /usr/share/cmake-3.17.5 && \ + tar --transform='s/cmake-3.17.5-Linux-x86_64/cmake-3.17.5-linux-x86_64/' -xvzf cmake-3.17.5-Linux-x86_64.tar.gz && \ + mv cmake-3.17.5-linux-x86_64 /usr/share/cmake-3.17.5 && \ wget https://github.com/Kitware/CMake/releases/download/v3.19.7/cmake-3.19.7-Linux-x86_64.tar.gz && \ - tar -xvzf cmake-3.19.7-Linux-x86_64.tar.gz && \ - mv cmake-3.19.7-Linux-x86_64 /usr/share/cmake-3.19.7 && \ + tar --transform='s/cmake-3.19.7-Linux-x86_64/cmake-3.19.7-linux-x86_64/' -xvzf cmake-3.19.7-Linux-x86_64.tar.gz && \ + mv cmake-3.19.7-linux-x86_64 /usr/share/cmake-3.19.7 && \ wget https://github.com/Kitware/CMake/releases/download/v3.23.5/cmake-3.23.5-Linux-x86_64.tar.gz && \ - tar -xvzf cmake-3.23.5-Linux-x86_64.tar.gz && \ + tar --transform='s/cmake-3.23.5-Linux-x86_64/cmake-3.23.5-linux-x86_64/' -xvzf cmake-3.23.5-Linux-x86_64.tar.gz && \ mv cmake-3.23.5-linux-x86_64 /usr/share/cmake-3.23.5 && \ update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.15.7/bin/cmake 10 && \ update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.16.9/bin/cmake 20 && \ @@ -88,25 +84,21 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.7/cmake-3.15.7 rm cmake-*.tar.gz RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc-7 g++-7 \ gcc-9 g++-9 \ gcc-11 g++-11 \ - clang-7 clang++-7 \ - clang-9 clang++-9 && \ - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 20 && \ + clang-14 clang++-14 && \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 10 && \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 30 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 20 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 10 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 30 && \ - update-alternatives --install /usr/bin/clang clang /usr/bin/clang-7 20 && \ - update-alternatives --install /usr/bin/clang clang /usr/bin/clang-9 10 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-7 20 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-9 10 && \ + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 10 && \ + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-11 30 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 10 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-11 30 && \ update-alternatives --set gcc /usr/bin/gcc-9 && \ update-alternatives --set g++ /usr/bin/g++-9 && \ - update-alternatives --set clang /usr/bin/clang-9 && \ - update-alternatives --set clang++ /usr/bin/clang++-9 && \ + update-alternatives --set clang /usr/bin/clang-14 && \ + update-alternatives --set clang++ /usr/bin/clang++-14 && \ rm -rf /var/lib/apt/lists/* RUN wget https://dl.google.com/android/repository/android-ndk-r23c-linux.zip && \ From e2f2b9a8fe834968bb56ba537c32091062492e06 Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 23 Sep 2024 18:31:09 +0200 Subject: [PATCH 24/54] wip --- ci/conan-tests | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/conan-tests b/ci/conan-tests index 3137e6f8475..495c9843337 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -92,9 +92,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 10 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 30 && \ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 10 && \ - update-alternatives --install /usr/bin/clang clang /usr/bin/clang-11 30 && \ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 10 && \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-11 30 && \ update-alternatives --set gcc /usr/bin/gcc-9 && \ update-alternatives --set g++ /usr/bin/g++-9 && \ update-alternatives --set clang /usr/bin/clang-14 && \ From 52658a0b531495214f1ef53e6f2e39996da58f32 Mon Sep 17 00:00:00 2001 From: czoido Date: Mon, 23 Sep 2024 18:44:19 +0200 Subject: [PATCH 25/54] wip --- ci/conan-tests | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/conan-tests b/ci/conan-tests index 495c9843337..fc6cc2b7e77 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -28,6 +28,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev \ liblzma-dev python3-openssl ca-certificates sudo tar \ software-properties-common \ + autotools-dev automake linux-libc-dev subversion subversion-tools ninja-build \ + gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf unzip \ && rm -rf /var/lib/apt/lists/* RUN useradd -m -s /bin/bash conan && \ From 90880a4122246cc0624180706ba5e7790c810347 Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 07:23:49 +0200 Subject: [PATCH 26/54] minor changes --- ci/conan-tests | 54 +++++++++++++++++++++----------------------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/ci/conan-tests b/ci/conan-tests index fc6cc2b7e77..294b7235cbd 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -22,15 +22,25 @@ ENV PY36=3.6.15 \ BAZEL_6_3_2=6.3.2 \ BAZEL_7_1_2=7.1.2 -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + add-apt-repository -y ppa:fish-shell/release-3 && \ + apt-get update + apt-get update && apt-get install -y --no-install-recommends \ git curl make build-essential libssl-dev zlib1g-dev \ libbz2-dev libreadline-dev libsqlite3-dev wget llvm \ libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev \ liblzma-dev python3-openssl ca-certificates sudo tar \ software-properties-common \ autotools-dev automake linux-libc-dev subversion subversion-tools ninja-build \ + gcc-9 g++-9 \ + gcc-11 g++-11 \ + clang-14 clang++-14 \ + fish \ gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf unzip \ - && rm -rf /var/lib/apt/lists/* + apt-transport-https \ + gnupg-agent \ + software-properties-common && \ + rm -rf /var/lib/apt/lists/* RUN useradd -m -s /bin/bash conan && \ echo 'conan ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers @@ -55,13 +65,6 @@ RUN chown -R conan:conan $HOME USER root -RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ - add-apt-repository -y ppa:fish-shell/release-3 && \ - apt-get update - -RUN apt-get install -y --no-install-recommends fish && \ - rm -rf /var/lib/apt/lists/* - RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.7/cmake-3.15.7-Linux-x86_64.tar.gz && \ tar --transform='s/cmake-3.15.7-Linux-x86_64/cmake-3.15.7-linux-x86_64/' -xvzf cmake-3.15.7-Linux-x86_64.tar.gz && \ mv cmake-3.15.7-linux-x86_64 /usr/share/cmake-3.15.7 && \ @@ -85,11 +88,7 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.7/cmake-3.15.7 update-alternatives --set cmake /usr/share/cmake-3.15.7/bin/cmake && \ rm cmake-*.tar.gz -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc-9 g++-9 \ - gcc-11 g++-11 \ - clang-14 clang++-14 && \ - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 10 && \ +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 10 && \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 30 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 10 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 30 && \ @@ -98,13 +97,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ update-alternatives --set gcc /usr/bin/gcc-9 && \ update-alternatives --set g++ /usr/bin/g++-9 && \ update-alternatives --set clang /usr/bin/clang-14 && \ - update-alternatives --set clang++ /usr/bin/clang++-14 && \ - rm -rf /var/lib/apt/lists/* + update-alternatives --set clang++ /usr/bin/clang++-14 RUN wget https://dl.google.com/android/repository/android-ndk-r23c-linux.zip && \ unzip android-ndk-r23c-linux.zip -d /opt/ && \ rm android-ndk-r23c-linux.zip + +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable" && \ + apt-get update && \ + apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io && \ + rm -rf /var/lib/apt/lists/* + RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_6_3_2}/bazel-${BAZEL_6_3_2}-linux-x86_64 && \ chmod +x bazel-${BAZEL_6_3_2}-linux-x86_64 && \ @@ -112,21 +117,8 @@ RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ mkdir -p /usr/share/bazel-$BAZEL_7_1_2/bin && \ wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_7_1_2}/bazel-${BAZEL_7_1_2}-linux-x86_64 && \ chmod +x bazel-${BAZEL_7_1_2}-linux-x86_64 && \ - mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel && \ - rm bazel-*.linux-x86_64 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg-agent \ - software-properties-common && \ - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable" && \ - apt-get update && \ - apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io && \ - rm -rf /var/lib/apt/lists/* - + mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel + RUN usermod -aG docker conan USER conan From 219b7a2880e8a62f4fff0fd20117a9e7efb4c056 Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 07:24:53 +0200 Subject: [PATCH 27/54] fix --- ci/conan-tests | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/conan-tests b/ci/conan-tests index 294b7235cbd..2fbdc5e0112 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -24,7 +24,6 @@ ENV PY36=3.6.15 \ RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ add-apt-repository -y ppa:fish-shell/release-3 && \ - apt-get update apt-get update && apt-get install -y --no-install-recommends \ git curl make build-essential libssl-dev zlib1g-dev \ libbz2-dev libreadline-dev libsqlite3-dev wget llvm \ From 9690c30f0ed7abb45b0a3f7c6b725d7277f86ea9 Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 07:31:01 +0200 Subject: [PATCH 28/54] fix --- ci/conan-tests | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/conan-tests b/ci/conan-tests index 2fbdc5e0112..d03c701860a 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -22,7 +22,9 @@ ENV PY36=3.6.15 \ BAZEL_6_3_2=6.3.2 \ BAZEL_7_1_2=7.1.2 -RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ +RUN apt-get update && apt-get install -y --no-install-recommends \ + software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ add-apt-repository -y ppa:fish-shell/release-3 && \ apt-get update && apt-get install -y --no-install-recommends \ git curl make build-essential libssl-dev zlib1g-dev \ @@ -37,8 +39,7 @@ RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ fish \ gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf unzip \ apt-transport-https \ - gnupg-agent \ - software-properties-common && \ + gnupg-agent && \ rm -rf /var/lib/apt/lists/* RUN useradd -m -s /bin/bash conan && \ From 90c7750262eedd9f4b17a30f0a44300e2a5cacae Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 07:55:57 +0200 Subject: [PATCH 29/54] wip --- .github/workflows/ci-pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 60ba0abbb2b..bb25e4c60ac 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -85,6 +85,7 @@ jobs: pip install meson - name: Run tests + shell: bash run: | if [ "${{ matrix.test-type }}" == "unittests" ]; then pytest test/unittests --durations=20 -n 4 From 2e8b84ad217a7e23d3faa4cca1dd6c2f9f9975ff Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 09:35:11 +0200 Subject: [PATCH 30/54] install autoconf and pkg-config --- ci/conan-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/conan-tests b/ci/conan-tests index d03c701860a..c1e568e0152 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -32,7 +32,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev \ liblzma-dev python3-openssl ca-certificates sudo tar \ software-properties-common \ - autotools-dev automake linux-libc-dev subversion subversion-tools ninja-build \ + autotools-dev automake autoconf pkg-config linux-libc-dev subversion subversion-tools ninja-build \ gcc-9 g++-9 \ gcc-11 g++-11 \ clang-14 clang++-14 \ From c997656f8f7a2faaef136bf9eacf730eb0cbd791 Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 09:35:45 +0200 Subject: [PATCH 31/54] wip --- ci/conan-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/conan-tests b/ci/conan-tests index c1e568e0152..5edb2c6c59a 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -32,7 +32,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev \ liblzma-dev python3-openssl ca-certificates sudo tar \ software-properties-common \ - autotools-dev automake autoconf pkg-config linux-libc-dev subversion subversion-tools ninja-build \ + autotools-dev automake autoconf libtool pkg-config gettext linux-libc-dev subversion subversion-tools ninja-build \ gcc-9 g++-9 \ gcc-11 g++-11 \ clang-14 clang++-14 \ From 0c947aa53b778904e6843bccd51dd94730fae99a Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 09:58:26 +0200 Subject: [PATCH 32/54] optimize --- ci/conan-tests | 113 ++++++++++++++++++++++++++++--------------------- 1 file changed, 65 insertions(+), 48 deletions(-) diff --git a/ci/conan-tests b/ci/conan-tests index 5edb2c6c59a..3574620bd75 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -22,24 +22,58 @@ ENV PY36=3.6.15 \ BAZEL_6_3_2=6.3.2 \ BAZEL_7_1_2=7.1.2 -RUN apt-get update && apt-get install -y --no-install-recommends \ - software-properties-common && \ +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + build-essential \ + libtool \ + automake \ + autoconf \ + pkg-config \ + gettext \ + git \ + curl \ + make \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + wget \ + llvm \ + libncurses5-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libffi-dev \ + liblzma-dev \ + python3-openssl \ + ca-certificates \ + sudo \ + tar \ + linux-libc-dev \ + subversion \ + subversion-tools \ + ninja-build \ + gcc-9 \ + g++-9 \ + gcc-11 \ + g++-11 \ + clang-14 \ + clang++-14 \ + fish \ + gcc-arm-linux-gnueabihf \ + g++-arm-linux-gnueabihf \ + unzip \ + apt-transport-https \ + gnupg-agent && \ add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ add-apt-repository -y ppa:fish-shell/release-3 && \ - apt-get update && apt-get install -y --no-install-recommends \ - git curl make build-essential libssl-dev zlib1g-dev \ - libbz2-dev libreadline-dev libsqlite3-dev wget llvm \ - libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev \ - liblzma-dev python3-openssl ca-certificates sudo tar \ - software-properties-common \ - autotools-dev automake autoconf libtool pkg-config gettext linux-libc-dev subversion subversion-tools ninja-build \ - gcc-9 g++-9 \ - gcc-11 g++-11 \ - clang-14 clang++-14 \ - fish \ - gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf unzip \ - apt-transport-https \ - gnupg-agent && \ + apt-get update && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable" && \ + apt-get update && \ + apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io && \ rm -rf /var/lib/apt/lists/* RUN useradd -m -s /bin/bash conan && \ @@ -49,15 +83,13 @@ ENV HOME /home/conan ENV PYENV_ROOT $HOME/.pyenv ENV PATH $PYENV_ROOT/bin:$PYENV_ROOT/shims:/usr/bin:/bin:$PATH -RUN curl https://pyenv.run | bash - -RUN pyenv install 3.6.15 && \ - pyenv install 3.8.6 && \ - pyenv install 3.9.2 && \ - pyenv install 3.12.3 && \ - pyenv global 3.9.2 - -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ +RUN curl https://pyenv.run | bash && \ + pyenv install $PY36 && \ + pyenv install $PY38 && \ + pyenv install $PY39 && \ + pyenv install $PY312 && \ + pyenv global $PY39 && \ + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python get-pip.py && \ rm get-pip.py @@ -66,20 +98,15 @@ RUN chown -R conan:conan $HOME USER root RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.7/cmake-3.15.7-Linux-x86_64.tar.gz && \ - tar --transform='s/cmake-3.15.7-Linux-x86_64/cmake-3.15.7-linux-x86_64/' -xvzf cmake-3.15.7-Linux-x86_64.tar.gz && \ - mv cmake-3.15.7-linux-x86_64 /usr/share/cmake-3.15.7 && \ + tar --strip-components=1 -xzf cmake-3.15.7-Linux-x86_64.tar.gz -C /usr/share/cmake-3.15.7 && \ wget https://github.com/Kitware/CMake/releases/download/v3.16.9/cmake-3.16.9-Linux-x86_64.tar.gz && \ - tar --transform='s/cmake-3.16.9-Linux-x86_64/cmake-3.16.9-linux-x86_64/' -xvzf cmake-3.16.9-Linux-x86_64.tar.gz && \ - mv cmake-3.16.9-linux-x86_64 /usr/share/cmake-3.16.9 && \ + tar --strip-components=1 -xzf cmake-3.16.9-Linux-x86_64.tar.gz -C /usr/share/cmake-3.16.9 && \ wget https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz && \ - tar --transform='s/cmake-3.17.5-Linux-x86_64/cmake-3.17.5-linux-x86_64/' -xvzf cmake-3.17.5-Linux-x86_64.tar.gz && \ - mv cmake-3.17.5-linux-x86_64 /usr/share/cmake-3.17.5 && \ + tar --strip-components=1 -xzf cmake-3.17.5-Linux-x86_64.tar.gz -C /usr/share/cmake-3.17.5 && \ wget https://github.com/Kitware/CMake/releases/download/v3.19.7/cmake-3.19.7-Linux-x86_64.tar.gz && \ - tar --transform='s/cmake-3.19.7-Linux-x86_64/cmake-3.19.7-linux-x86_64/' -xvzf cmake-3.19.7-Linux-x86_64.tar.gz && \ - mv cmake-3.19.7-linux-x86_64 /usr/share/cmake-3.19.7 && \ + tar --strip-components=1 -xzf cmake-3.19.7-Linux-x86_64.tar.gz -C /usr/share/cmake-3.19.7 && \ wget https://github.com/Kitware/CMake/releases/download/v3.23.5/cmake-3.23.5-Linux-x86_64.tar.gz && \ - tar --transform='s/cmake-3.23.5-Linux-x86_64/cmake-3.23.5-linux-x86_64/' -xvzf cmake-3.23.5-Linux-x86_64.tar.gz && \ - mv cmake-3.23.5-linux-x86_64 /usr/share/cmake-3.23.5 && \ + tar --strip-components=1 -xzf cmake-3.23.5-Linux-x86_64.tar.gz -C /usr/share/cmake-3.23.5 && \ update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.15.7/bin/cmake 10 && \ update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.16.9/bin/cmake 20 && \ update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.17.5/bin/cmake 30 && \ @@ -99,17 +126,6 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 10 && \ update-alternatives --set clang /usr/bin/clang-14 && \ update-alternatives --set clang++ /usr/bin/clang++-14 -RUN wget https://dl.google.com/android/repository/android-ndk-r23c-linux.zip && \ - unzip android-ndk-r23c-linux.zip -d /opt/ && \ - rm android-ndk-r23c-linux.zip - - -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu jammy stable" && \ - apt-get update && \ - apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io && \ - rm -rf /var/lib/apt/lists/* - RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_6_3_2}/bazel-${BAZEL_6_3_2}-linux-x86_64 && \ chmod +x bazel-${BAZEL_6_3_2}-linux-x86_64 && \ @@ -117,8 +133,9 @@ RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ mkdir -p /usr/share/bazel-$BAZEL_7_1_2/bin && \ wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_7_1_2}/bazel-${BAZEL_7_1_2}-linux-x86_64 && \ chmod +x bazel-${BAZEL_7_1_2}-linux-x86_64 && \ - mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel - + mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel && \ + rm bazel-${BAZEL_6_3_2}-linux-x86_64 bazel-${BAZEL_7_1_2}-linux-x86_64 + RUN usermod -aG docker conan USER conan From 724e277af8caa3b9b57d40cc839415184de0765c Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 10:01:03 +0200 Subject: [PATCH 33/54] add scons --- ci/conan-tests | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/conan-tests b/ci/conan-tests index 3574620bd75..592c45e78a9 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -66,7 +66,12 @@ RUN apt-get update && \ g++-arm-linux-gnueabihf \ unzip \ apt-transport-https \ - gnupg-agent && \ + gnupg-agent \ + gcc-9-multilib \ + g++-9-multilib \ + gcc-11-multilib \ + g++-11-multilib \ + scons && \ add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ add-apt-repository -y ppa:fish-shell/release-3 && \ apt-get update && \ From 3e6e28bcb12e45ddeed9cfab15f05b3c00ad3750 Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 10:20:02 +0200 Subject: [PATCH 34/54] wip --- ci/conan-tests | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/ci/conan-tests b/ci/conan-tests index 592c45e78a9..14d56edc453 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -103,22 +103,20 @@ RUN chown -R conan:conan $HOME USER root RUN wget https://github.com/Kitware/CMake/releases/download/v3.15.7/cmake-3.15.7-Linux-x86_64.tar.gz && \ - tar --strip-components=1 -xzf cmake-3.15.7-Linux-x86_64.tar.gz -C /usr/share/cmake-3.15.7 && \ - wget https://github.com/Kitware/CMake/releases/download/v3.16.9/cmake-3.16.9-Linux-x86_64.tar.gz && \ - tar --strip-components=1 -xzf cmake-3.16.9-Linux-x86_64.tar.gz -C /usr/share/cmake-3.16.9 && \ - wget https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz && \ - tar --strip-components=1 -xzf cmake-3.17.5-Linux-x86_64.tar.gz -C /usr/share/cmake-3.17.5 && \ - wget https://github.com/Kitware/CMake/releases/download/v3.19.7/cmake-3.19.7-Linux-x86_64.tar.gz && \ - tar --strip-components=1 -xzf cmake-3.19.7-Linux-x86_64.tar.gz -C /usr/share/cmake-3.19.7 && \ - wget https://github.com/Kitware/CMake/releases/download/v3.23.5/cmake-3.23.5-Linux-x86_64.tar.gz && \ - tar --strip-components=1 -xzf cmake-3.23.5-Linux-x86_64.tar.gz -C /usr/share/cmake-3.23.5 && \ - update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.15.7/bin/cmake 10 && \ - update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.16.9/bin/cmake 20 && \ - update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.17.5/bin/cmake 30 && \ - update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.19.7/bin/cmake 40 && \ - update-alternatives --install /usr/bin/cmake cmake /usr/share/cmake-3.23.5/bin/cmake 50 && \ - update-alternatives --set cmake /usr/share/cmake-3.15.7/bin/cmake && \ - rm cmake-*.tar.gz + tar -xvzf cmake-3.15.7-Linux-x86_64.tar.gz && mv cmake-3.15.7-Linux-x86_64 /usr/share/cmake-3.15.7 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.16.9/cmake-3.16.9-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.16.9-Linux-x86_64.tar.gz && mv cmake-3.16.9-Linux-x86_64 /usr/share/cmake-3.16.9 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.17.5/cmake-3.17.5-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.17.5-Linux-x86_64.tar.gz && mv cmake-3.17.5-Linux-x86_64 /usr/share/cmake-3.17.5 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.19.7/cmake-3.19.7-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.19.7-Linux-x86_64.tar.gz && mv cmake-3.19.7-Linux-x86_64 /usr/share/cmake-3.19.7 && \ + wget https://github.com/Kitware/CMake/releases/download/v3.23.5/cmake-3.23.5-Linux-x86_64.tar.gz && \ + tar -xvzf cmake-3.23.5-Linux-x86_64.tar.gz && mv cmake-3.23.5-linux-x86_64/ /usr/share/cmake-3.23.5 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_15_7 10 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_16_9 20 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_17_5 30 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_19_7 40 && \ + update-alternatives --install /usr/bin/cmake cmake $CMAKE_3_23_5 50 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 10 && \ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 30 && \ From 634be01e255e044f29f5ecaeec09d194f4c408c3 Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 10:29:37 +0200 Subject: [PATCH 35/54] wip --- ci/conan-tests | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/conan-tests b/ci/conan-tests index 14d56edc453..ee3652e6d7a 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -136,8 +136,7 @@ RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ mkdir -p /usr/share/bazel-$BAZEL_7_1_2/bin && \ wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_7_1_2}/bazel-${BAZEL_7_1_2}-linux-x86_64 && \ chmod +x bazel-${BAZEL_7_1_2}-linux-x86_64 && \ - mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel && \ - rm bazel-${BAZEL_6_3_2}-linux-x86_64 bazel-${BAZEL_7_1_2}-linux-x86_64 + mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel RUN usermod -aG docker conan From 0e083256eb5838151a6e4168aeeb7786ef487d0e Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 10:51:07 +0200 Subject: [PATCH 36/54] fix asm err --- ci/conan-tests | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/conan-tests b/ci/conan-tests index ee3652e6d7a..4a81f687aa6 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -72,6 +72,8 @@ RUN apt-get update && \ gcc-11-multilib \ g++-11-multilib \ scons && \ + # fix: asm/errno.h: No such file or directory + ln -s /usr/include/asm-generic/ /usr/include/asm && \ add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ add-apt-repository -y ppa:fish-shell/release-3 && \ apt-get update && \ From ee53811c23178e3317bf7474c1bc4b5f3d0f652a Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 11:28:23 +0200 Subject: [PATCH 37/54] wip --- .github/workflows/ci-pipeline.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index bb25e4c60ac..b0e5107332b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -60,7 +60,6 @@ jobs: strategy: matrix: python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] - test-type: ['unittests', 'integration', 'functional'] steps: - name: Checkout code uses: actions/checkout@v3 @@ -84,13 +83,17 @@ jobs: pip install -r conans/requirements_server.txt pip install meson - - name: Run tests + - name: Run unit tests shell: bash run: | - if [ "${{ matrix.test-type }}" == "unittests" ]; then - pytest test/unittests --durations=20 -n 4 - elif [ "${{ matrix.test-type }}" == "integration" ]; then - pytest test/integration --durations=20 -n 4 - elif [ "${{ matrix.test-type }}" == "functional" ]; then - pytest test/functional --durations=20 -n 4 - fi + pytest test/unittests --durations=20 -n 4 + + - name: Run integration tests + shell: bash + run: | + pytest test/integration --durations=20 -n 4 + + - name: Run functional tests + shell: bash + run: | + pytest test/functional --durations=20 -n 4 From 9af8d0c79f8a0ad12675d824c8fbc1abedfe669c Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 11:31:46 +0200 Subject: [PATCH 38/54] wip --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b0e5107332b..533f6cdf906 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -51,7 +51,7 @@ jobs: if: steps.cache_docker_image.outputs.cache-hit != 'true' run: docker push ghcr.io/${{ github.repository_owner }}/conan-tests:latest - linux_unittests: + linux: needs: build_container runs-on: ubuntu-latest container: From 0fc2c3e0338b919942e0eab1cb11496c5c431a50 Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 11:51:18 +0200 Subject: [PATCH 39/54] update name --- .github/workflows/ci-pipeline.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 533f6cdf906..36112a9038b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -60,6 +60,8 @@ jobs: strategy: matrix: python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] + test-type: ['unittests', 'integration', 'functional'] + name: Python ${{ matrix.python-version }} - ${{ matrix.test-type }} steps: - name: Checkout code uses: actions/checkout@v3 @@ -83,17 +85,13 @@ jobs: pip install -r conans/requirements_server.txt pip install meson - - name: Run unit tests + - name: Run tests shell: bash run: | - pytest test/unittests --durations=20 -n 4 - - - name: Run integration tests - shell: bash - run: | - pytest test/integration --durations=20 -n 4 - - - name: Run functional tests - shell: bash - run: | - pytest test/functional --durations=20 -n 4 + if [ "${{ matrix.test-type }}" == "unittests" ]; then + pytest test/unittests --durations=20 -n 4 + elif [ "${{ matrix.test-type }}" == "integration" ]; then + pytest test/integration --durations=20 -n 4 + elif [ "${{ matrix.test-type }}" == "functional" ]; then + pytest test/functional --durations=20 -n 4 + fi From e0f2d33a29de816ea72ad2d630380782d7655efb Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 12:16:36 +0200 Subject: [PATCH 40/54] wip --- .github/workflows/{ci-pipeline.yml => linux-tests.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{ci-pipeline.yml => linux-tests.yml} (99%) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/linux-tests.yml similarity index 99% rename from .github/workflows/ci-pipeline.yml rename to .github/workflows/linux-tests.yml index 36112a9038b..5461e4d16b6 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/linux-tests.yml @@ -1,4 +1,4 @@ -name: CI Pipeline +name: Linux tests on: push: From e4923626f6a501db5a861bddee786f25f6e6296a Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 15:24:45 +0200 Subject: [PATCH 41/54] check if docker tests run by any chance --- test/functional/command/runner_test.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index fb601288aeb..2b264f019a6 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -36,7 +36,7 @@ def dockerfile_path(name=None): return path -@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared(): """ Tests the ``conan create . `` @@ -79,7 +79,7 @@ def test_create_docker_runner_cache_shared(): assert "Removing container" in client.out -@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared_profile_from_cache(): """ Tests the ``conan create . `` @@ -122,7 +122,7 @@ def test_create_docker_runner_cache_shared_profile_from_cache(): assert "Removing container" in client.out -@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared_profile_folder(): """ Tests the ``conan create . `` @@ -165,7 +165,7 @@ def test_create_docker_runner_cache_shared_profile_folder(): assert "[100%] Built target example" in client.out assert "Removing container" in client.out -@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_dockerfile_folder_path(): """ Tests the ``conan create . `` @@ -235,7 +235,7 @@ def test_create_docker_runner_dockerfile_folder_path(): assert "Removing container" in client.out -@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_profile_default_folder(): """ Tests the ``conan create . `` @@ -280,7 +280,7 @@ def test_create_docker_runner_profile_default_folder(): assert "Removing container" in client.out -@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_dockerfile_file_path(): """ Tests the ``conan create . `` @@ -324,7 +324,7 @@ def test_create_docker_runner_dockerfile_file_path(): assert "Removing container" in client.out -@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") @pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) @pytest.mark.tool("ninja") def test_create_docker_runner_with_ninja(build_type, shared): @@ -389,7 +389,7 @@ def package(self): assert 'cmake -G "Ninja"' in client.out assert "main: {}!".format(build_type) in client.out -@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_from_configfile(): """ Tests the ``conan create . `` @@ -443,7 +443,7 @@ def test_create_docker_runner_from_configfile(): assert "Removing container" in client.out -@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_from_configfile_with_args(): """ Tests the ``conan create . `` @@ -499,7 +499,7 @@ def test_create_docker_runner_from_configfile_with_args(): assert "Removing container" in client.out -@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_default_build_profile(): """ Tests the ``conan create . `` From 6bdd0322860f82f7b2b445439827950340c5f9f2 Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 15:34:09 +0200 Subject: [PATCH 42/54] revert change --- test/functional/command/runner_test.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index 2b264f019a6..fb601288aeb 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -36,7 +36,7 @@ def dockerfile_path(name=None): return path -#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared(): """ Tests the ``conan create . `` @@ -79,7 +79,7 @@ def test_create_docker_runner_cache_shared(): assert "Removing container" in client.out -#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared_profile_from_cache(): """ Tests the ``conan create . `` @@ -122,7 +122,7 @@ def test_create_docker_runner_cache_shared_profile_from_cache(): assert "Removing container" in client.out -#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared_profile_folder(): """ Tests the ``conan create . `` @@ -165,7 +165,7 @@ def test_create_docker_runner_cache_shared_profile_folder(): assert "[100%] Built target example" in client.out assert "Removing container" in client.out -#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_dockerfile_folder_path(): """ Tests the ``conan create . `` @@ -235,7 +235,7 @@ def test_create_docker_runner_dockerfile_folder_path(): assert "Removing container" in client.out -#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_profile_default_folder(): """ Tests the ``conan create . `` @@ -280,7 +280,7 @@ def test_create_docker_runner_profile_default_folder(): assert "Removing container" in client.out -#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_dockerfile_file_path(): """ Tests the ``conan create . `` @@ -324,7 +324,7 @@ def test_create_docker_runner_dockerfile_file_path(): assert "Removing container" in client.out -#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") @pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) @pytest.mark.tool("ninja") def test_create_docker_runner_with_ninja(build_type, shared): @@ -389,7 +389,7 @@ def package(self): assert 'cmake -G "Ninja"' in client.out assert "main: {}!".format(build_type) in client.out -#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_from_configfile(): """ Tests the ``conan create . `` @@ -443,7 +443,7 @@ def test_create_docker_runner_from_configfile(): assert "Removing container" in client.out -#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_from_configfile_with_args(): """ Tests the ``conan create . `` @@ -499,7 +499,7 @@ def test_create_docker_runner_from_configfile_with_args(): assert "Removing container" in client.out -#@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_default_build_profile(): """ Tests the ``conan create . `` From 3c56a3d21d6746b4eff17595340c3518c984fca6 Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 24 Sep 2024 15:35:41 +0200 Subject: [PATCH 43/54] show skip --- .github/workflows/linux-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 5461e4d16b6..2c081c37414 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -89,9 +89,9 @@ jobs: shell: bash run: | if [ "${{ matrix.test-type }}" == "unittests" ]; then - pytest test/unittests --durations=20 -n 4 + pytest test/unittests --durations=20 -n 4 -rs elif [ "${{ matrix.test-type }}" == "integration" ]; then - pytest test/integration --durations=20 -n 4 + pytest test/integration --durations=20 -n 4 -rs elif [ "${{ matrix.test-type }}" == "functional" ]; then - pytest test/functional --durations=20 -n 4 + pytest test/functional --durations=20 -n 4 -rs fi From 4a714547f33097a683b16300bd4b57d381ea786b Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 26 Sep 2024 17:48:00 +0200 Subject: [PATCH 44/54] check if we can run docker tests --- .github/workflows/linux-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 2c081c37414..73d1126e334 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -56,7 +56,7 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest - options: --user conan + options: --user conan --volume /var/run/docker.sock:/var/run/docker.sock strategy: matrix: python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] From 5dbd8b91fcc7ef54c81bf0d6177eef0292c2ce2f Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 26 Sep 2024 17:59:22 +0200 Subject: [PATCH 45/54] try to run docker tests --- .github/workflows/linux-tests.yml | 19 ++++++++++--------- pytest.ini | 1 + test/functional/command/runner_test.py | 10 ++++++++++ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 73d1126e334..26009428dd0 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -59,8 +59,8 @@ jobs: options: --user conan --volume /var/run/docker.sock:/var/run/docker.sock strategy: matrix: - python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] - test-type: ['unittests', 'integration', 'functional'] + python-version: ['3.12.3']#, '3.9.2', '3.8.6', '3.6.15'] + test-type: ['unittests']#, 'integration', 'functional'] name: Python ${{ matrix.python-version }} - ${{ matrix.test-type }} steps: - name: Checkout code @@ -88,10 +88,11 @@ jobs: - name: Run tests shell: bash run: | - if [ "${{ matrix.test-type }}" == "unittests" ]; then - pytest test/unittests --durations=20 -n 4 -rs - elif [ "${{ matrix.test-type }}" == "integration" ]; then - pytest test/integration --durations=20 -n 4 -rs - elif [ "${{ matrix.test-type }}" == "functional" ]; then - pytest test/functional --durations=20 -n 4 -rs - fi + pytest -m docker_runner +# if [ "${{ matrix.test-type }}" == "unittests" ]; then +# pytest test/unittests --durations=20 -n 4 -rs +# elif [ "${{ matrix.test-type }}" == "integration" ]; then +# pytest test/integration --durations=20 -n 4 -rs +# elif [ "${{ matrix.test-type }}" == "functional" ]; then +# pytest test/functional --durations=20 -n 4 -rs +# fi diff --git a/pytest.ini b/pytest.ini index a69be4e9060..26456fc871e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,4 @@ [pytest] norecursedirs = '.*', 'dist', 'CVS', '_darcs', '{arch}', '*.egg', 'venv', 'assets' testpaths = 'test' +markers = docker_runner: Mark tests that require Docker to run. diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index fb601288aeb..4d4f58fa610 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -36,6 +36,7 @@ def dockerfile_path(name=None): return path +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared(): """ @@ -79,6 +80,7 @@ def test_create_docker_runner_cache_shared(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared_profile_from_cache(): """ @@ -122,6 +124,7 @@ def test_create_docker_runner_cache_shared_profile_from_cache(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_cache_shared_profile_folder(): """ @@ -165,6 +168,7 @@ def test_create_docker_runner_cache_shared_profile_folder(): assert "[100%] Built target example" in client.out assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_dockerfile_folder_path(): """ @@ -235,6 +239,7 @@ def test_create_docker_runner_dockerfile_folder_path(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_profile_default_folder(): """ @@ -280,6 +285,7 @@ def test_create_docker_runner_profile_default_folder(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_dockerfile_file_path(): """ @@ -324,6 +330,7 @@ def test_create_docker_runner_dockerfile_file_path(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") @pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) @pytest.mark.tool("ninja") @@ -389,6 +396,7 @@ def package(self): assert 'cmake -G "Ninja"' in client.out assert "main: {}!".format(build_type) in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_from_configfile(): """ @@ -443,6 +451,7 @@ def test_create_docker_runner_from_configfile(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_from_configfile_with_args(): """ @@ -499,6 +508,7 @@ def test_create_docker_runner_from_configfile_with_args(): assert "Removing container" in client.out +@pytest.mark.docker_runner @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") def test_create_docker_runner_default_build_profile(): """ From 8bb566f7bf3082ad3477091aeddd32b5760f3f06 Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 26 Sep 2024 18:06:37 +0200 Subject: [PATCH 46/54] WIP --- .github/workflows/linux-tests.yml | 2 +- test/functional/command/runner_test.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 26009428dd0..b9af4af0abe 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -88,7 +88,7 @@ jobs: - name: Run tests shell: bash run: | - pytest -m docker_runner + pytest -m docker_runner -rs # if [ "${{ matrix.test-type }}" == "unittests" ]; then # pytest test/unittests --durations=20 -n 4 -rs # elif [ "${{ matrix.test-type }}" == "integration" ]; then diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index 4d4f58fa610..1d41a7ecc52 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -15,12 +15,12 @@ def docker_skip(test_image=None): docker_client = docker.DockerClient(base_url=f'unix://{os.path.expanduser("~")}/.rd/docker.sock', version='auto') # Rancher if test_image: docker_client.images.pull(test_image) - except docker.errors.DockerException: - return True - except docker.errors.ImageNotFound: - return True - except docker.errors.APIError: - return True + except docker.errors.DockerException as e: + raise e + except docker.errors.ImageNotFound as e: + raise e + except docker.errors.APIError as e: + raise e return False From a0398e46d17c62f632fad7a137cb3571ec63ae43 Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 26 Sep 2024 18:15:33 +0200 Subject: [PATCH 47/54] wip --- .github/workflows/linux-tests.yml | 5 +++++ ci/conan-tests | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index b9af4af0abe..27be1704721 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -85,6 +85,11 @@ jobs: pip install -r conans/requirements_server.txt pip install meson + - name: Check Docker socket permissions + run: | + ls -l /var/run/docker.sock + id + - name: Run tests shell: bash run: | diff --git a/ci/conan-tests b/ci/conan-tests index 4a81f687aa6..eb2ffc957c9 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -140,7 +140,7 @@ RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ chmod +x bazel-${BAZEL_7_1_2}-linux-x86_64 && \ mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel -RUN usermod -aG docker conan +RUN usermod -aG docker conan && newgrp docker USER conan WORKDIR $HOME From abc865e687e40cf0aa34dbfb86907e48118604b8 Mon Sep 17 00:00:00 2001 From: czoido Date: Thu, 26 Sep 2024 18:38:11 +0200 Subject: [PATCH 48/54] test --- test/functional/command/runner_test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/functional/command/runner_test.py b/test/functional/command/runner_test.py index 1d41a7ecc52..d2bc8d31eb4 100644 --- a/test/functional/command/runner_test.py +++ b/test/functional/command/runner_test.py @@ -9,10 +9,7 @@ def docker_skip(test_image=None): try: - try: - docker_client = docker.from_env() - except: - docker_client = docker.DockerClient(base_url=f'unix://{os.path.expanduser("~")}/.rd/docker.sock', version='auto') # Rancher + docker_client = docker.from_env() if test_image: docker_client.images.pull(test_image) except docker.errors.DockerException as e: From c7e68b498534856f0b67839ec7ff09bf373d38e8 Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 27 Sep 2024 08:04:03 +0200 Subject: [PATCH 49/54] wip --- .github/workflows/linux-tests.yml | 77 ++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 16 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 27be1704721..317e70b9838 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -56,11 +56,11 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest - options: --user conan --volume /var/run/docker.sock:/var/run/docker.sock + options: --user conan strategy: matrix: - python-version: ['3.12.3']#, '3.9.2', '3.8.6', '3.6.15'] - test-type: ['unittests']#, 'integration', 'functional'] + python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] + test-type: ['unittests', 'integration', 'functional'] name: Python ${{ matrix.python-version }} - ${{ matrix.test-type }} steps: - name: Checkout code @@ -85,19 +85,64 @@ jobs: pip install -r conans/requirements_server.txt pip install meson - - name: Check Docker socket permissions - run: | - ls -l /var/run/docker.sock - id - - name: Run tests shell: bash run: | - pytest -m docker_runner -rs -# if [ "${{ matrix.test-type }}" == "unittests" ]; then -# pytest test/unittests --durations=20 -n 4 -rs -# elif [ "${{ matrix.test-type }}" == "integration" ]; then -# pytest test/integration --durations=20 -n 4 -rs -# elif [ "${{ matrix.test-type }}" == "functional" ]; then -# pytest test/functional --durations=20 -n 4 -rs -# fi + if [ "${{ matrix.test-type }}" == "unittests" ]; then + pytest test/unittests --durations=20 -n 4 -rs + elif [ "${{ matrix.test-type }}" == "integration" ]; then + pytest test/integration --durations=20 -n 4 -rs + elif [ "${{ matrix.test-type }}" == "functional" ]; then + pytest test/functional --durations=20 -n 4 -rs + fi + + linux_docker_tests: + needs: build_container + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.12.3'] + name: Docker Runner Tests - Python ${{ matrix.python-version }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r conans/requirements.txt + pip install -r conans/requirements_dev.txt + pip install -r conans/requirements_server.txt + pip install meson + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull Docker image + run: docker pull ghcr.io/${{ github.repository_owner }}/conan-tests:latest + + - name: Run tests inside Docker container + run: | + docker run --rm \ + -v "${{ github.workspace }}":/workspace \ + -w /workspace \ + ghcr.io/${{ github.repository_owner }}/conan-tests:latest \ + bash -c " + pyenv global ${{ matrix.python-version }} && \ + python --version && \ + pip install --upgrade pip && \ + pip install -r conans/requirements.txt && \ + pip install -r conans/requirements_dev.txt && \ + pip install -r conans/requirements_server.txt && \ + pip install meson && \ + pytest -m docker_runner --durations=20 -n 4 -rs + " From ffb7063f291022e157400eb50df13ac84f92725f Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 27 Sep 2024 08:20:30 +0200 Subject: [PATCH 50/54] wip --- .github/workflows/linux-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 317e70b9838..df128185794 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -134,6 +134,7 @@ jobs: run: | docker run --rm \ -v "${{ github.workspace }}":/workspace \ + -v /var/run/docker.sock:/var/run/docker.sock \ -w /workspace \ ghcr.io/${{ github.repository_owner }}/conan-tests:latest \ bash -c " From b6e46d8262aeef3d806bab0dfc766d35c301fd0e Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 27 Sep 2024 08:33:38 +0200 Subject: [PATCH 51/54] wip --- .github/workflows/linux-tests.yml | 37 +++++++++++++++++++------------ ci/conan-tests | 6 ++++- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index df128185794..bfcad88b317 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -40,10 +40,20 @@ jobs: if: steps.cache_docker_image.outputs.cache-hit == 'true' run: docker load -i docker_image_cache/conan-tests.tar + - name: Get Docker group GID + if: steps.cache_docker_image.outputs.cache-hit != 'true' + id: docker_gid + run: | + DOCKER_GID=$(getent group docker | cut -d: -f3) + echo "docker_gid=$DOCKER_GID" >> $GITHUB_OUTPUT + - name: Build Docker image (if not cached) if: steps.cache_docker_image.outputs.cache-hit != 'true' run: | - docker build -t ghcr.io/${{ github.repository_owner }}/conan-tests:latest -f ./ci/conan-tests . + docker build \ + --build-arg DOCKER_GID=${{ steps.docker_gid.outputs.docker_gid }} \ + -t ghcr.io/${{ github.repository_owner }}/conan-tests:latest \ + -f ./ci/conan-tests . mkdir -p docker_image_cache docker save ghcr.io/${{ github.repository_owner }}/conan-tests:latest -o docker_image_cache/conan-tests.tar @@ -107,19 +117,6 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - pip install --upgrade pip - pip install -r conans/requirements.txt - pip install -r conans/requirements_dev.txt - pip install -r conans/requirements_server.txt - pip install meson - - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: @@ -130,6 +127,18 @@ jobs: - name: Pull Docker image run: docker pull ghcr.io/${{ github.repository_owner }}/conan-tests:latest + - name: Check Docker socket permissions inside container + run: | + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + ghcr.io/${{ github.repository_owner }}/conan-tests:latest \ + bash -c " + ls -l /var/run/docker.sock + id + groups conan + getent group docker + " + - name: Run tests inside Docker container run: | docker run --rm \ diff --git a/ci/conan-tests b/ci/conan-tests index eb2ffc957c9..218be27efdd 100644 --- a/ci/conan-tests +++ b/ci/conan-tests @@ -2,6 +2,8 @@ FROM ubuntu:24.04 LABEL maintainer="Conan.io " +ARG DOCKER_GID=999 + ENV DEBIAN_FRONTEND=noninteractive ENV PY36=3.6.15 \ @@ -140,7 +142,9 @@ RUN mkdir -p /usr/share/bazel-$BAZEL_6_3_2/bin && \ chmod +x bazel-${BAZEL_7_1_2}-linux-x86_64 && \ mv bazel-${BAZEL_7_1_2}-linux-x86_64 /usr/share/bazel-$BAZEL_7_1_2/bin/bazel -RUN usermod -aG docker conan && newgrp docker +RUN groupadd -g $DOCKER_GID docker || groupmod -g $DOCKER_GID docker + +RUN usermod -aG docker conan USER conan WORKDIR $HOME From 00cfa058b81467287257c46df643ea73959ac51c Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 27 Sep 2024 09:51:41 +0200 Subject: [PATCH 52/54] wip --- .github/workflows/linux-tests.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index bfcad88b317..0ccd224bdce 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -142,9 +142,7 @@ jobs: - name: Run tests inside Docker container run: | docker run --rm \ - -v "${{ github.workspace }}":/workspace \ -v /var/run/docker.sock:/var/run/docker.sock \ - -w /workspace \ ghcr.io/${{ github.repository_owner }}/conan-tests:latest \ bash -c " pyenv global ${{ matrix.python-version }} && \ @@ -154,5 +152,5 @@ jobs: pip install -r conans/requirements_dev.txt && \ pip install -r conans/requirements_server.txt && \ pip install meson && \ - pytest -m docker_runner --durations=20 -n 4 -rs + pytest -m docker_runner --durations=20 -rs " From 5e41ae1c2142736b48e0bcae8e4957a5f408806b Mon Sep 17 00:00:00 2001 From: czoido Date: Fri, 27 Sep 2024 10:01:12 +0200 Subject: [PATCH 53/54] wip --- .github/workflows/linux-tests.yml | 38 +++++++++++-------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 0ccd224bdce..299f06933eb 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -109,6 +109,9 @@ jobs: linux_docker_tests: needs: build_container runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest + options: --user conan -v /var/run/docker.sock:/var/run/docker.sock strategy: matrix: python-version: ['3.12.3'] @@ -127,30 +130,15 @@ jobs: - name: Pull Docker image run: docker pull ghcr.io/${{ github.repository_owner }}/conan-tests:latest - - name: Check Docker socket permissions inside container + - name: Install dependencies run: | - docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - ghcr.io/${{ github.repository_owner }}/conan-tests:latest \ - bash -c " - ls -l /var/run/docker.sock - id - groups conan - getent group docker - " - - - name: Run tests inside Docker container + pip install --upgrade pip + pip install -r conans/requirements.txt + pip install -r conans/requirements_dev.txt + pip install -r conans/requirements_server.txt + pip install meson + + - name: Run tests + shell: bash run: | - docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - ghcr.io/${{ github.repository_owner }}/conan-tests:latest \ - bash -c " - pyenv global ${{ matrix.python-version }} && \ - python --version && \ - pip install --upgrade pip && \ - pip install -r conans/requirements.txt && \ - pip install -r conans/requirements_dev.txt && \ - pip install -r conans/requirements_server.txt && \ - pip install meson && \ - pytest -m docker_runner --durations=20 -rs - " + pytest -m docker_runner --durations=20 -rs From 31c7f450506b4e65d10f757862e510dfe848f57b Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 1 Oct 2024 14:53:49 +0200 Subject: [PATCH 54/54] wip --- .github/workflows/linux-tests.yml | 88 +++++++++++++++---------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index 299f06933eb..43f836708b2 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -61,50 +61,50 @@ jobs: if: steps.cache_docker_image.outputs.cache-hit != 'true' run: docker push ghcr.io/${{ github.repository_owner }}/conan-tests:latest - linux: - needs: build_container - runs-on: ubuntu-latest - container: - image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest - options: --user conan - strategy: - matrix: - python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] - test-type: ['unittests', 'integration', 'functional'] - name: Python ${{ matrix.python-version }} - ${{ matrix.test-type }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - run: | - pyenv global ${{ matrix.python-version }} - python --version - - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} - - - name: Install dependencies - run: | - pip install --upgrade pip - pip install -r conans/requirements.txt - pip install -r conans/requirements_dev.txt - pip install -r conans/requirements_server.txt - pip install meson - - - name: Run tests - shell: bash - run: | - if [ "${{ matrix.test-type }}" == "unittests" ]; then - pytest test/unittests --durations=20 -n 4 -rs - elif [ "${{ matrix.test-type }}" == "integration" ]; then - pytest test/integration --durations=20 -n 4 -rs - elif [ "${{ matrix.test-type }}" == "functional" ]; then - pytest test/functional --durations=20 -n 4 -rs - fi + # linux: + # needs: build_container + # runs-on: ubuntu-latest + # container: + # image: ghcr.io/${{ github.repository_owner }}/conan-tests:latest + # options: --user conan + # strategy: + # matrix: + # python-version: ['3.12.3', '3.9.2', '3.8.6', '3.6.15'] + # test-type: ['unittests', 'integration', 'functional'] + # name: Python ${{ matrix.python-version }} - ${{ matrix.test-type }} + # steps: + # - name: Checkout code + # uses: actions/checkout@v3 + + # - name: Set up Python ${{ matrix.python-version }} + # run: | + # pyenv global ${{ matrix.python-version }} + # python --version + + # - name: Cache pip + # uses: actions/cache@v3 + # with: + # path: ~/.cache/pip + # key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }} + + # - name: Install dependencies + # run: | + # pip install --upgrade pip + # pip install -r conans/requirements.txt + # pip install -r conans/requirements_dev.txt + # pip install -r conans/requirements_server.txt + # pip install meson + + # - name: Run tests + # shell: bash + # run: | + # if [ "${{ matrix.test-type }}" == "unittests" ]; then + # pytest test/unittests --durations=20 -n 4 -rs + # elif [ "${{ matrix.test-type }}" == "integration" ]; then + # pytest test/integration --durations=20 -n 4 -rs + # elif [ "${{ matrix.test-type }}" == "functional" ]; then + # pytest test/functional --durations=20 -n 4 -rs + # fi linux_docker_tests: needs: build_container