From 6b486f3f75ca5707cff0d13dd8614e97b2fb06c2 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Wed, 1 May 2024 02:48:01 +0200 Subject: [PATCH] Run tests as separate jobs (#813) --- .../workflows/test_build_docker_image.yaml | 62 +++++ .github/workflows/test_conda_store.yaml | 84 ++++++ .../test_conda_store_server_integration.yaml | 82 ++++++ .../test_conda_store_server_unit.yaml | 79 ++++++ .github/workflows/tests.yaml | 247 ------------------ .../tests/user_journeys/utils/api_utils.py | 2 +- tests/test_api.py | 43 ++- 7 files changed, 339 insertions(+), 260 deletions(-) create mode 100644 .github/workflows/test_build_docker_image.yaml create mode 100644 .github/workflows/test_conda_store.yaml create mode 100644 .github/workflows/test_conda_store_server_integration.yaml create mode 100644 .github/workflows/test_conda_store_server_unit.yaml delete mode 100644 .github/workflows/tests.yaml diff --git a/.github/workflows/test_build_docker_image.yaml b/.github/workflows/test_build_docker_image.yaml new file mode 100644 index 000000000..2fc1fb943 --- /dev/null +++ b/.github/workflows/test_build_docker_image.yaml @@ -0,0 +1,62 @@ +name: "Test build Docker image" + +env: + DEFAULT_PYTHON_VERSION: "3.10" + FORCE_COLOR: "1" # Make tools pretty. + +on: + pull_request: + push: + branches: + - main + + +# ensuring only one instance is running at a given time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-docker-image: + name: "Build docker images" + runs-on: ubuntu-latest + strategy: + matrix: + docker-image: + - conda-store + - conda-store-server + steps: + - name: "Checkout Repository ๐Ÿ›Ž" + uses: actions/checkout@v4 + + - name: "Set up Docker Buildx" + uses: docker/setup-buildx-action@v3 + + - name: "Lint Dockerfiles ๐Ÿ”" + uses: jbergstroem/hadolint-gh-action@v1 + with: + dockerfile: ${{ matrix.docker-image }}/Dockerfile + output_format: tty + error_level: 0 + + - name: "Docker Meta" + id: meta + uses: docker/metadata-action@v5 + with: + images: | + quansight/${{ matrix.docker-image }} + tags: | + type=sha + + - name: "Build Docker images" + uses: docker/build-push-action@v5 + with: + context: "${{ matrix.docker-image }}" + file: "${{ matrix.docker-image }}/Dockerfile" + tags: | + ${{ steps.meta.outputs.tags }} + target: "dev" + push: false + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/test_conda_store.yaml b/.github/workflows/test_conda_store.yaml new file mode 100644 index 000000000..41d16bca0 --- /dev/null +++ b/.github/workflows/test_conda_store.yaml @@ -0,0 +1,84 @@ +name: "Test conda-store" + +env: + DEFAULT_PYTHON_VERSION: "3.10" + FORCE_COLOR: "1" # Make tools pretty. + +on: + pull_request: + push: + branches: + - main + + +# ensuring only one instance is running at a given time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test-conda-store: + name: "integration-test conda-store" + runs-on: ubuntu-latest + defaults: + run: + working-directory: conda-store + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - name: "Checkout Repository ๐Ÿ›Ž" + uses: actions/checkout@v4 + + - name: "Set up Python ๐Ÿ" + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: "Install Dependencies ๐Ÿ“ฆ" + run: | + pip install hatch + sudo apt install wait-for-it -y + + - name: "Linting Checks ๐Ÿงน" + run: | + hatch env run -e lint lint + + - name: "Build package ๐Ÿ“ฆ" + run: | + hatch build + + - name: "Deploy docker-compose" + run: | + docker-compose up -d + docker ps + + wait-for-it localhost:5432 # postgresql + wait-for-it localhost:9000 # minio + wait-for-it localhost:8080 # conda-store-server + + - name: "Install conda-store for tests ๐Ÿ“ฆ" + run: | + pip install . + + - name: "Run basic tests - not authenticated" + run: | + sleep 20 + ./tests/unauthenticated-tests.sh + + - name: "Run basic tests - authenticated" + run: | + ./tests/authenticated-tests.sh + + - name: "Test shebang" + run: | + export CONDA_STORE_URL=http://localhost:8080/conda-store + export CONDA_STORE_AUTH=basic + export CONDA_STORE_USERNAME=username + export CONDA_STORE_PASSWORD=password + ./tests/shebang.sh + + - name: "Get Docker logs ๐Ÿ”" + if: ${{ failure() }} + run: | + docker-compose logs diff --git a/.github/workflows/test_conda_store_server_integration.yaml b/.github/workflows/test_conda_store_server_integration.yaml new file mode 100644 index 000000000..603701a72 --- /dev/null +++ b/.github/workflows/test_conda_store_server_integration.yaml @@ -0,0 +1,82 @@ +name: "Test conda-store-server (integration)" + +env: + DEFAULT_PYTHON_VERSION: "3.10" + FORCE_COLOR: "1" # Make tools pretty. + +on: + pull_request: + push: + branches: + - main + + +# ensuring only one instance is running at a given time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + integration-test-conda-store-server: + name: "integration-test conda-store-server" + runs-on: ubuntu-latest + strategy: + matrix: + test-type: ["playwright", "integration", "user-journey"] + defaults: + run: + shell: bash -el {0} + working-directory: conda-store-server + steps: + - name: "Checkout Repository ๐Ÿ›Ž" + uses: actions/checkout@v4 + + - name: "Set up env ๐Ÿ" + uses: conda-incubator/setup-miniconda@v3 + with: + environment-file: conda-store-server/environment-dev.yaml + miniforge-version: latest + + - name: "Install build dependencies ๐Ÿ“ฆ" + run: | + pip install hatch + sudo apt install wait-for-it -y + + - name: "Deploy docker-compose" + run: | + docker-compose up -d + docker ps + + wait-for-it localhost:5432 # postgresql + wait-for-it localhost:9000 # minio + wait-for-it localhost:8080 # conda-store-server + sleep 10 # give the server more time to initialize + + - name: "Run Playwright tests ๐ŸŽญ" + run: | + playwright install + pytest --video on ../tests/test_playwright.py + if: matrix.test-type == 'playwright' + + - name: "Upload test results ๐Ÿ“ค" + uses: actions/upload-artifact@v4 + if: ${{ always() }} && matrix.test-type == 'playwright' + with: + name: playwright-tests + path: conda-store-server/test-results + + - name: "Run integration tests โœ…" + run: | + export PYTHONPATH=$PYTHONPATH:$PWD + pytest ../tests/test_api.py ../tests/test_metrics.py + if: matrix.test-type == 'integration' + + - name: "Run user journey tests โœ…" + run: | + pytest -m "user_journey" + if: matrix.test-type == 'user-journey' + + - name: "Get Docker logs ๐Ÿ”" + if: ${{ failure() }} + run: | + docker-compose logs diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml new file mode 100644 index 000000000..07e72bac6 --- /dev/null +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -0,0 +1,79 @@ +name: "Test conda-store-server (unit)" + +env: + DEFAULT_PYTHON_VERSION: "3.10" + FORCE_COLOR: "1" # Make tools pretty. + +on: + pull_request: + push: + branches: + - main + + +# ensuring only one instance is running at a given time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test-conda-store-server: + name: "unit-test conda-store-server" + strategy: + matrix: + os: ["ubuntu", "macos", "windows"] + include: + - os: ubuntu + environment-file: conda-store-server/environment-dev.yaml + - os: macos + environment-file: conda-store-server/environment-macos-dev.yaml + - os: windows + environment-file: conda-store-server/environment-windows-dev.yaml + runs-on: ${{ matrix.os }}-latest + defaults: + run: + shell: bash -el {0} + working-directory: conda-store-server + steps: + - name: "Checkout Repository ๐Ÿ›Ž" + uses: actions/checkout@v4 + + - name: "Set up env ${{ matrix.os }} ๐Ÿ" + uses: conda-incubator/setup-miniconda@v2 + with: + environment-file: ${{ matrix.environment-file }} + miniforge-version: latest + + # This fixes a "DLL not found" issue importing ctypes from the hatch env + - name: "Reinstall Python 3.10 on Windows runner" + uses: nick-fields/retry@v2.8.3 + with: + timeout_minutes: 9999 + max_attempts: 6 + command: + conda install --channel=conda-forge --quiet --yes python=${{ matrix.python }} + if: matrix.os == 'windows' + + - name: "Linting Checks ๐Ÿงน" + run: | + hatch env run -e lint lint + + - name: "Build package ๐Ÿ“ฆ" + run: | + hatch build + + - name: "Unit tests โœ…" + run: | + pytest -m "not extended_prefix and not user_journey" tests + + # https://github.com/actions/runner-images/issues/1052 + - name: "Windows extended prefix unit tests โœ…" + shell: pwsh + run: | + Set-ItemProperty "HKLM:\System\CurrentControlSet\Control\FileSystem" ` + -Name "LongPathsEnabled" ` + -Value 0 ` + -Type DWord + (Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled + pytest -m "extended_prefix" tests + if: matrix.os == 'windows' diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml deleted file mode 100644 index 889d1b881..000000000 --- a/.github/workflows/tests.yaml +++ /dev/null @@ -1,247 +0,0 @@ -name: Tests - -env: - DEFAULT_PYTHON_VERSION: "3.10" - FORCE_COLOR: "1" # Make tools pretty. - -on: - pull_request: - push: - branches: - - main - - -# ensuring only one instance is running at a given time -concurrency: - group: ${{ github.ref }} - cancel-in-progress: true - -jobs: - test-conda-store-server: - name: "unit-test conda-store-server" - strategy: - matrix: - os: ["ubuntu", "macos", "windows"] - include: - - os: ubuntu - environment-file: conda-store-server/environment-dev.yaml - - os: macos - environment-file: conda-store-server/environment-macos-dev.yaml - - os: windows - environment-file: conda-store-server/environment-windows-dev.yaml - runs-on: ${{ matrix.os }}-latest - defaults: - run: - shell: bash -el {0} - working-directory: conda-store-server - steps: - - name: "Checkout Repository ๐Ÿ›Ž" - uses: actions/checkout@v4 - - - name: "Set up env ${{ matrix.os }} ๐Ÿ" - uses: conda-incubator/setup-miniconda@v2 - with: - environment-file: ${{ matrix.environment-file }} - miniforge-version: latest - - # This fixes a "DLL not found" issue importing ctypes from the hatch env - - name: "Reinstall Python 3.10 on Windows runner" - uses: nick-fields/retry@v2.8.3 - with: - timeout_minutes: 9999 - max_attempts: 6 - command: - conda install --channel=conda-forge --quiet --yes python=${{ matrix.python }} - if: matrix.os == 'windows' - - - name: "Linting Checks ๐Ÿงน" - run: | - hatch env run -e lint lint - - - name: "Build package ๐Ÿ“ฆ" - run: | - hatch build - - - name: "Unit tests โœ…" - run: | - pytest -m "not extended_prefix and not user_journey" tests - - # https://github.com/actions/runner-images/issues/1052 - - name: "Windows extended prefix unit tests โœ…" - shell: pwsh - run: | - Set-ItemProperty "HKLM:\System\CurrentControlSet\Control\FileSystem" ` - -Name "LongPathsEnabled" ` - -Value 0 ` - -Type DWord - (Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled - pytest -m "extended_prefix" tests - if: matrix.os == 'windows' - - - integration-test-conda-store-server: - name: "integration-test conda-store-server" - runs-on: ubuntu-latest - defaults: - run: - shell: bash -el {0} - working-directory: conda-store-server - steps: - - name: "Checkout Repository ๐Ÿ›Ž" - uses: actions/checkout@v4 - - - name: "Set up env ๐Ÿ" - uses: conda-incubator/setup-miniconda@v3 - with: - environment-file: conda-store-server/environment-dev.yaml - miniforge-version: latest - - - name: "Install build dependencies ๐Ÿ“ฆ" - run: | - pip install hatch - sudo apt install wait-for-it -y - - - name: "Deploy docker-compose" - run: | - docker-compose up -d - docker ps - - wait-for-it localhost:5432 # postgresql - wait-for-it localhost:9000 # minio - wait-for-it localhost:8080 # conda-store-server - - - name: "Run Playwright tests ๐ŸŽญ" - run: | - playwright install - pytest --video on ../tests/test_playwright.py - - - name: "Upload test results ๐Ÿ“ค" - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: playwright-tests - path: conda-store-server/test-results - - - name: "Run integration tests โœ…" - run: | - export PYTHONPATH=$PYTHONPATH:$PWD - pytest ../tests/test_api.py ../tests/test_metrics.py - - - name: "Run user journey tests โœ…" - run: | - pytest -m "user_journey" - - - name: "Get Docker logs ๐Ÿ”" - if: ${{ failure() }} - run: | - docker-compose logs - - test-conda-store: - name: "integration-test conda-store" - runs-on: ubuntu-latest - defaults: - run: - working-directory: conda-store - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - steps: - - name: "Checkout Repository ๐Ÿ›Ž" - uses: actions/checkout@v4 - - - name: "Set up Python ๐Ÿ" - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: "Install Dependencies ๐Ÿ“ฆ" - run: | - pip install hatch - sudo apt install wait-for-it -y - - - name: "Linting Checks ๐Ÿงน" - run: | - hatch env run -e lint lint - - - name: "Build package ๐Ÿ“ฆ" - run: | - hatch build - - - name: "Deploy docker-compose" - run: | - docker-compose up -d - docker ps - - wait-for-it localhost:5432 # postgresql - wait-for-it localhost:9000 # minio - wait-for-it localhost:8080 # conda-store-server - - - name: "Install conda-store for tests ๐Ÿ“ฆ" - run: | - pip install . - - - name: "Run basic tests - not authenticated" - run: | - sleep 20 - ./tests/unauthenticated-tests.sh - - - name: "Run basic tests - authenticated" - run: | - ./tests/authenticated-tests.sh - - - name: "Test shebang" - run: | - export CONDA_STORE_URL=http://localhost:8080/conda-store - export CONDA_STORE_AUTH=basic - export CONDA_STORE_USERNAME=username - export CONDA_STORE_PASSWORD=password - ./tests/shebang.sh - - - name: "Get Docker logs ๐Ÿ”" - if: ${{ failure() }} - run: | - docker-compose logs - - build-docker-image: - name: "Build docker images" - runs-on: ubuntu-latest - strategy: - matrix: - docker-image: - - conda-store - - conda-store-server - steps: - - name: "Checkout Repository ๐Ÿ›Ž" - uses: actions/checkout@v4 - - - name: "Set up Docker Buildx" - uses: docker/setup-buildx-action@v3 - - - name: "Lint Dockerfiles ๐Ÿ”" - uses: jbergstroem/hadolint-gh-action@v1 - with: - dockerfile: ${{ matrix.docker-image }}/Dockerfile - output_format: tty - error_level: 0 - - - name: "Docker Meta" - id: meta - uses: docker/metadata-action@v5 - with: - images: | - quansight/${{ matrix.docker-image }} - tags: | - type=sha - - - name: "Build Docker images" - uses: docker/build-push-action@v5 - with: - context: "${{ matrix.docker-image }}" - file: "${{ matrix.docker-image }}/Dockerfile" - tags: | - ${{ steps.meta.outputs.tags }} - target: "dev" - push: false - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/conda-store-server/tests/user_journeys/utils/api_utils.py b/conda-store-server/tests/user_journeys/utils/api_utils.py index 773a99f88..9aa0a18c4 100644 --- a/conda-store-server/tests/user_journeys/utils/api_utils.py +++ b/conda-store-server/tests/user_journeys/utils/api_utils.py @@ -179,7 +179,7 @@ def check_status(): BuildStatus.COMPLETED, ] - wait_for_condition(check_status, timeout=120, interval=1) + wait_for_condition(check_status, timeout=240, interval=1) return self._make_request(f"api/v1/build/{build_id}/") def delete_environment( diff --git a/tests/test_api.py b/tests/test_api.py index c2d758056..23a2060b6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -275,14 +275,25 @@ def test_api_get_build_one_unauth(testclient): def test_api_get_build_one_auth(testclient): testclient.login() - response = testclient.get("api/v1/build/1") - response.raise_for_status() + success = False + for _ in range(50): + response = testclient.get("api/v1/build/1") + response.raise_for_status() - r = schema.APIGetBuild.parse_obj(response.json()) - assert r.status == schema.APIStatus.OK - assert r.data.id == 1 - assert r.data.specification.name == "python-flask-env" - assert r.data.status == schema.BuildStatus.COMPLETED.value + r = schema.APIGetBuild.parse_obj(response.json()) + assert r.status == schema.APIStatus.OK + assert r.data.id == 1 + assert r.data.specification.name == "python-flask-env" + if r.data.status in [ + schema.BuildStatus.QUEUED.value, + schema.BuildStatus.BUILDING.value, + ]: + time.sleep(10) + continue + assert r.data.status == schema.BuildStatus.COMPLETED.value + success = True + break + assert success def test_api_get_build_one_unauth_packages(testclient): @@ -295,12 +306,20 @@ def test_api_get_build_one_unauth_packages(testclient): def test_api_get_build_one_auth_packages(testclient): testclient.login() - response = testclient.get("api/v1/build/1/packages?size=5") - response.raise_for_status() + success = False + for _ in range(50): + response = testclient.get("api/v1/build/1/packages?size=5") + response.raise_for_status() - r = schema.APIListCondaPackage.parse_obj(response.json()) - assert r.status == schema.APIStatus.OK - assert len(r.data) == 5 + r = schema.APIListCondaPackage.parse_obj(response.json()) + assert r.status == schema.APIStatus.OK + if len(r.data) == 0: + time.sleep(10) + continue + assert len(r.data) == 5 + success = True + break + assert success def test_api_get_build_auth_packages_no_exist(testclient):