From 1c5dd7f819ef12a417cb62bcaa49c38ad588cfc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Wed, 31 Jul 2024 13:48:54 +0000 Subject: [PATCH 1/5] Add local cluster integration tests --- .github/workflows/cluster-build.yml | 105 ---------------------------- .github/workflows/lint-test.yml | 64 ++++++++++++++++- scripts/local-k8s-diagnostics.sh | 43 ++++++++++++ 3 files changed, 106 insertions(+), 106 deletions(-) delete mode 100644 .github/workflows/cluster-build.yml create mode 100644 scripts/local-k8s-diagnostics.sh diff --git a/.github/workflows/cluster-build.yml b/.github/workflows/cluster-build.yml deleted file mode 100644 index 4b84cf20..00000000 --- a/.github/workflows/cluster-build.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: Build FarmVibes.AI cluster -run-name: Cluster build and helloworld test -on: - push: - branches: - - dev - - main - pull_request: - branches: - - dev - - main - workflow_dispatch: -env: - FARMVIBES_AI_SKIP_DOCKER_FREE_SPACE_CHECK: yes -jobs: - build: - name: Build and test - runs-on: ubuntu-latest - steps: - - name: Free space before cleanup - shell: bash - run: | - echo "Memory and swap:" - free -h - echo - echo "Available storage:" - df -h - echo - - name: Remove unused software - shell: bash - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -fr /usr/local/lib/android - sudo rm -fr /opt/ghc - - name: Free space after cleanup - shell: bash - run: | - echo "Memory and swap:" - free -h - echo - echo "Available storage:" - df -h - echo - - - name: Update pip version - shell: bash - run: pip install --upgrade pip - - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Python requirements - shell: bash - run: pip install --user ./src/vibe_core - - - name: Actually build cluster - shell: bash - run: farmvibes-ai local setup - - - name: Cluster status - shell: bash - run: | - echo "Deployments:" - kubectl get deployments - kubectl describe deployments - echo "Pods:" - kubectl get pods - echo "Events:" - kubectl get events - - - name: Free space after cluster creation - shell: bash - run: | - echo "Memory and swap:" - free -h - echo - echo "Available storage:" - df -h - echo - - - name: Run Hello World workflow - shell: bash - run: python -m vibe_core.farmvibes_ai_hello_world 300 - - - name: Install tool to view helloworld output - shell: bash - run: yes | sudo apt install caca-utils || echo - - - name: Show helloworld output - shell: bash - run: find ~/.cache/farmvibes-ai/data -type f -name '*.tif' -exec img2txt {} \; 2> /dev/null - - - name: Restart cluster - shell: bash - run: farmvibes-ai local restart - - - name: Wait a bit before submitting a new run (as documented) - shell: bash - run: sleep 90 - - - name: Run Hello World workflow again - shell: bash - run: | - rm -fr ~/.cache/farmvibes-ai/data/{assets,stac} - python -m vibe_core.farmvibes_ai_hello_world 300 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 0698d743..e43e2a30 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -126,4 +126,66 @@ jobs: ruff check --select D,D401 --ignore D105 --force-exclude --exclude src/vibe_core/vibe_core/farmvibes_ai_hello_world.py --config "lint.pydocstyle.convention = 'google'" src/vibe_core/vibe_core/*.py - name: "Check docstrings for vibe_core/data" run: | - ruff check --select D,D401 --ignore D105 --config "lint.pydocstyle.convention = 'google'" src/vibe_core/vibe_core/data/*.py \ No newline at end of file + ruff check --select D,D401 --ignore D105 --config "lint.pydocstyle.convention = 'google'" src/vibe_core/vibe_core/data/*.py + local-integration-tests: + runs-on: ubuntu-latest + steps: + - name: Free space before cleanup + shell: bash + run: | + echo "Memory and swap:" + free -h + echo + echo "Available storage:" + df -h + echo + - name: Remove unused software + shell: bash + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -fr /usr/local/lib/android + sudo rm -fr /opt/ghc + - name: Free space after cleanup + shell: bash + run: | + echo "Memory and swap:" + free -h + echo + echo "Available storage:" + df -h + echo + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v3 + with: + python-version: '3.11' + - name: Install packages + run: | + pip install ./src/vibe_core + pip install ./src/vibe_lib + pip install ./src/vibe_dev + - name: Actually build cluster + run: farmvibes-ai local setup --auto-confirm + - name: Cluster status before building local images + run: | + bash ./scripts/local-k8s-diagnostics.sh + - name: Build images + run: | + WAIT_AT_THE_END=1 make local + - name: Cluster status after building local images + run: | + bash ./scripts/local-k8s-diagnostics.sh + - name: Free space after cluster creation + run: | + echo "Memory and swap:" + free -h + echo + echo "Available storage:" + df -h + echo + - name: Run integration tests + run: | + pytest ./src/tests_local_cluster/ -v --junitxml=junit/test-results.xml || (bash ./scripts/local-k8s-diagnostics.sh ; exit 1) + - name: Cluster status after running tests + run: | + bash ./scripts/local-k8s-diagnostics.sh \ No newline at end of file diff --git a/scripts/local-k8s-diagnostics.sh b/scripts/local-k8s-diagnostics.sh new file mode 100644 index 00000000..c6d18b0c --- /dev/null +++ b/scripts/local-k8s-diagnostics.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +PATH=$PATH:~/.config/farmvibes-ai + +echo "kubectl location:" +which kubectl + +echo "Cluster pods:" +kubectl get pods +# kubectl get pods -o yaml + +# echo "Docker images:" +# docker images + +# echo "REST API description:" +# kubectl describe deployment terravibes-rest-api + +# echo "Orchestrator description:" +# kubectl describe deployment terravibes-orchestrator + +# echo "Worker description:" +# kubectl describe deployment terravibes-worker + +# echo "Cache description:" +# kubectl describe deployment terravibes-cache + +# echo "REST API logs:" +# kubectl logs -l app=terravibes-rest-api --all-containers=true --tail=-1 + +# echo "Orchestrator logs:" +# kubectl logs -l app=terravibes-orchestrator --all-containers=true --tail=-1 + +# echo "Worker logs:" +# kubectl logs -l app=terravibes-worker --max-log-requests=8 --all-containers=true --tail=-1 + +# echo "Cache logs:" +# kubectl logs -l app=terravibes-cache --all-containers=true --tail=-1 + +# echo "Data Ops logs:" +# kubectl logs -l app=terravibes-data-ops --all-containers=true --tail=-1 + +# echo "Kubernetes logs:" +# docker ps | egrep 'k3d-farmvibes-ai-.*-0' | awk '{ print $1 }' | xargs docker logs From bd6e2e2c4b35631e18285006872c8dc9213a89f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Wed, 31 Jul 2024 14:23:51 +0000 Subject: [PATCH 2/5] Add step for simple workflow run --- .github/workflows/lint-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index e43e2a30..06727182 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -183,6 +183,8 @@ jobs: echo "Available storage:" df -h echo + - name: Run Hello World workflow + run: python -m vibe_core.farmvibes_ai_hello_world 300 - name: Run integration tests run: | pytest ./src/tests_local_cluster/ -v --junitxml=junit/test-results.xml || (bash ./scripts/local-k8s-diagnostics.sh ; exit 1) From 257bd2106ddccd33215ff64b6efe87fc889d2ab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Wed, 31 Jul 2024 15:48:47 +0000 Subject: [PATCH 3/5] Add all diagnostics back --- scripts/local-k8s-diagnostics.sh | 46 ++++++++++++++++---------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/scripts/local-k8s-diagnostics.sh b/scripts/local-k8s-diagnostics.sh index c6d18b0c..aca46057 100644 --- a/scripts/local-k8s-diagnostics.sh +++ b/scripts/local-k8s-diagnostics.sh @@ -7,37 +7,37 @@ which kubectl echo "Cluster pods:" kubectl get pods -# kubectl get pods -o yaml +kubectl get pods -o yaml -# echo "Docker images:" -# docker images +echo "Docker images:" +docker images -# echo "REST API description:" -# kubectl describe deployment terravibes-rest-api +echo "REST API description:" +kubectl describe deployment terravibes-rest-api -# echo "Orchestrator description:" -# kubectl describe deployment terravibes-orchestrator +echo "Orchestrator description:" +kubectl describe deployment terravibes-orchestrator -# echo "Worker description:" -# kubectl describe deployment terravibes-worker +echo "Worker description:" +kubectl describe deployment terravibes-worker -# echo "Cache description:" -# kubectl describe deployment terravibes-cache +echo "Cache description:" +kubectl describe deployment terravibes-cache -# echo "REST API logs:" -# kubectl logs -l app=terravibes-rest-api --all-containers=true --tail=-1 +echo "REST API logs:" +kubectl logs -l app=terravibes-rest-api --all-containers=true --tail=-1 -# echo "Orchestrator logs:" -# kubectl logs -l app=terravibes-orchestrator --all-containers=true --tail=-1 +echo "Orchestrator logs:" +kubectl logs -l app=terravibes-orchestrator --all-containers=true --tail=-1 -# echo "Worker logs:" -# kubectl logs -l app=terravibes-worker --max-log-requests=8 --all-containers=true --tail=-1 +echo "Worker logs:" +kubectl logs -l app=terravibes-worker --max-log-requests=8 --all-containers=true --tail=-1 -# echo "Cache logs:" -# kubectl logs -l app=terravibes-cache --all-containers=true --tail=-1 +echo "Cache logs:" +kubectl logs -l app=terravibes-cache --all-containers=true --tail=-1 -# echo "Data Ops logs:" -# kubectl logs -l app=terravibes-data-ops --all-containers=true --tail=-1 +echo "Data Ops logs:" +kubectl logs -l app=terravibes-data-ops --all-containers=true --tail=-1 -# echo "Kubernetes logs:" -# docker ps | egrep 'k3d-farmvibes-ai-.*-0' | awk '{ print $1 }' | xargs docker logs +echo "Kubernetes logs:" +docker ps | egrep 'k3d-farmvibes-ai-.*-0' | awk '{ print $1 }' | xargs docker logs From 548d8f00dbd87b15c3f5cd50a47a2f1842a46742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Wed, 31 Jul 2024 16:18:02 +0000 Subject: [PATCH 4/5] Update action to download LFS files --- .github/workflows/lint-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 06727182..c6a28940 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -155,6 +155,8 @@ jobs: df -h echo - uses: actions/checkout@v4 + with: + lfs: true - name: Set up Python 3.11 uses: actions/setup-python@v3 with: From 7dc61a748fe0333725dae9fcba3ea2ba34066209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Wed, 31 Jul 2024 17:08:58 +0000 Subject: [PATCH 5/5] Always run diagnostics at the end --- .github/workflows/lint-test.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index c6a28940..8fac7957 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -164,6 +164,9 @@ jobs: - name: Install packages run: | pip install ./src/vibe_core + pip install ./src/vibe_common + pip install ./src/vibe_agent + pip install ./src/vibe_server pip install ./src/vibe_lib pip install ./src/vibe_dev - name: Actually build cluster @@ -185,11 +188,10 @@ jobs: echo "Available storage:" df -h echo - - name: Run Hello World workflow - run: python -m vibe_core.farmvibes_ai_hello_world 300 - name: Run integration tests run: | - pytest ./src/tests_local_cluster/ -v --junitxml=junit/test-results.xml || (bash ./scripts/local-k8s-diagnostics.sh ; exit 1) + pytest ./src/tests_local_cluster/ -v --junitxml=junit/test-results.xml - name: Cluster status after running tests + if: always() run: | bash ./scripts/local-k8s-diagnostics.sh \ No newline at end of file