From 75ef72cfe2ac61c643576c37dce673cfee211375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Tue, 16 Jul 2024 14:21:03 +0000 Subject: [PATCH] Add workflow for linting and testing (#179) Add workflow for linting, static type checking, and testing. - Check each package individually - Check ops and integration tests - Check docstrings for `vibe_core` --- .github/workflows/cluster-build.yml | 11 ++- .github/workflows/lint-test.yml | 129 ++++++++++++++++++++++++++++ scripts/export_sam_models.py | 11 +++ 3 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/lint-test.yml diff --git a/.github/workflows/cluster-build.yml b/.github/workflows/cluster-build.yml index 5ab517e8..4b84cf20 100644 --- a/.github/workflows/cluster-build.yml +++ b/.github/workflows/cluster-build.yml @@ -1,6 +1,15 @@ name: Build FarmVibes.AI cluster run-name: Cluster build and helloworld test -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: + - dev + - main + pull_request: + branches: + - dev + - main + workflow_dispatch: env: FARMVIBES_AI_SKIP_DOCKER_FREE_SPACE_CHECK: yes jobs: diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml new file mode 100644 index 00000000..0698d743 --- /dev/null +++ b/.github/workflows/lint-test.yml @@ -0,0 +1,129 @@ +name: Linting and testing +on: + push: + branches: + - dev + - main + pull_request: + branches: + - dev + - main + workflow_dispatch: + +env: + PYRIGHT_PYTHON_FORCE_VERSION: 1.1.268 + +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v3 + with: + python-version: '3.11' + - name: Install dependencies + run: | + pip install wheel setuptools + - name: Build packages + run: | + for pkg in vibe_core vibe_common vibe_agent vibe_server vibe_dev; do cd src/$pkg && python setup.py bdist_wheel --dist-dir ../../dist; cd ../../; done + - name: Save packages + uses: actions/upload-artifact@v4 + with: + name: packages + path: dist + test: + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + package-to-test: [vibe_core, vibe_common, vibe_server, vibe_agent] + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v3 + with: + python-version: '3.11' + - name: Retrieve packages + uses: actions/download-artifact@v4 + with: + name: packages + path: dist + - name: Install dependencies + run: | + pip install pyright ruff + - name: Install package + run: | + pip install ${{ matrix.package-to-test }}[test] --find-links dist + - name: Lint with ruff + run: | + ruff check ./src/${{ matrix.package-to-test }} --config ./.ruff.toml + - name: Type checking with pyright + run: | + pyright ./src/${{ matrix.package-to-test }} + - name: Test with pytest + run: | + pip install vibe_dev --find-links dist + pytest ./src/${{ matrix.package-to-test}} -v --junitxml=junit/test-results.xml --cov=. --cov-report=xml + + ops-test: + runs-on: ubuntu-latest + container: + image: mcr.microsoft.com/farmai/terravibes/worker-base:12380 + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + pip install pyright ruff + - name: Setup op resources + run: | + mkdir -p /opt/terravibes/ops + ln -sf $(pwd)/op_resources /opt/terravibes/ops/resources + mkdir /app + ln -sf $(pwd)/ops /app/ops + ln -sf $(pwd)/workflows /app/workflows + - name: Install packages + run: | + pip install ./src/vibe_core + pip install ./src/vibe_common + pip install ./src/vibe_agent + pip install ./src/vibe_server + pip install ./src/vibe_lib + pip install ./src/vibe_dev + - name: Linting ops + run: | + ruff check ./ops --config ./.ruff.toml + - name: Type checking ops + run: | + pyright ./ops + - name: Get SAM model + run: | + pip install git+https://github.com/facebookresearch/segment-anything.git + mkdir -p /mnt/onnx_resources + python -c "from scripts.export_sam_models import dev; dev()" + - name: Run integration tests + run: | + pytest ./src/vibe_lib ./ops ./src/tests -v --durations=0 --full-trace --junitxml=test-output.xml + check-docstrings: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v3 + with: + python-version: '3.11' + - name: Install ruff + run: | + pip install ruff + - name: "Check docstrings for vibe_core" + run: | + ruff check --select D,D401 --ignore D105 --force-exclude --exclude src/vibe_core/vibe_core/farmvibes_ai_hello_world.py --config "lint.pydocstyle.convention = 'google'" src/vibe_core/vibe_core/*.py + - name: "Check docstrings for vibe_core/data" + run: | + ruff check --select D,D401 --ignore D105 --config "lint.pydocstyle.convention = 'google'" src/vibe_core/vibe_core/data/*.py \ No newline at end of file diff --git a/scripts/export_sam_models.py b/scripts/export_sam_models.py index dbd8f26c..2f2aaf4d 100644 --- a/scripts/export_sam_models.py +++ b/scripts/export_sam_models.py @@ -263,5 +263,16 @@ def main(): add_to_cluster(exported_paths, args.cluster) + +def dev(): + model_type = "vit_b" + out_path = "/mnt/onnx_resources/" + with TemporaryDirectory() as tmp_dir: + model_url = MODELS[model_type].url + downloaded_path = download_file(model_url, os.path.join(tmp_dir, f"{model_type}.pth")) + export_model(model_type, downloaded_path, out_path) + + + if __name__ == "__main__": main()