From 3c8513f0a79fdbc1b5ee5ed911ad9ea3efc5f0a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Thu, 1 Aug 2024 19:19:54 +0000 Subject: [PATCH 01/19] Add workflow to build base images --- .github/workflows/base-build.yml | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/base-build.yml diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml new file mode 100644 index 00000000..7cc141c7 --- /dev/null +++ b/.github/workflows/base-build.yml @@ -0,0 +1,64 @@ +name: Build base images +on: + workflow_dispatch: + pull_request: + branches: + - dev + +permissions: + id-token: write + contents: write + +jobs: + + build-and-push: + runs-on: ubuntu-latest + environment: build + strategy: + matrix: + include: + - dockerfile: Dockerfile-services-base + image-name: services-base + - dockerfile: Dockerfile-worker-base + image-name: worker-base + steps: + - uses: actions/checkout@v4 + - name: 'Az CLI login' + uses: azure/login@v1 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - name: 'Build and push image' + run: | + az acr login -n ${{ secrets.ACR_NAME }} + VERSION_TAG=${{ github.run_id }} + export IMAGE_TAG=${{ secrets.ACR_NAME }}.azurecr.io/unlisted/farmai/terravibes/${{ matrix.image-name }}:$VERSION_TAG + docker build . -f ./resources/docker/${{ matrix.dockerfile }} -t $IMAGE_TAG + docker push $IMAGE_TAG + + update-tags: + needs: build-and-push + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: 'Update tags' + run: | + TAG=${{ github.run_id }} + sed -i "s|\(\s\+image.*:\).*|\1${TAG}|" ./.github/workflows/lint-test.yml + sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-api_orchestrator + sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-cache + sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-worker + sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-devcontainer + - name: Commit changes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config --global user.email "farmvibesaicd@microsoft.com" + git config --global user.name "FarmVibes.AI Release Pipeline" + BRANCH=update-base-${{ github.run_id }} + git checkout -b $BRANCH + git add ./.github/workflows/lint-test.yml + git add ./resources/docker/ + git commit -m "Update base tag to latest image" + git push --set-upstream origin $BRANCH From 0067165f36e1691d62e81dcc19727bae5b4a64bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Thu, 1 Aug 2024 21:09:20 +0000 Subject: [PATCH 02/19] Try to fix permissions --- .github/workflows/base-build.yml | 10 +++------- .github/workflows/release-to-main.yml | 1 - 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index 7cc141c7..ba33e032 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -9,6 +9,7 @@ permissions: id-token: write contents: write + jobs: build-and-push: @@ -31,12 +32,7 @@ jobs: subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: 'Build and push image' run: | - az acr login -n ${{ secrets.ACR_NAME }} - VERSION_TAG=${{ github.run_id }} - export IMAGE_TAG=${{ secrets.ACR_NAME }}.azurecr.io/unlisted/farmai/terravibes/${{ matrix.image-name }}:$VERSION_TAG - docker build . -f ./resources/docker/${{ matrix.dockerfile }} -t $IMAGE_TAG - docker push $IMAGE_TAG - + echo noop update-tags: needs: build-and-push runs-on: ubuntu-latest @@ -52,7 +48,7 @@ jobs: sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-devcontainer - name: Commit changes env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.WORKFLOW_PAT }} run: | git config --global user.email "farmvibesaicd@microsoft.com" git config --global user.name "FarmVibes.AI Release Pipeline" diff --git a/.github/workflows/release-to-main.yml b/.github/workflows/release-to-main.yml index 3636d74d..c3e27eda 100644 --- a/.github/workflows/release-to-main.yml +++ b/.github/workflows/release-to-main.yml @@ -5,7 +5,6 @@ on: permissions: id-token: write contents: write - pull-requests: write jobs: release: From 9b57806c9234026c8f17a645ad4459249dd555ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 13:43:42 +0000 Subject: [PATCH 03/19] try to authenticate with an ssh-key --- .github/workflows/base-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index ba33e032..3fbf36a6 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -38,6 +38,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + ssh-key: ${{ secrets.WORKFLOW_KEY }} - name: 'Update tags' run: | TAG=${{ github.run_id }} @@ -47,8 +49,6 @@ jobs: sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-worker sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-devcontainer - name: Commit changes - env: - GITHUB_TOKEN: ${{ secrets.WORKFLOW_PAT }} run: | git config --global user.email "farmvibesaicd@microsoft.com" git config --global user.name "FarmVibes.AI Release Pipeline" From da951571536e3244040546b9406d3605aa354838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 13:48:01 +0000 Subject: [PATCH 04/19] check remotes --- .github/workflows/base-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index 3fbf36a6..cbcc3f33 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -50,6 +50,7 @@ jobs: sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-devcontainer - name: Commit changes run: | + echo "git remote -v" git config --global user.email "farmvibesaicd@microsoft.com" git config --global user.name "FarmVibes.AI Release Pipeline" BRANCH=update-base-${{ github.run_id }} From 5ef7e8cc3ce7b69a4e09584586ef4d2be60644cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 13:52:21 +0000 Subject: [PATCH 05/19] fix command --- .github/workflows/base-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index cbcc3f33..8ce4f3ff 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -50,7 +50,7 @@ jobs: sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-devcontainer - name: Commit changes run: | - echo "git remote -v" + git remote -v git config --global user.email "farmvibesaicd@microsoft.com" git config --global user.name "FarmVibes.AI Release Pipeline" BRANCH=update-base-${{ github.run_id }} From 993f41c6e5eea0c8fa735a4e736d68d171ee317d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 13:59:24 +0000 Subject: [PATCH 06/19] Do not use https please? --- .github/workflows/base-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index 8ce4f3ff..c9681556 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -39,6 +39,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + token: '' ssh-key: ${{ secrets.WORKFLOW_KEY }} - name: 'Update tags' run: | From fbf01db5d51772198bdb30aa8baf26cbd4acf325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 14:21:01 +0000 Subject: [PATCH 07/19] debug key --- .github/workflows/base-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index c9681556..86c37ca0 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -34,15 +34,15 @@ jobs: run: | echo noop update-tags: - needs: build-and-push + # needs: build-and-push runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: - token: '' ssh-key: ${{ secrets.WORKFLOW_KEY }} - name: 'Update tags' run: | + if [ -n ${{ secrets.WORKFLOW_KEY }} ]; then echo "have key"; else echo "no key"; fi TAG=${{ github.run_id }} sed -i "s|\(\s\+image.*:\).*|\1${TAG}|" ./.github/workflows/lint-test.yml sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-api_orchestrator From a9551998b538b1940333916f5e862a62c0d883ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 14:31:07 +0000 Subject: [PATCH 08/19] Change url myself? --- .github/workflows/base-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index 86c37ca0..19438a59 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -42,7 +42,6 @@ jobs: ssh-key: ${{ secrets.WORKFLOW_KEY }} - name: 'Update tags' run: | - if [ -n ${{ secrets.WORKFLOW_KEY }} ]; then echo "have key"; else echo "no key"; fi TAG=${{ github.run_id }} sed -i "s|\(\s\+image.*:\).*|\1${TAG}|" ./.github/workflows/lint-test.yml sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-api_orchestrator @@ -51,6 +50,8 @@ jobs: sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-devcontainer - name: Commit changes run: | + git remote -v + git remote set-url origin git@github.com:microsoft/farmvibes-ai.git git remote -v git config --global user.email "farmvibesaicd@microsoft.com" git config --global user.name "FarmVibes.AI Release Pipeline" From f4f2e17bc84c836b51f2cf67c1804012f1527dec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 14:33:47 +0000 Subject: [PATCH 09/19] write key --- .github/workflows/base-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index 19438a59..cbb8e2c0 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -53,6 +53,7 @@ jobs: git remote -v git remote set-url origin git@github.com:microsoft/farmvibes-ai.git git remote -v + echo ${{ secrets.WORKFLOW_KEY }} >> ~/.ssh/id_rsa git config --global user.email "farmvibesaicd@microsoft.com" git config --global user.name "FarmVibes.AI Release Pipeline" BRANCH=update-base-${{ github.run_id }} From a09d284ed421d479c4b5198397f78a8639ad8383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 14:34:51 +0000 Subject: [PATCH 10/19] makedir --- .github/workflows/base-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index cbb8e2c0..09956ec4 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -53,6 +53,7 @@ jobs: git remote -v git remote set-url origin git@github.com:microsoft/farmvibes-ai.git git remote -v + mkdir -p ~/.ssh/ echo ${{ secrets.WORKFLOW_KEY }} >> ~/.ssh/id_rsa git config --global user.email "farmvibesaicd@microsoft.com" git config --global user.name "FarmVibes.AI Release Pipeline" From 0bbfc9644573d1e75ca714bc2d9c6a1f8cc6d313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 14:37:01 +0000 Subject: [PATCH 11/19] change permission --- .github/workflows/base-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index 09956ec4..4a3e7125 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -55,6 +55,7 @@ jobs: git remote -v mkdir -p ~/.ssh/ echo ${{ secrets.WORKFLOW_KEY }} >> ~/.ssh/id_rsa + chmod 400 ~/.ssh/id_rsa git config --global user.email "farmvibesaicd@microsoft.com" git config --global user.name "FarmVibes.AI Release Pipeline" BRANCH=update-base-${{ github.run_id }} From 7607b449e23a9649270bf52c70bd37c9ca5ce8b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 15:20:26 +0000 Subject: [PATCH 12/19] Check key? --- .github/workflows/base-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index 4a3e7125..bf49e72e 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -55,6 +55,7 @@ jobs: git remote -v mkdir -p ~/.ssh/ echo ${{ secrets.WORKFLOW_KEY }} >> ~/.ssh/id_rsa + ssh-keygen -y -f ~/.ssh/id_rsa chmod 400 ~/.ssh/id_rsa git config --global user.email "farmvibesaicd@microsoft.com" git config --global user.name "FarmVibes.AI Release Pipeline" From b98264b3b815636ce7a539b4ecb6740f9abffc05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 15:21:49 +0000 Subject: [PATCH 13/19] change order --- .github/workflows/base-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index bf49e72e..ec8d70dd 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -55,8 +55,8 @@ jobs: git remote -v mkdir -p ~/.ssh/ echo ${{ secrets.WORKFLOW_KEY }} >> ~/.ssh/id_rsa - ssh-keygen -y -f ~/.ssh/id_rsa chmod 400 ~/.ssh/id_rsa + ssh-keygen -y -f ~/.ssh/id_rsa git config --global user.email "farmvibesaicd@microsoft.com" git config --global user.name "FarmVibes.AI Release Pipeline" BRANCH=update-base-${{ github.run_id }} From 6ffb9504510520545f6b13743735a32496f9997f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 17:04:58 +0000 Subject: [PATCH 14/19] try to use PAT again --- .github/workflows/base-build.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index ec8d70dd..aadc54c7 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -38,8 +38,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - ssh-key: ${{ secrets.WORKFLOW_KEY }} + token: ${{ secrets.WORKFLOW_PAT }} - name: 'Update tags' run: | TAG=${{ github.run_id }} @@ -50,13 +49,6 @@ jobs: sed -i "s|\(FROM.*:\).*|\1${TAG}|" ./resources/docker/Dockerfile-devcontainer - name: Commit changes run: | - git remote -v - git remote set-url origin git@github.com:microsoft/farmvibes-ai.git - git remote -v - mkdir -p ~/.ssh/ - echo ${{ secrets.WORKFLOW_KEY }} >> ~/.ssh/id_rsa - chmod 400 ~/.ssh/id_rsa - ssh-keygen -y -f ~/.ssh/id_rsa git config --global user.email "farmvibesaicd@microsoft.com" git config --global user.name "FarmVibes.AI Release Pipeline" BRANCH=update-base-${{ github.run_id }} From 586a1bdaceea4fb8754c876f6e6fae48544c905d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 17:08:19 +0000 Subject: [PATCH 15/19] fix yaml --- .github/workflows/base-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index aadc54c7..67c04747 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -38,7 +38,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - token: ${{ secrets.WORKFLOW_PAT }} + with: + token: ${{ secrets.WORKFLOW_PAT }} - name: 'Update tags' run: | TAG=${{ github.run_id }} From 74d56535d6e2051ae020872e9be07bd0d6662cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 17:12:40 +0000 Subject: [PATCH 16/19] Add environment --- .github/workflows/base-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index 67c04747..e57790ff 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -36,6 +36,7 @@ jobs: update-tags: # needs: build-and-push runs-on: ubuntu-latest + environment: build steps: - uses: actions/checkout@v4 with: From 86d41d3689288b498bff961ecc9f588074d9b0ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 18:52:14 +0000 Subject: [PATCH 17/19] ssh again --- .github/workflows/base-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index e57790ff..258feeb1 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -40,7 +40,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - token: ${{ secrets.WORKFLOW_PAT }} + ssh-key: ${{ secrets.WORKFLOW_KEY }} - name: 'Update tags' run: | TAG=${{ github.run_id }} From 2f32793332b1f8353f7b2961b2cb6c8a967f9b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 18:59:10 +0000 Subject: [PATCH 18/19] Put other stuff back --- .github/workflows/base-build.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index 258feeb1..e63aa76b 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -9,7 +9,6 @@ permissions: id-token: write contents: write - jobs: build-and-push: @@ -24,6 +23,8 @@ jobs: image-name: worker-base steps: - uses: actions/checkout@v4 + with: + ref: dev - name: 'Az CLI login' uses: azure/login@v1 with: @@ -32,14 +33,19 @@ jobs: subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: 'Build and push image' run: | - echo noop + az acr login -n ${{ secrets.ACR_NAME }} + VERSION_TAG=${{ github.run_id }} + export IMAGE_TAG=${{ secrets.ACR_NAME }}.azurecr.io/unlisted/farmai/terravibes/${{ matrix.image-name }}:$VERSION_TAG + docker build . -f ./resources/docker/${{ matrix.dockerfile }} -t $IMAGE_TAG + docker push $IMAGE_TAG update-tags: - # needs: build-and-push + needs: build-and-push runs-on: ubuntu-latest environment: build steps: - uses: actions/checkout@v4 with: + ref: dev ssh-key: ${{ secrets.WORKFLOW_KEY }} - name: 'Update tags' run: | From 0ee70d9768051acb3b315640537557851252589c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Estev=C3=A3o?= Date: Fri, 2 Aug 2024 19:09:00 +0000 Subject: [PATCH 19/19] Remove test trigger --- .github/workflows/base-build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/base-build.yml b/.github/workflows/base-build.yml index e63aa76b..87b4178e 100644 --- a/.github/workflows/base-build.yml +++ b/.github/workflows/base-build.yml @@ -1,9 +1,6 @@ name: Build base images on: workflow_dispatch: - pull_request: - branches: - - dev permissions: id-token: write