From 0e1c8d5c3bc47c897c853da3981fa324e1d01038 Mon Sep 17 00:00:00 2001 From: Diogo Recharte Date: Fri, 17 Jan 2025 21:44:49 +0000 Subject: [PATCH 1/3] EVEREST-107 Simplify steps to run a FB By embedding all the artifacts in a single directory we can remove the complexity of setting the env vars. --- .github/workflows/feature-build.yaml | 101 ++++++++------------------- dev/fb/cli.sh | 33 ++++----- dev/fb/helm.sh | 14 +--- dev/fb/vs.sh | 15 ---- 4 files changed, 45 insertions(+), 118 deletions(-) mode change 100644 => 100755 dev/fb/cli.sh mode change 100644 => 100755 dev/fb/helm.sh mode change 100644 => 100755 dev/fb/vs.sh diff --git a/.github/workflows/feature-build.yaml b/.github/workflows/feature-build.yaml index a6d7e2977..c81360843 100644 --- a/.github/workflows/feature-build.yaml +++ b/.github/workflows/feature-build.yaml @@ -99,6 +99,9 @@ jobs: echo "VERSION_TAG=v$V" >> $GITHUB_ENV echo "VS_TAG=everest-test$TIMESTAMP" >> $GITHUB_ENV + - name: Create temporary directory for storing artifacts + run: | + mkdir -p /tmp/${VERSION} - name: Set GO_VERSION environment variable run: | @@ -211,15 +214,9 @@ jobs: CRD_VERSION=${{ env.VERSION_TAG }} make crds-gen make release-dev - - name: Helm - Create tar.gz archive + - name: Helm - copy chart run: | - tar -czvf helm.tar.gz percona-helm-charts/charts/everest - - - name: Helm - Upload artifact - uses: actions/upload-artifact@v4 - with: - name: helm - path: helm.tar.gz + cp -r percona-helm-charts/charts/everest /tmp/${{ env.VERSION }}/helm-chart - name: Operator - check out again uses: actions/checkout@v4 @@ -353,45 +350,19 @@ jobs: RELEASE_VERSION=${VERSION_TAG} CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make rc RELEASE_VERSION=${VERSION_TAG} make release-cli - - name: Upload CLI artefacts - uses: actions/upload-artifact@v4 - id: cli-upload-step-1 - with: - name: everestctl.exe - path: ./dist/everestctl.exe - retention-days: ${{ env.ARTIFACTS_RETENTION_DAYS }} - - - name: Upload CLI artefacts - uses: actions/upload-artifact@v4 - id: cli-upload-step-2 - with: - name: everestctl-darwin-amd64 - path: ./dist/everestctl-darwin-amd64 - retention-days: ${{ env.ARTIFACTS_RETENTION_DAYS }} - - - name: Upload CLI artefacts - uses: actions/upload-artifact@v4 - id: cli-upload-step-3 - with: - name: everestctl-darwin-arm64 - path: ./dist/everestctl-darwin-arm64 - retention-days: ${{ env.ARTIFACTS_RETENTION_DAYS }} - - - name: Upload CLI artefacts - uses: actions/upload-artifact@v4 - id: cli-upload-step-4 - with: - name: everestctl-linux-amd64 - path: ./dist/everestctl-linux-amd64 - retention-days: ${{ env.ARTIFACTS_RETENTION_DAYS }} + - name: Everest - copy CLI builds + run: | + cp dist/everestctl.exe /tmp/${VERSION}/everestctl.exe + cp dist/everestctl-darwin-amd64 /tmp/${VERSION}/everestctl-darwin-amd64 + cp dist/everestctl-darwin-arm64 /tmp/${VERSION}/everestctl-darwin-arm64 + cp dist/everestctl-linux-amd64 /tmp/${VERSION}/everestctl-linux-amd64 + cp dist/everestctl-linux-arm64 /tmp/${VERSION}/everestctl-linux-arm64 - - name: Upload CLI artefacts - uses: actions/upload-artifact@v4 - id: cli-upload-step-5 - with: - name: everestctl-linux-arm64 - path: ./dist/everestctl-linux-arm64 - retention-days: ${{ env.ARTIFACTS_RETENTION_DAYS }} + - name: Everest - copy feature build helper scripts + run: | + cp dev/fb/cli.sh /tmp/${VERSION}/cli.sh + cp dev/fb/helm.sh /tmp/${VERSION}/helm.sh + cp dev/fb/vs.sh /tmp/${VERSION}/vs.sh - name: Everest - push Everest image uses: docker/build-push-action@v6 @@ -402,47 +373,37 @@ jobs: - name: Create vs_tag file run: | - echo "$VS_TAG" > vs_tag.txt - echo "$VERSION" > fb.txt + echo "$VS_TAG" > /tmp/${VERSION}/vs_tag.txt + echo "$VERSION" > /tmp/${VERSION}/version.txt - - name: Upload VS tag info artefact - uses: actions/upload-artifact@v4 - id: vs_file - with: - name: vs_tag.txt - path: ./vs_tag.txt - retention-days: ${{ env.ARTIFACTS_RETENTION_DAYS }} + - name: Version Service - create deployment manifest + run: | + curl https://raw.githubusercontent.com/Percona-Lab/percona-version-service/main/deploy.yaml > /tmp/${VERSION}/vs_deploy.yaml + sed -i "s/perconalab\/version-service:.*/perconalab\/version-service:$VS_TAG/g" /tmp/${VERSION}/vs_deploy.yaml - - name: Upload FB info artefact + - name: Upload artifacts uses: actions/upload-artifact@v4 - id: fb_file with: - name: fb.txt - path: ./fb.txt + name: ${{ env.VERSION }} + path: /tmp/${{ env.VERSION }} retention-days: ${{ env.ARTIFACTS_RETENTION_DAYS }} - name: Info run: | printf "To install FB using helm:\n\ - 1. Download the 'helm' artifact below.\n\ - 2. Navigate to \`/everest/dev/fb directory\`.\n\ - 3. Run the command:\n\`\`\`VS_IMAGE=$VS_TAG HELM_PATH= bash helm.sh\`\`\`\n\ - \nexample:\n\`\`\`VS_IMAGE=$VS_TAG HELM_PATH=/Users/oxana/Downloads/percona-helm-charts bash helm.sh\`\`\`\n" >> "$GITHUB_STEP_SUMMARY" + 1. Download and unzip the \`$VERSION\` artifact below.\n\ + 2. Run the command:\n\`\`\`./helm.sh\`\`\`\n" >> "$GITHUB_STEP_SUMMARY" printf "____________________\n" >> "$GITHUB_STEP_SUMMARY" printf "To install FB using CLI:\n\ - 1. Download the 'helm' artifact below.\n\ - 2. Download a CLI build from artifacts below.\n\ - 3. Navigate to \`/everest/dev/fb\` directory.\n\ - 4. Run the command:\n\`\`\`VS_IMAGE=$VS_TAG VERSION=$VERSION EVEREST_CTL_PATH= HELM_PATH= bash cli.sh\`\`\`\n\ - \nexample:\n\`\`\`VS_IMAGE=$VS_TAG VERSION=$VERSION HELM_PATH=/Users/oxana/Downloads/percona-helm-charts EVEREST_CTL_PATH=/Users/oxana/Downloads/everestctl-darwin-arm64 bash cli.sh\`\`\`\n" >> "$GITHUB_STEP_SUMMARY" - + 1. Download and unzip the \`$VERSION\` artifact below.\n\ + 2. Run the command:\n\`\`\`./cli.sh\`\`\`\n" >> "$GITHUB_STEP_SUMMARY" printf "____________________\n" >> "$GITHUB_STEP_SUMMARY" printf "To clean up the environment after a FB:\n\ 1. Delete VS: \`kubectl delete deployment percona-version-service && kubectl delete svc percona-version-service\`\n\ - 2. Uninstall Everest using the FB everestctl build: \` uninstall --assume-yes --force\` \n" >> "$GITHUB_STEP_SUMMARY" + 2. Uninstall Everest using the FB everestctl build: \`./ uninstall --assume-yes --force\` \n" >> "$GITHUB_STEP_SUMMARY" diff --git a/dev/fb/cli.sh b/dev/fb/cli.sh old mode 100644 new mode 100755 index a7f3414a5..d2071ad76 --- a/dev/fb/cli.sh +++ b/dev/fb/cli.sh @@ -1,29 +1,20 @@ #!/usr/bin/env bash -if [[ -z "$VS_IMAGE" ]]; then - echo "Error: VS_IMAGE environment variable is empty." >&2 - exit 1 -fi +# deploy VS and get it's internal IP +SERVICE_IP=$(./vs.sh) -if [[ -z "$VERSION" ]]; then - echo "Error: VERSION environment variable is empty." >&2 - exit 1 -fi +kubectl port-forward svc/percona-version-service 8081:80 & -if [[ -z "$HELM_PATH" ]]; then - echo "Error: HELM_PATH environment variable is empty. Please add the absolute path to your helm build to the HELM_PATH environment variable." >&2 - exit 1 -fi +os=$(uname -s | tr '[:upper:]' '[:lower:]') +arch=$(uname -m | tr '[:upper:]' '[:lower:]') -if [[ -z "$EVEREST_CTL_PATH" ]]; then - echo "Error: EVEREST_CTL_PATH environment variable is empty." >&2 - exit 1 +if [[ ($os == "linux" || $os == "darwin") && $arch == "x86_64" ]] +then + arch="amd64" +elif [[ $os == "linux" && $arch == "aarch64" ]] +then + arch="arm64" fi -# deploy VS and get it's internal IP -SERVICE_IP=$(curl -sfL https://raw.githubusercontent.com/percona/everest/main/dev/fb/vs.sh | bash -s) - -kubectl port-forward svc/percona-version-service 8081:80 & - # run everest installation with everest CLI -"$EVEREST_CTL_PATH" install --chart-dir "$HELM_PATH/charts/everest" --version "$VERSION" --version-metadata-url http://localhost:8081 --operator.xtradb-cluster --operator.mongodb --operator.postgresql --skip-wizard --namespaces everest -v --helm.set "versionMetadataURL=http://$SERVICE_IP" +"./everestctl-$os-$arch" install --chart-dir "helm-chart" --version "$(cat version.txt)" --version-metadata-url http://localhost:8081 --operator.xtradb-cluster --operator.mongodb --operator.postgresql --skip-wizard --namespaces everest -v --helm.set "versionMetadataURL=http://$SERVICE_IP" diff --git a/dev/fb/helm.sh b/dev/fb/helm.sh old mode 100644 new mode 100755 index 8efee5fde..8f058e05a --- a/dev/fb/helm.sh +++ b/dev/fb/helm.sh @@ -1,17 +1,7 @@ #!/usr/bin/env bash -if [[ -z "$HELM_PATH" ]]; then - echo "Error: HELM_PATH environment variable is empty. Please add the absolute path to your helm build to the HELM_PATH environment variable." >&2 - exit 1 -fi - -if [[ -z "$VS_IMAGE" ]]; then - echo "Error: VS_IMAGE environment variable is empty." >&2 - exit 1 -fi - # deploy VS and get it's internal IP -SERVICE_IP=$(curl -sfL https://raw.githubusercontent.com/percona/everest/main/dev/fb/vs.sh | bash -s) +SERVICE_IP=$(./vs.sh) # run everest installation with helm -helm install everest-core "$HELM_PATH/charts/everest" --namespace=everest-system --create-namespace --set versionMetadataURL="http://$SERVICE_IP" --timeout=10m --devel +helm install everest-core "helm-chart" --namespace=everest-system --create-namespace --set versionMetadataURL="http://$SERVICE_IP" --timeout=10m --devel diff --git a/dev/fb/vs.sh b/dev/fb/vs.sh old mode 100644 new mode 100755 index 34000a95c..0265933c1 --- a/dev/fb/vs.sh +++ b/dev/fb/vs.sh @@ -1,20 +1,5 @@ #!/usr/bin/env bash -if [[ -z "$VS_IMAGE" ]]; then - echo "Error: VS_IMAGE environment variable is empty." >&2 - exit 1 -fi - -curl https://raw.githubusercontent.com/Percona-Lab/percona-version-service/main/deploy.yaml > vs_deploy.yaml - -# use FB VS image in the VS configuration -if [[ "$(uname)" == "Darwin" ]]; then - sed -i '' "s/perconalab\/version-service:.*/perconalab\/version-service:$VS_IMAGE/g" vs_deploy.yaml -else - sed -i "s/perconalab\/version-service:.*/perconalab\/version-service:$VS_IMAGE/g" vs_deploy.yaml -fi - - # deploy VS kubectl apply -f vs_deploy.yaml > /dev/null From c1d845162b0bac5ee48c7454355fedb61d46328d Mon Sep 17 00:00:00 2001 From: Diogo Recharte Date: Sun, 19 Jan 2025 17:52:13 +0000 Subject: [PATCH 2/3] EVEREST-107 feature build - wait for the port-forward to be ready --- dev/fb/cli.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/fb/cli.sh b/dev/fb/cli.sh index d2071ad76..fc1ac843d 100755 --- a/dev/fb/cli.sh +++ b/dev/fb/cli.sh @@ -5,6 +5,9 @@ SERVICE_IP=$(./vs.sh) kubectl port-forward svc/percona-version-service 8081:80 & +# sleep for 2 seconds to make sure the port-forward is ready +sleep 2 + os=$(uname -s | tr '[:upper:]' '[:lower:]') arch=$(uname -m | tr '[:upper:]' '[:lower:]') From 0905b189e2c0c83d398e2509f18e082388b54e36 Mon Sep 17 00:00:00 2001 From: Diogo Recharte Date: Mon, 20 Jan 2025 16:51:05 +0000 Subject: [PATCH 3/3] EVEREST-107 increase FB VS ready timeout to 60s Sometimes the default 30s isn't enought and we are seeing error: readyReplicas is not found --- dev/fb/vs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/fb/vs.sh b/dev/fb/vs.sh index 0265933c1..be44d0269 100755 --- a/dev/fb/vs.sh +++ b/dev/fb/vs.sh @@ -4,7 +4,7 @@ kubectl apply -f vs_deploy.yaml > /dev/null # wait until the VS is ready -kubectl wait --for=jsonpath='{.status.readyReplicas}'=3 deployment/percona-version-service > /dev/null +kubectl wait --for=jsonpath='{.status.readyReplicas}'=3 deployment/percona-version-service --timeout 60s > /dev/null # get the internal IP of the VS echo "$(kubectl get service percona-version-service -o jsonpath='{.spec.clusterIP}')"