playground_deploy_examples.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Collect And Deploy Playground Examples

on:
  push:
    tags: 'v*'
    branches: ['master', 'release-*']
  pull_request:
  workflow_dispatch:
env:
  BEAM_ROOT_DIR: ../../
  BEAM_EXAMPLE_CATEGORIES: ../categories.yaml
  BEAM_VERSION: 2.33.0
  K8S_NAMESPACE: playground-backend
  HELM_APP_NAME: playground-backend
jobs:
  check_examples:
    name: Check examples
    runs-on: ubuntu-latest
    outputs:
      example_has_changed: ${{ steps.check_has_example.outputs.example_has_changed }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v2
        with:
          python-version: '3.8'
      - name: install deps
        run: pip install -r requirements.txt
        working-directory: playground/infrastructure
      - shell: pwsh
        name: get Difference
        id: check_file_changed
        run: |
          $diff = git diff --name-only master
          Write-Host "::set-output name=example_diff::$diff"
      - name: has Examples
        run: |
          output=$(python3 checker.py ${{ steps.check_file_changed.outputs.example_diff }})
          echo "::set-output name=example_has_changed::$output"
        id: check_has_example
        working-directory: playground/infrastructure
      - name: Print has_example
        run: echo "${{ steps.check_has_example.outputs.example_has_changed }}"
  deploy_examples:
    name: Deploy examples
    runs-on: ubuntu-latest
    needs: [ check_examples ]
    if: needs.check_examples.outputs.example_has_changed == 'True'
    steps:
      - name: Check out the repo
        uses: actions/checkout@v2
      - uses: actions/setup-python@v2
        with:
          python-version: '3.8'
      - uses: actions/setup-java@v2
        with:
            distribution: 'zulu'
            java-version: '8'
      - name: Install kubectl
        run: |
             curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" &&\
             chmod +x kubectl &&\
             mv kubectl /usr/local/bin/
      - name: Install helm
        run: |
             curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 &&\
             chmod 700 get_helm.sh &&\
             ./get_helm.sh
      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@v0
      - name: install deps
        run: pip install -r requirements.txt
        working-directory: playground/infrastructure
      - name: Remove default github maven configuration
        # This step is a workaround to avoid a decryption issue
        run: rm ~/.m2/settings.xml
      - name: Setup GCP account
        run: |
             echo "${{ secrets.GCP_PLAYGROUND_SA_KEY }}" | base64 -d > /tmp/gcp_access.json
             which gcloud
             gcloud auth activate-service-account --project=apache-beam-testing --key-file=/tmp/gcp_access.json
      - name: Set Docker Tag
        run: echo "DOCKERTAG=${GITHUB_SHA}" >> $GITHUB_ENV
      - name: Set Docker Tag If Github Tag was trigger
        run: echo "DOCKERTAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
        if: startsWith(github.ref, 'refs/tags/')
      - name: Get K8s Config
        run: gcloud container clusters get-credentials --region     us-central1-a playground-examples
      - name: Login to Docker Registry
        run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.PLAYGROUND_REGISTRY_NAME }}
      - name: Build And Push Java Backend
        run: |
             ./gradlew playground:backend:containers:java:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.GCP_PLAYGROUND_PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' -Pdocker-tag=${{ env.DOCKERTAG }}
      - name: Build And Push Go Backend
        run: ./gradlew playground:backend:containers:go:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.GCP_PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
      - name: Build And Push Python Backend
        run: ./gradlew playground:backend:containers:python:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.GCP_PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
      - name: Clean All Build directories
        run: ./gradlew clean
      - name: Install helm chart
        run: |
             kubectl create namespace $K8S_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - &&\
             helm install  --namespace $K8S_NAMESPACE $HELM_APP_NAME . --set global.registry="${{ secrets.PLAYGROUND_REGISTRY_NAME }}/${{ secrets.GCP_PLAYGROUND_PROJECT_ID }}/playground-repository" --set global.tag=${{ env.DOCKERTAG }} &&\
             sleep 120
        working-directory: playground/infrastructure/helm
      - name: Run Python Examples CI
        run: |
             K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
             BEAM_ROOT_DIR="../../"
             BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
             echo $K8S_SERVER_ADDRESS
             export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081" && python3 ci_cd.py --step CI --sdk SDK_PYTHON
        working-directory: playground/infrastructure
      - name: Run Python Examples CD
        run: |
             K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
             BEAM_ROOT_DIR="../../"
             BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
             export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081" && python3 ci_cd.py --step CD --sdk SDK_PYTHON
        working-directory: playground/infrastructure
        env:
          GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
      - name: Run Go Examples CI
        run: |
             K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-go" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
             BEAM_ROOT_DIR="../../"
             BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
             export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8082" && python3 ci_cd.py --step CI --sdk SDK_GO
        working-directory: playground/infrastructure
      - name: Run Go Examples CD
        run: |
             K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-go" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
             BEAM_ROOT_DIR="../../"
             BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
             export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8082" && python3 ci_cd.py --step CD --sdk SDK_GO
        working-directory: playground/infrastructure
        env:
          GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
      - name: Run Java Examples CI
        run: |
             K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-java" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
             BEAM_ROOT_DIR="../../"
             BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
             export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8080" && python3 ci_cd.py --step CI --sdk SDK_JAVA
        working-directory: playground/infrastructure
      - name: Run Java Examples CD
        run: |
             K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-java" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
             BEAM_ROOT_DIR="../../"
             BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
             export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8080" && python3 ci_cd.py --step CD --sdk SDK_JAVA
        working-directory: playground/infrastructure
        env:
          GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
      - name: Delete Helm Chart
        if: always()
        run: |
             helm del --namespace $K8S_NAMESPACE $HELM_APP_NAME