Update GCP installation process

Change-Id: I009474b4c4d8f67ed28e600a8c870345a2354fe9

gcp/cloud-functions/create-vm/index.js

@@ -146,10 +146,18 @@ functions.cloudEvent('createInstance', async (cloudEvent) => {
   // Get a config uri (config.yaml) and ads config uri (google-ads.yaml) from the pub/sub message payload,
   // And if it exists pass it as a custom metadata key-value to VM
 
-  setMetadata(vmConfig.metadata.items, 'gcs_source_uri', data.gcs_source_uri);
-  setMetadata(vmConfig.metadata.items, 'gcs_base_path_public', data.gcs_base_path_public);
-  if (data.delete_vm !== undefined) {
-    setMetadata(vmConfig.metadata.items, 'delete_vm', data.delete_vm);
+  // all keys from vm object in request data forward as VM's attributes
+  if (data.vm) {
+    for(let key of Object.keys(data.vm)) {
+      let val = data.vm[key];
+      if (val === true || val === "true" || val === "True") {
+        val = "TRUE";
+      }
+      else if (val === false || val === "false" || val === "False") {
+        val = "FALSE";
+      }
+      setMetadata(vmConfig.metadata.items, key, val);
+    }
   }
 
   // org policy can prevent using external IPs, if so we'll remove accessConfig and this will prevent assigning an external IP

gcp/cloud-run-button/prebuild.sh

@@ -30,6 +30,6 @@ gcloud config set project $GOOGLE_CLOUD_PROJECT
 
 ./gcp/install.sh
 
-echo -e "${CYAN}!!!!! Please ignore all output below !!!!!${WHITE}"
-echo
-echo
+echo -e "${CYAN}Please ignore all output below${WHITE}"
+
+gcloud auth configure-docker --quiet

gcp/cloudbuild-gcr.yaml

@@ -0,0 +1,5 @@
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/${_IMAGE}', '-f', 'gcp/workload-vm/Dockerfile', '.' ]
+
+images: [ 'gcr.io/$PROJECT_ID/${_IMAGE}' ]

gcp/setup.sh

@@ -1,4 +1,19 @@
 #!/bin/bash
+#
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # ansi colors
 GREEN='\033[0;32m'
 CYAN='\033[0;36m'
@@ -9,7 +24,9 @@ SETTING_FILE="./settings.ini"
 SCRIPT_PATH=$(readlink -f "$0" | xargs dirname)
 SETTING_FILE="${SCRIPT_PATH}/settings.ini"
 
-# changing the cwd to the script's contining folder so all pathes inside can be local to it
+trap _upload_install_log EXIT
+
+# changing the cwd to the script's containing folder so all pathes inside can be local to it
 # (important as the script can be called via absolute path and as a nested path)
 pushd $SCRIPT_PATH >/dev/null
 
@@ -25,35 +42,77 @@ while :; do
   shift
 done
 
+NAME=$(git config -f $SETTING_FILE config.name)
+PROJECT_ID=$(gcloud config get-value project 2> /dev/null)
+PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="csv(projectNumber)" | tail -n 1)
+USER_EMAIL=$(gcloud config get-value account 2> /dev/null)
+
+APP_CONFIG_FILE=$(eval echo $(git config -f $SETTING_FILE config.config-file))
 REPOSITORY=$(eval echo $(git config -f $SETTING_FILE repository.name))
 IMAGE_NAME=$(eval echo $(git config -f $SETTING_FILE repository.image))
 REPOSITORY_LOCATION=$(git config -f $SETTING_FILE repository.location)
 TOPIC=$(eval echo $(git config -f $SETTING_FILE pubsub.topic))
 
-NAME=$(git config -f $SETTING_FILE config.name)
-PROJECT_ID=$(gcloud config get-value project 2> /dev/null)
-PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="csv(projectNumber)" | tail -n 1)
 SERVICE_ACCOUNT=$PROJECT_NUMBER[email protected]
 
+check_billing() {
+  BILLING_ENABLED=$(gcloud beta billing projects describe $PROJECT_ID --format="csv(billingEnabled)" | tail -n 1)
+  if [[ "$BILLING_ENABLED" = 'False' ]]
+  then
+    echo -e "${RED}The project $PROJECT_ID does not have a billing enabled. Please activate billing${NC}"
+    exit -1
+  fi
+}
+
+deploy_files() {
+  echo 'Deploying files to GCS'
+  if ! gsutil ls gs://$PROJECT_ID > /dev/null 2> /dev/null; then
+    echo "Creating GCS bucket gs://$PROJECT_ID"
+    gsutil mb -b on gs://$PROJECT_ID
+  fi
+
+  GCS_BASE_PATH=gs://$PROJECT_ID/$NAME
+
+  # NOTE: DO NOT add -m flag for gsutil! When executed under cloudshell_open (via Cloud Run Button) it won't copy files
+  echo "Removing existing files at $GCS_BASE_PATH"
+  gsutil rm -r $GCS_BASE_PATH/
+
+  # NOTE: if an error "module 'sys' has no attribute 'maxint'" occures, run this: `pip3 install -U crcmod`
+  echo "Copying application files to $GCS_BASE_PATH"
+  gsutil rsync -r -x ".*/__pycache__/.*|[.].*" ./../app $GCS_BASE_PATH
+  echo "Copying configs to $GCS_BASE_PATH"
+  gsutil -h "Content-Type:text/plain" cp ./../app/*.yaml $GCS_BASE_PATH/
+  if [[ -f ./../google-ads.yaml ]]; then
+    gsutil -h "Content-Type:text/plain" cp ./../google-ads.yaml $GCS_BASE_PATH/google-ads.yaml
+  elif [[ -f $HOME/google-ads.yaml ]]; then
+    gsutil -h "Content-Type:text/plain" cp $HOME/google-ads.yaml $GCS_BASE_PATH/google-ads.yaml
+  else
+    echo "Please upload google-ads.yaml"
+  fi
+}
+
 
 enable_apis() {
   echo "Enabling APIs"
+  gcloud services enable bigquery.googleapis.com
   gcloud services enable compute.googleapis.com
-  gcloud services enable artifactregistry.googleapis.com
+  #gcloud services enable artifactregistry.googleapis.com
+  gcloud services enable containerregistry.googleapis.com
   gcloud services enable run.googleapis.com
   gcloud services enable cloudresourcemanager.googleapis.com
   gcloud services enable iamcredentials.googleapis.com
   gcloud services enable cloudbuild.googleapis.com
   gcloud services enable cloudfunctions.googleapis.com
   gcloud services enable eventarc.googleapis.com
   gcloud services enable cloudscheduler.googleapis.com
+  gcloud services enable googleads.googleapis.com
 }
 
 
 create_registry() {
-  echo "Creating a repository in Artifact Registry"
-  REPO_EXISTS=$(gcloud artifacts repositories list --location=europe --filter="REPOSITORY:'"$REPOSITORY"'" --format="value(REPOSITORY)" 2>/dev/null)
+  REPO_EXISTS=$(gcloud artifacts repositories list --location=$REPOSITORY_LOCATION --filter="REPOSITORY=projects/'$PROJECT_ID'/locations/'$REPOSITORY_LOCATION'/repositories/'"$REPOSITORY"'" --format="value(REPOSITORY)" 2>/dev/null)
   if [[ ! -n $REPO_EXISTS ]]; then
+    echo "Creating a repository in Artifact Registry"
     # repo doesn't exist, creating
     gcloud artifacts repositories create ${REPOSITORY} \
         --repository-format=docker \
@@ -75,26 +134,30 @@ build_docker_image() {
 build_docker_image_gcr() {
   # NOTE: it's an alternative to build_docker_image if you want to use GCR instead of AR
   echo "Building and pushing Docker image to Container Registry"
-  gcloud builds submit --config=cloudbuild-gcr.yaml --substitutions=_IMAGE="workload" ./workload-vm
+  gcloud builds submit --config=cloudbuild-gcr.yaml --substitutions=_IMAGE="$IMAGE_NAME" ./..
 }
 
 
 set_iam_permissions() {
+  required_roles="storage.objectViewer artifactregistry.repoAdmin compute.admin monitoring.editor logging.logWriter iam.serviceAccountTokenCreator pubsub.publisher run.invoker"
   echo "Setting up IAM permissions"
-  gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/storage.objectViewer
-  gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/artifactregistry.repoAdmin
-  gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/compute.admin
-  gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/monitoring.editor
+  for role in $required_roles; do
+    gcloud projects add-iam-policy-binding $PROJECT_ID \
+      --member=serviceAccount:$SERVICE_ACCOUNT \
+      --role=roles/$role \
+      --no-user-output-enabled
+  done
 }
 
 
 create_topic() {
-  TOPIC_EXISTS=$(gcloud pubsub topics list --filter="name.scope(topic):'$TOPIC'" --format="get(name)")
+  TOPIC_EXISTS=$(gcloud pubsub topics list --filter="name=projects/'$PROJECT_ID'/topics/'$TOPIC'" --format="get(name)")
   if [[ ! -n $TOPIC_EXISTS ]]; then
     gcloud pubsub topics create $TOPIC
   fi
 }
 
+
 deploy_cf() {
   echo "Deploying Cloud Function"
   CF_REGION=$(git config -f $SETTING_FILE function.region)
@@ -109,7 +172,8 @@ deploy_cf() {
   fi
   # initialize env.yaml - environment variables for CF:
   #   - docker image url
-  url="$REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/docker/$IMAGE_NAME"
+  #url="$REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/docker/$IMAGE_NAME"
+  url="gcr.io/$PROJECT_ID/$IMAGE_NAME"
   sed -i'.bak' -e "s|#*[[:space:]]*DOCKER_IMAGE[[:space:]]*:[[:space:]]*.*$|DOCKER_IMAGE: $url|" ./cloud-functions/create-vm/env.yaml
   #   - GCE VM name (base)
   instance=$(eval echo $(git config -f $SETTING_FILE compute.name))
@@ -150,41 +214,17 @@ deploy_cf() {
 }
 
 
-deploy_files() {
-  echo 'Deploying files to GCS'
-  if ! gsutil ls gs://$PROJECT_ID > /dev/null 2> /dev/null; then
-    echo "Creating GCS bucket gs://$PROJECT_ID"
-    gsutil mb -b on gs://$PROJECT_ID
-  fi
-
-  GCS_BASE_PATH=gs://$PROJECT_ID/$NAME
-
-  # NOTE: DO NOT add -m flag for gsutil! When executed under cloudshell_open (via Cloud Run Button) it won't copy files
-  echo "Removing existing files at $GCS_BASE_PATH"
-  gsutil rm -r $GCS_BASE_PATH/
-
-  # NOTE: if an error "module 'sys' has no attribute 'maxint'" occures, run this: `pip3 install -U crcmod`
-  echo "Copying application files to $GCS_BASE_PATH"
-  gsutil rsync -r -x ".*/__pycache__/.*|[.].*" ./../app $GCS_BASE_PATH
-  echo "Copying configs to $GCS_BASE_PATH"
-  gsutil -h "Content-Type:text/plain" cp ./../app/*.yaml $GCS_BASE_PATH/
-  if [[ -f ./../google-ads.yaml ]]; then
-    gsutil -h "Content-Type:text/plain" cp ./../google-ads.yaml $GCS_BASE_PATH/google-ads.yaml
-  elif [[ -f $HOME/google-ads.yaml ]]; then
-    gsutil -h "Content-Type:text/plain" cp $HOME/google-ads.yaml $GCS_BASE_PATH/google-ads.yaml
-  else
-    echo "Please upload google-ads.yaml"
-  fi
-}
-
 deploy_public_index() {
   echo 'Deploying index.html to GCS'
 
-  gsutil mb -b on gs://${PROJECT_ID}-public
+  if ! gsutil ls gs://$PROJECT_ID-public > /dev/null 2> /dev/null; then
+    gsutil mb -b on gs://$PROJECT_ID-public
+  fi
+
   gsutil iam ch -f allUsers:objectViewer gs://${PROJECT_ID}-public 2> /dev/null
   exitcode=$?
   if [ $exitcode -ne 0 ]; then
-    echo "Could not add public access to public cloud bucket"
+    echo -e "${RED}[ ! ] Could not add public access to public cloud bucket${NC}"
   else
     GCS_BASE_PATH_PUBLIC=gs://${PROJECT_ID}-public/$NAME
     gsutil -h "Content-Type:text/html" -h "Cache-Control: no-store" cp "${SCRIPT_PATH}/index.html" $GCS_BASE_PATH_PUBLIC/index.html
@@ -196,16 +236,19 @@ deploy_public_index() {
 
 
 get_run_data() {
+  local dashboard_url="$1"
   # arguments for the CF (to be passed via pubsub message or scheduler job's arguments):
   #   * project_id
   #   * machine_type
   #   * service_account
-  #   * gcs_source_uri
-  #   * gcs_base_path_public
   #   * docker_image - a docker image url, can be CR or AR
   #       gcr.io/$PROJECT_ID/workload
   #       europe-docker.pkg.dev/$PROJECT_ID/docker/workload
-  #   * delete_vm - by default it's TRUE (set inside create-vm CF)
+  #   * vm - an object with attributes for VM (they will be passed to main.sh via VM's metadata):
+  #     * gcs_source_uri
+  #     * gcs_base_path_public
+  #     * create_dashboard_link
+  #     * delete_vm - by default it's TRUE (set inside create-vm CF)
   GCS_BASE_PATH=gs://$PROJECT_ID/$NAME
   GCS_BASE_PATH_PUBLIC=gs://${PROJECT_ID}-public/$NAME
 
@@ -216,9 +259,12 @@ get_run_data() {
   # if you need to prevent VM deletion add this:
   #  "delete_vm": "FALSE"
   data='{
-    "gcs_source_uri": "'$GCS_BASE_PATH'",
-    "gcs_base_path_public": "'$GCS_BASE_PATH_PUBLIC'",
-    "delete_vm": "TRUE"
+    "vm": {
+      "gcs_source_uri": "'$GCS_BASE_PATH'",
+      "gcs_base_path_public": "'$GCS_BASE_PATH_PUBLIC'",
+      "create_dashboard_url": "'$dashboard_url'",
+      "delete_vm": "TRUE"
+    }
   }'
   echo $data
 }
@@ -234,24 +280,27 @@ start() {
   # example:
   # --message="{\"project_id\":\"$PROJECT_ID\", \"docker_image\":\"europe-docker.pkg.dev/$PROJECT_ID/docker/workload\", \"service_account\":\"$SERVICE_ACCOUNT\"}"
 
-  local DATA=$(get_run_data)
+  dashboard_url=$(./../app/scripts/create_dashboard.sh -L --config ./../app/$APP_CONFIG_FILE)
+
+  local DATA=$(get_run_data $dashboard_url)
   echo 'Publishing a pubsub with args: '$DATA
   gcloud pubsub topics publish $TOPIC --message="$DATA"
 
   # Check if there is a public bucket and index.html and echo the url
   local PUBLIC_URL=$(print_public_gcs_url)/index.html
+  local GOOGLE_GROUP="https://groups.google.com/g/dactionboard"
+  echo -e "${CYAN}[ * ] Please join Google group to get access to the dashboard - ${GREEN}${GOOGLE_GROUP}${NC}"
   STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" $PUBLIC_URL)
 
   if [[ $STATUS_CODE -eq 200 ]]; then
     echo -e "${CYAN}[ * ] To access your new dashboard, click this link - ${GREEN}${PUBLIC_URL}${NC}"
   else
     echo -e "${CYAN}[ * ] Your GCP project does not allow public access.${NC}"
-    if [[ -f ./../dactionboard.yaml ]]; then
-      dashboard_url=$(./../scripts/create_dashboard.sh -L --config dactionboard.yaml)
+    if [[ -f ./../app/$APP_CONFIG_FILE ]]; then
       echo -e "${CYAN}[ * ] To create your dashboard, click the following link once the installation process completes and all the relevant tables have been created in the DB:"
       echo -e "${GREEN}$dashboard_url${NC}"
     else
-      echo -e "${CYAN}[ * ] To create your dashboard, please run the ${GREEN}./scripts/create_dashboard.sh -c dactionboard.yaml -L${CYAN} shell script once the installation process completes and all the relevant tables have been created in the DB.${NC}"
+      echo -e "${CYAN}[ * ] To create your dashboard, please run the ${GREEN}./scripts/create_dashboard.sh -c $APP_CONFIG_FILE -L${CYAN} shell script once the installation process completes and all the relevant tables have been created in the DB.${NC}"
     fi
   fi
 }
@@ -274,7 +323,6 @@ schedule_run() {
   delete_schedule
 
   echo 'Scheduling a job with args: '$DATA
-
   gcloud scheduler jobs create pubsub $JOB_NAME \
     --schedule="$SCHEDULE" \
     --location=$REGION \
@@ -283,11 +331,12 @@ schedule_run() {
     --time-zone=$SCHEDULE_TZ
 }
 
+
 delete_schedule() {
   JOB_NAME=$(eval echo $(git config -f $SETTING_FILE scheduler.name))
   REGION=$(git config -f $SETTING_FILE scheduler.region)
 
-  JOB_EXISTS=$(gcloud scheduler jobs list --location=$REGION --format="value(ID)" --filter="ID:'$JOB_NAME'" 2>/dev/null)
+  JOB_EXISTS=$(gcloud scheduler jobs list --location=$REGION --format="value(ID)" --filter="ID=projects/'$PROJECT_ID'/locations/'$REGION'/jobs/'$JOB_NAME'" 2>/dev/null)
   if [[ -n $JOB_EXISTS ]]; then
     echo 'Deleting Cloud Scheduler job '$JOB_NAME
     gcloud scheduler jobs delete $JOB_NAME --location $REGION --quiet
@@ -298,16 +347,39 @@ enable_private_google_access() {
   REGION=$(git config -f $SETTING_FILE compute.region)
   gcloud compute networks subnets update default --region=$REGION --enable-private-ip-google-access
 }
+
+check_owners() {
+  local project_admins=$(gcloud projects get-iam-policy $PROJECT_ID \
+    --flatten="bindings" \
+    --filter="bindings.role=roles/owner" \
+    --format="value(bindings.members[])"
+  )
+  if [[ ! $project_admins =~ $USER_EMAIL ]]; then
+      echo "User $USER_EMAIL does not have admin right to project $PROJECT_ID"
+      exit
+  fi
+}
+
+
 deploy_all() {
+  check_owners
+  check_billing
   enable_apis
   set_iam_permissions
   deploy_files
-  create_registry
-  build_docker_image
+  #create_registry - uncomment then migrated back to Artifact Registry
+  #build_docker_image - uncomment to migrated back to Artifact Registry
+  build_docker_image_gcr  # using Container Registry as most reliable service but it's only safe till May 2024 (due to upcomming deprication)
   deploy_cf
   schedule_run
 }
 
+_upload_install_log() {
+  if [[ -f "/tmp/${NAME}_installer.log" ]]; then
+    gsutil cp /tmp/${NAME}_installer.log gs://$PROJECT_ID/$NAME/
+    rm "/tmp/${NAME}_installer.log"
+  fi
+}
 
 _list_functions() {
   # list all functions in this file not starting with "_"
@@ -320,7 +392,7 @@ if [[ $# -eq 0 ]]; then
 else
   for i in "$@"; do
     if declare -F "$i" > /dev/null; then
-      "$i"
+      "$i" 2>&1 | tee -a /tmp/${NAME}_installer.log
       exitcode=$?
       if [ $exitcode -ne 0 ]; then
         echo "Breaking script as command '$i' failed"