diff --git a/gcp/cloud-functions/create-vm/index.js b/gcp/cloud-functions/create-vm/index.js index 61151c4..2454188 100644 --- a/gcp/cloud-functions/create-vm/index.js +++ b/gcp/cloud-functions/create-vm/index.js @@ -146,10 +146,18 @@ functions.cloudEvent('createInstance', async (cloudEvent) => { // Get a config uri (config.yaml) and ads config uri (google-ads.yaml) from the pub/sub message payload, // And if it exists pass it as a custom metadata key-value to VM - setMetadata(vmConfig.metadata.items, 'gcs_source_uri', data.gcs_source_uri); - setMetadata(vmConfig.metadata.items, 'gcs_base_path_public', data.gcs_base_path_public); - if (data.delete_vm !== undefined) { - setMetadata(vmConfig.metadata.items, 'delete_vm', data.delete_vm); + // all keys from vm object in request data forward as VM's attributes + if (data.vm) { + for(let key of Object.keys(data.vm)) { + let val = data.vm[key]; + if (val === true || val === "true" || val === "True") { + val = "TRUE"; + } + else if (val === false || val === "false" || val === "False") { + val = "FALSE"; + } + setMetadata(vmConfig.metadata.items, key, val); + } } // org policy can prevent using external IPs, if so we'll remove accessConfig and this will prevent assigning an external IP diff --git a/gcp/cloud-run-button/prebuild.sh b/gcp/cloud-run-button/prebuild.sh index 2a4f6cf..0226ce6 100755 --- a/gcp/cloud-run-button/prebuild.sh +++ b/gcp/cloud-run-button/prebuild.sh @@ -30,6 +30,6 @@ gcloud config set project $GOOGLE_CLOUD_PROJECT ./gcp/install.sh -echo -e "${CYAN}!!!!! Please ignore all output below !!!!!${WHITE}" -echo -echo +echo -e "${CYAN}Please ignore all output below${WHITE}" + +gcloud auth configure-docker --quiet diff --git a/gcp/cloudbuild-gcr.yaml b/gcp/cloudbuild-gcr.yaml new file mode 100644 index 0000000..ae1ae60 --- /dev/null +++ b/gcp/cloudbuild-gcr.yaml @@ -0,0 +1,5 @@ +steps: +- name: 'gcr.io/cloud-builders/docker' + args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/${_IMAGE}', '-f', 'gcp/workload-vm/Dockerfile', '.' ] + +images: [ 'gcr.io/$PROJECT_ID/${_IMAGE}' ] \ No newline at end of file diff --git a/gcp/setup.sh b/gcp/setup.sh index e64b46d..9fe2696 100755 --- a/gcp/setup.sh +++ b/gcp/setup.sh @@ -1,4 +1,19 @@ #!/bin/bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # ansi colors GREEN='\033[0;32m' CYAN='\033[0;36m' @@ -9,7 +24,9 @@ SETTING_FILE="./settings.ini" SCRIPT_PATH=$(readlink -f "$0" | xargs dirname) SETTING_FILE="${SCRIPT_PATH}/settings.ini" -# changing the cwd to the script's contining folder so all pathes inside can be local to it +trap _upload_install_log EXIT + +# changing the cwd to the script's containing folder so all pathes inside can be local to it # (important as the script can be called via absolute path and as a nested path) pushd $SCRIPT_PATH >/dev/null @@ -25,21 +42,62 @@ while :; do shift done +NAME=$(git config -f $SETTING_FILE config.name) +PROJECT_ID=$(gcloud config get-value project 2> /dev/null) +PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="csv(projectNumber)" | tail -n 1) +USER_EMAIL=$(gcloud config get-value account 2> /dev/null) + +APP_CONFIG_FILE=$(eval echo $(git config -f $SETTING_FILE config.config-file)) REPOSITORY=$(eval echo $(git config -f $SETTING_FILE repository.name)) IMAGE_NAME=$(eval echo $(git config -f $SETTING_FILE repository.image)) REPOSITORY_LOCATION=$(git config -f $SETTING_FILE repository.location) TOPIC=$(eval echo $(git config -f $SETTING_FILE pubsub.topic)) -NAME=$(git config -f $SETTING_FILE config.name) -PROJECT_ID=$(gcloud config get-value project 2> /dev/null) -PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="csv(projectNumber)" | tail -n 1) SERVICE_ACCOUNT=$PROJECT_NUMBER-compute@developer.gserviceaccount.com +check_billing() { + BILLING_ENABLED=$(gcloud beta billing projects describe $PROJECT_ID --format="csv(billingEnabled)" | tail -n 1) + if [[ "$BILLING_ENABLED" = 'False' ]] + then + echo -e "${RED}The project $PROJECT_ID does not have a billing enabled. Please activate billing${NC}" + exit -1 + fi +} + +deploy_files() { + echo 'Deploying files to GCS' + if ! gsutil ls gs://$PROJECT_ID > /dev/null 2> /dev/null; then + echo "Creating GCS bucket gs://$PROJECT_ID" + gsutil mb -b on gs://$PROJECT_ID + fi + + GCS_BASE_PATH=gs://$PROJECT_ID/$NAME + + # NOTE: DO NOT add -m flag for gsutil! When executed under cloudshell_open (via Cloud Run Button) it won't copy files + echo "Removing existing files at $GCS_BASE_PATH" + gsutil rm -r $GCS_BASE_PATH/ + + # NOTE: if an error "module 'sys' has no attribute 'maxint'" occures, run this: `pip3 install -U crcmod` + echo "Copying application files to $GCS_BASE_PATH" + gsutil rsync -r -x ".*/__pycache__/.*|[.].*" ./../app $GCS_BASE_PATH + echo "Copying configs to $GCS_BASE_PATH" + gsutil -h "Content-Type:text/plain" cp ./../app/*.yaml $GCS_BASE_PATH/ + if [[ -f ./../google-ads.yaml ]]; then + gsutil -h "Content-Type:text/plain" cp ./../google-ads.yaml $GCS_BASE_PATH/google-ads.yaml + elif [[ -f $HOME/google-ads.yaml ]]; then + gsutil -h "Content-Type:text/plain" cp $HOME/google-ads.yaml $GCS_BASE_PATH/google-ads.yaml + else + echo "Please upload google-ads.yaml" + fi +} + enable_apis() { echo "Enabling APIs" + gcloud services enable bigquery.googleapis.com gcloud services enable compute.googleapis.com - gcloud services enable artifactregistry.googleapis.com + #gcloud services enable artifactregistry.googleapis.com + gcloud services enable containerregistry.googleapis.com gcloud services enable run.googleapis.com gcloud services enable cloudresourcemanager.googleapis.com gcloud services enable iamcredentials.googleapis.com @@ -47,13 +105,14 @@ enable_apis() { gcloud services enable cloudfunctions.googleapis.com gcloud services enable eventarc.googleapis.com gcloud services enable cloudscheduler.googleapis.com + gcloud services enable googleads.googleapis.com } create_registry() { - echo "Creating a repository in Artifact Registry" - REPO_EXISTS=$(gcloud artifacts repositories list --location=europe --filter="REPOSITORY:'"$REPOSITORY"'" --format="value(REPOSITORY)" 2>/dev/null) + REPO_EXISTS=$(gcloud artifacts repositories list --location=$REPOSITORY_LOCATION --filter="REPOSITORY=projects/'$PROJECT_ID'/locations/'$REPOSITORY_LOCATION'/repositories/'"$REPOSITORY"'" --format="value(REPOSITORY)" 2>/dev/null) if [[ ! -n $REPO_EXISTS ]]; then + echo "Creating a repository in Artifact Registry" # repo doesn't exist, creating gcloud artifacts repositories create ${REPOSITORY} \ --repository-format=docker \ @@ -75,26 +134,30 @@ build_docker_image() { build_docker_image_gcr() { # NOTE: it's an alternative to build_docker_image if you want to use GCR instead of AR echo "Building and pushing Docker image to Container Registry" - gcloud builds submit --config=cloudbuild-gcr.yaml --substitutions=_IMAGE="workload" ./workload-vm + gcloud builds submit --config=cloudbuild-gcr.yaml --substitutions=_IMAGE="$IMAGE_NAME" ./.. } set_iam_permissions() { + required_roles="storage.objectViewer artifactregistry.repoAdmin compute.admin monitoring.editor logging.logWriter iam.serviceAccountTokenCreator pubsub.publisher run.invoker" echo "Setting up IAM permissions" - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/storage.objectViewer - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/artifactregistry.repoAdmin - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/compute.admin - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/monitoring.editor + for role in $required_roles; do + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member=serviceAccount:$SERVICE_ACCOUNT \ + --role=roles/$role \ + --no-user-output-enabled + done } create_topic() { - TOPIC_EXISTS=$(gcloud pubsub topics list --filter="name.scope(topic):'$TOPIC'" --format="get(name)") + TOPIC_EXISTS=$(gcloud pubsub topics list --filter="name=projects/'$PROJECT_ID'/topics/'$TOPIC'" --format="get(name)") if [[ ! -n $TOPIC_EXISTS ]]; then gcloud pubsub topics create $TOPIC fi } + deploy_cf() { echo "Deploying Cloud Function" CF_REGION=$(git config -f $SETTING_FILE function.region) @@ -109,7 +172,8 @@ deploy_cf() { fi # initialize env.yaml - environment variables for CF: # - docker image url - url="$REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/docker/$IMAGE_NAME" + #url="$REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/docker/$IMAGE_NAME" + url="gcr.io/$PROJECT_ID/$IMAGE_NAME" sed -i'.bak' -e "s|#*[[:space:]]*DOCKER_IMAGE[[:space:]]*:[[:space:]]*.*$|DOCKER_IMAGE: $url|" ./cloud-functions/create-vm/env.yaml # - GCE VM name (base) instance=$(eval echo $(git config -f $SETTING_FILE compute.name)) @@ -150,41 +214,17 @@ deploy_cf() { } -deploy_files() { - echo 'Deploying files to GCS' - if ! gsutil ls gs://$PROJECT_ID > /dev/null 2> /dev/null; then - echo "Creating GCS bucket gs://$PROJECT_ID" - gsutil mb -b on gs://$PROJECT_ID - fi - - GCS_BASE_PATH=gs://$PROJECT_ID/$NAME - - # NOTE: DO NOT add -m flag for gsutil! When executed under cloudshell_open (via Cloud Run Button) it won't copy files - echo "Removing existing files at $GCS_BASE_PATH" - gsutil rm -r $GCS_BASE_PATH/ - - # NOTE: if an error "module 'sys' has no attribute 'maxint'" occures, run this: `pip3 install -U crcmod` - echo "Copying application files to $GCS_BASE_PATH" - gsutil rsync -r -x ".*/__pycache__/.*|[.].*" ./../app $GCS_BASE_PATH - echo "Copying configs to $GCS_BASE_PATH" - gsutil -h "Content-Type:text/plain" cp ./../app/*.yaml $GCS_BASE_PATH/ - if [[ -f ./../google-ads.yaml ]]; then - gsutil -h "Content-Type:text/plain" cp ./../google-ads.yaml $GCS_BASE_PATH/google-ads.yaml - elif [[ -f $HOME/google-ads.yaml ]]; then - gsutil -h "Content-Type:text/plain" cp $HOME/google-ads.yaml $GCS_BASE_PATH/google-ads.yaml - else - echo "Please upload google-ads.yaml" - fi -} - deploy_public_index() { echo 'Deploying index.html to GCS' - gsutil mb -b on gs://${PROJECT_ID}-public + if ! gsutil ls gs://$PROJECT_ID-public > /dev/null 2> /dev/null; then + gsutil mb -b on gs://$PROJECT_ID-public + fi + gsutil iam ch -f allUsers:objectViewer gs://${PROJECT_ID}-public 2> /dev/null exitcode=$? if [ $exitcode -ne 0 ]; then - echo "Could not add public access to public cloud bucket" + echo -e "${RED}[ ! ] Could not add public access to public cloud bucket${NC}" else GCS_BASE_PATH_PUBLIC=gs://${PROJECT_ID}-public/$NAME gsutil -h "Content-Type:text/html" -h "Cache-Control: no-store" cp "${SCRIPT_PATH}/index.html" $GCS_BASE_PATH_PUBLIC/index.html @@ -196,16 +236,19 @@ deploy_public_index() { get_run_data() { + local dashboard_url="$1" # arguments for the CF (to be passed via pubsub message or scheduler job's arguments): # * project_id # * machine_type # * service_account - # * gcs_source_uri - # * gcs_base_path_public # * docker_image - a docker image url, can be CR or AR # gcr.io/$PROJECT_ID/workload # europe-docker.pkg.dev/$PROJECT_ID/docker/workload - # * delete_vm - by default it's TRUE (set inside create-vm CF) + # * vm - an object with attributes for VM (they will be passed to main.sh via VM's metadata): + # * gcs_source_uri + # * gcs_base_path_public + # * create_dashboard_link + # * delete_vm - by default it's TRUE (set inside create-vm CF) GCS_BASE_PATH=gs://$PROJECT_ID/$NAME GCS_BASE_PATH_PUBLIC=gs://${PROJECT_ID}-public/$NAME @@ -216,9 +259,12 @@ get_run_data() { # if you need to prevent VM deletion add this: # "delete_vm": "FALSE" data='{ - "gcs_source_uri": "'$GCS_BASE_PATH'", - "gcs_base_path_public": "'$GCS_BASE_PATH_PUBLIC'", - "delete_vm": "TRUE" + "vm": { + "gcs_source_uri": "'$GCS_BASE_PATH'", + "gcs_base_path_public": "'$GCS_BASE_PATH_PUBLIC'", + "create_dashboard_url": "'$dashboard_url'", + "delete_vm": "TRUE" + } }' echo $data } @@ -234,24 +280,27 @@ start() { # example: # --message="{\"project_id\":\"$PROJECT_ID\", \"docker_image\":\"europe-docker.pkg.dev/$PROJECT_ID/docker/workload\", \"service_account\":\"$SERVICE_ACCOUNT\"}" - local DATA=$(get_run_data) + dashboard_url=$(./../app/scripts/create_dashboard.sh -L --config ./../app/$APP_CONFIG_FILE) + + local DATA=$(get_run_data $dashboard_url) echo 'Publishing a pubsub with args: '$DATA gcloud pubsub topics publish $TOPIC --message="$DATA" # Check if there is a public bucket and index.html and echo the url local PUBLIC_URL=$(print_public_gcs_url)/index.html + local GOOGLE_GROUP="https://groups.google.com/g/dactionboard" + echo -e "${CYAN}[ * ] Please join Google group to get access to the dashboard - ${GREEN}${GOOGLE_GROUP}${NC}" STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" $PUBLIC_URL) if [[ $STATUS_CODE -eq 200 ]]; then echo -e "${CYAN}[ * ] To access your new dashboard, click this link - ${GREEN}${PUBLIC_URL}${NC}" else echo -e "${CYAN}[ * ] Your GCP project does not allow public access.${NC}" - if [[ -f ./../dactionboard.yaml ]]; then - dashboard_url=$(./../scripts/create_dashboard.sh -L --config dactionboard.yaml) + if [[ -f ./../app/$APP_CONFIG_FILE ]]; then echo -e "${CYAN}[ * ] To create your dashboard, click the following link once the installation process completes and all the relevant tables have been created in the DB:" echo -e "${GREEN}$dashboard_url${NC}" else - echo -e "${CYAN}[ * ] To create your dashboard, please run the ${GREEN}./scripts/create_dashboard.sh -c dactionboard.yaml -L${CYAN} shell script once the installation process completes and all the relevant tables have been created in the DB.${NC}" + echo -e "${CYAN}[ * ] To create your dashboard, please run the ${GREEN}./scripts/create_dashboard.sh -c $APP_CONFIG_FILE -L${CYAN} shell script once the installation process completes and all the relevant tables have been created in the DB.${NC}" fi fi } @@ -274,7 +323,6 @@ schedule_run() { delete_schedule echo 'Scheduling a job with args: '$DATA - gcloud scheduler jobs create pubsub $JOB_NAME \ --schedule="$SCHEDULE" \ --location=$REGION \ @@ -283,11 +331,12 @@ schedule_run() { --time-zone=$SCHEDULE_TZ } + delete_schedule() { JOB_NAME=$(eval echo $(git config -f $SETTING_FILE scheduler.name)) REGION=$(git config -f $SETTING_FILE scheduler.region) - JOB_EXISTS=$(gcloud scheduler jobs list --location=$REGION --format="value(ID)" --filter="ID:'$JOB_NAME'" 2>/dev/null) + JOB_EXISTS=$(gcloud scheduler jobs list --location=$REGION --format="value(ID)" --filter="ID=projects/'$PROJECT_ID'/locations/'$REGION'/jobs/'$JOB_NAME'" 2>/dev/null) if [[ -n $JOB_EXISTS ]]; then echo 'Deleting Cloud Scheduler job '$JOB_NAME gcloud scheduler jobs delete $JOB_NAME --location $REGION --quiet @@ -298,16 +347,39 @@ enable_private_google_access() { REGION=$(git config -f $SETTING_FILE compute.region) gcloud compute networks subnets update default --region=$REGION --enable-private-ip-google-access } + +check_owners() { + local project_admins=$(gcloud projects get-iam-policy $PROJECT_ID \ + --flatten="bindings" \ + --filter="bindings.role=roles/owner" \ + --format="value(bindings.members[])" + ) + if [[ ! $project_admins =~ $USER_EMAIL ]]; then + echo "User $USER_EMAIL does not have admin right to project $PROJECT_ID" + exit + fi +} + + deploy_all() { + check_owners + check_billing enable_apis set_iam_permissions deploy_files - create_registry - build_docker_image + #create_registry - uncomment then migrated back to Artifact Registry + #build_docker_image - uncomment to migrated back to Artifact Registry + build_docker_image_gcr # using Container Registry as most reliable service but it's only safe till May 2024 (due to upcomming deprication) deploy_cf schedule_run } +_upload_install_log() { + if [[ -f "/tmp/${NAME}_installer.log" ]]; then + gsutil cp /tmp/${NAME}_installer.log gs://$PROJECT_ID/$NAME/ + rm "/tmp/${NAME}_installer.log" + fi +} _list_functions() { # list all functions in this file not starting with "_" @@ -320,7 +392,7 @@ if [[ $# -eq 0 ]]; then else for i in "$@"; do if declare -F "$i" > /dev/null; then - "$i" + "$i" 2>&1 | tee -a /tmp/${NAME}_installer.log exitcode=$? if [ $exitcode -ne 0 ]; then echo "Breaking script as command '$i' failed" diff --git a/gcp/workload-vm/Dockerfile b/gcp/workload-vm/Dockerfile index b923b3a..391c607 100644 --- a/gcp/workload-vm/Dockerfile +++ b/gcp/workload-vm/Dockerfile @@ -1,11 +1,12 @@ FROM google/cloud-sdk WORKDIR /app -COPY requirements.txt . -RUN python3 -m pip install --require-hashes -r requirements.txt --no-deps +COPY ./app/requirements.txt . +RUN python3 -m pip install -r requirements.txt --require-hashes --no-deps -COPY run-local.sh . +COPY gcp/workload-vm/start.sh start.sh COPY gcp/workload-vm/main.sh main.sh +COPY gcp/settings.ini settings.ini # Run the app ENTRYPOINT ["./main.sh"] diff --git a/gcp/workload-vm/main.sh b/gcp/workload-vm/main.sh index fdf6049..7fefbdc 100755 --- a/gcp/workload-vm/main.sh +++ b/gcp/workload-vm/main.sh @@ -1,6 +1,6 @@ #!/bin/bash -LOG_NAME=dactionboard-vm +LOG_NAME=$(git config -f "./settings.ini" config.name) echo "Starting entrypoint script" @@ -12,13 +12,13 @@ gcloud config set project $project_id # Fetch gcs uris fro the current instance metadata gcs_source_uri=$(curl -H Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcs_source_uri -s --fail) -gcloud logging write $LOG_NAME "[$(hostname)] Starting dActionBoard application (gcs_source_uri: $gcs_source_uri)" +gcloud logging write $LOG_NAME "[$(hostname)] Starting application (gcs_source_uri: $gcs_source_uri)" -# fetch dActionBoard files from GCS +# fetch application files from GCS if [[ -n $gcs_source_uri ]]; then folder_name=$(basename "$gcs_source_uri") gsutil -m cp -R $gcs_source_uri . - mv "$folder_name/*" . + mv $folder_name/* . fi # run the application @@ -31,23 +31,19 @@ do done < $LOG_NAME.log if [ $exitcode -ne 0 ]; then - gcloud logging write $LOG_NAME "[$(hostname)] dActionBoard application has finished execution with an error ($exitcode)" --severity ERROR + gcloud logging write $LOG_NAME "[$(hostname)] Application has finished execution with an error ($exitcode)" --severity ERROR # TODO: send the error somewhere else - gcloud logging write $LOG_NAME "[$(hostname)] dActionBoard application has finished execution successfully" + gcloud logging write $LOG_NAME "[$(hostname)] Application has finished execution successfully" fi # Check if index.html exists in the bucket. If so - create and upload dashboard.json gcs_base_path_public=$(curl -H Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcs_base_path_public -s --fail) -if [[ -n gcs_base_path_public ]]; then - # TODO: if run-local.sh failed we shouldn't create dashboard_url - if gsutil ls $gcs_base_path_public/index.html >/dev/null 2>&1; then - chmod +x ./scripts/create_dashboard.sh - dashboard_url=$(./scripts/create_dashboard.sh -L --config dactionboard.yaml) - echo "Created dashboard cloning url: $dashboard_url" - echo "{\"dashboardUrl\":\"$dashboard_url\"}" > dashboard.json - gsutil -h "Content-Type:application/json" -h "Cache-Control: no-store" cp dashboard.json $gcs_base_path_public/dashboard.json - fi +create_dashboard_url=$(curl -H Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/attributes/create_dashboard_url -s --fail) +if [[ $exitcode -eq 0 && -n "$gcs_base_path_public" && -n "$create_dashboard_url" ]]; then + echo "{\"dashboardUrl\":\"$create_dashboard_url\"}" > dashboard.json + echo "Created dashboard.json with cloning url: $create_dashboard_link" + gsutil -h "Content-Type:application/json" -h "Cache-Control: no-store" cp dashboard.json $gcs_base_path_public/dashboard.json fi # Delete the VM (fetch a custom metadata key, it can be absent, so returns 404 - handling it with --fail options) diff --git a/gcp/workload-vm/start.sh b/gcp/workload-vm/start.sh index c6837b4..a24e09e 100755 --- a/gcp/workload-vm/start.sh +++ b/gcp/workload-vm/start.sh @@ -4,4 +4,3 @@ APP_CONFIG_FILE=$(git config -f "./settings.ini" config.config-file) chmod +x ./run-local.sh ./run-local.sh --quiet --config $APP_CONFIG_FILE --google-ads-config google-ads.yaml -