From b31b4a16cb60ce41c03800bc36b175e29e408405 Mon Sep 17 00:00:00 2001 From: Sergei Dorogin Date: Thu, 15 Aug 2024 15:30:44 +0300 Subject: [PATCH] gaarf-wf: setup.sh: refactored set_iam_permissions; restored support for MEMORY_GETCIDS (memory for getcids function); added run_job task; deploy_all stops executing tasks on first failure Change-Id: I30bca2341dc036addd78e5bb9225ccbbd57c51ea --- gcp/setup.sh | 80 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/gcp/setup.sh b/gcp/setup.sh index 96ea8abd..12ec1fc4 100755 --- a/gcp/setup.sh +++ b/gcp/setup.sh @@ -62,28 +62,34 @@ enable_apis() { set_iam_permissions() { - # Permissions for Workflow: - # grant the default service account with read permissions on Cloud Storage - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/storage.objectViewer - # grant the default service account with execute permissions on Cloud Functions gen2 (CF) - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/run.invoker - # grant the default service account with write permissions on Cloud Logging - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/logging.logWriter - # grant the default service account with view permissions (cloudfunctions.functions.get) on CF - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/cloudfunctions.viewer - # grant the default service account with execute permissions on Cloud Workflow (this is for Scheduler which also runs under the default GCE SA) - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role roles/workflows.invoker - # grant the default service account with permissions on to publish pubsub messages - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role roles/pubsub.admin #topics.publish - # Permissions for Cloud Functions: - # Required for calling Cloud Function Gen2 - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/cloudfunctions.serviceAgent - # Grant the default service account with read permissions on Cloud Storage - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/storage.objectViewer - # Grant the default service account with admin permissions in BigQuery - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/bigquery.admin - # For deploying Gen2 CF 'artifactregistry.repositories.list' and 'artifactregistry.repositories.get' permissions are required - gcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$SERVICE_ACCOUNT --role=roles/artifactregistry.repoAdmin + echo -e "${COLOR}Setting up IAM permissions...${NC}" + declare -ar ROLES=( + # Permissions for Workflow: + # Cloud Storage + roles/storage.admin + # Cloud Functions gen2 (CF) + roles/run.invoker + # Cloud Logging + roles/logging.logWriter + # view permissions (cloudfunctions.functions.get) on CF + roles/cloudfunctions.viewer + #execute permissions on Cloud Workflow (this is for Scheduler which also runs under the default GCE SA) + roles/workflows.invoker + # to publish pubsub messages + roles/pubsub.admin + roles/cloudfunctions.serviceAgent + roles/bigquery.admin + # For deploying Gen2 CF 'artifactregistry.repositories.list' and 'artifactregistry.repositories.get' permissions are required + roles/artifactregistry.repoAdmin + roles/iam.serviceAccountUser + ) + for role in "${ROLES[@]}" + do + gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member=serviceAccount:$SERVICE_ACCOUNT \ + --role=$role \ + --condition=None + done } @@ -103,20 +109,26 @@ deploy_functions() { CF_MEMORY="--memory $CF_MEMORY" fi - ./functions/deploy.sh --name $NAME $CF_MEMORY --region $REGION --service-account $SERVICE_ACCOUNT + MEMORY_GETCIDS=$(git config -f $SETTING_FILE functions.memory-getcids) + if [[ -n $MEMORY_GETCIDS ]]; then + MEMORY_GETCIDS="--memory-getcids $MEMORY_GETCIDS" + fi + ./functions/deploy.sh --name $NAME $CF_MEMORY $MEMORY_GETCIDS --region $REGION --service-account $SERVICE_ACCOUNT + return $? } enable_debug_logging() { - gcloud functions deploy $NAME --update-env-vars LOG_LEVEL=debug --region $REGION - gcloud functions deploy $NAME-getcids --update-env-vars LOG_LEVEL=debug --region $REGION - gcloud functions deploy $NAME-bq --update-env-vars LOG_LEVEL=debug --region $REGION - gcloud functions deploy $NAME-bq-view --update-env-vars LOG_LEVEL=debug --region $REGION + gcloud functions deploy $NAME --update-env-vars LOG_LEVEL=debug --region $REGION --source ./functions --runtime=nodejs20 + gcloud functions deploy $NAME-getcids --update-env-vars LOG_LEVEL=debug --region $REGION --source ./functions --runtime=nodejs20 + gcloud functions deploy $NAME-bq --update-env-vars LOG_LEVEL=debug --region $REGION --source ./functions --runtime=nodejs20 + gcloud functions deploy $NAME-bq-view --update-env-vars LOG_LEVEL=debug --region $REGION --source ./functions --runtime=nodejs20 } deploy_wf() { ./workflow/deploy.sh --name $WORKFLOW_NAME --region $REGION --service-account $SERVICE_ACCOUNT + return $? } @@ -154,6 +166,7 @@ schedule_wf() { --message-body="{\"argument\": \"$escaped_data\"}" \ --oauth-service-account-email="$SERVICE_ACCOUNT" \ --time-zone="$SCHEDULE_TZ" + return $? } @@ -169,6 +182,9 @@ run_wf() { fi } +run_job() { + gcloud scheduler jobs run $WORKFLOW_NAME --location=$REGION +} delete_all() { echo "Deleting Functions" @@ -188,11 +204,11 @@ delete_all() { deploy_all() { - enable_apis - set_iam_permissions - deploy_functions - create_wf_completion_topic - deploy_wf + enable_apis || return $? + set_iam_permissions || return $? + deploy_functions || return $? + create_wf_completion_topic || return $? + deploy_wf || return $? }