From 63f524d55c7edd4cda4717889864d3bf1c200aa8 Mon Sep 17 00:00:00 2001 From: bendnorman Date: Tue, 10 Sep 2024 16:14:06 -0800 Subject: [PATCH 1/4] Add pudl usage metrics gcp infrastructure - Mount pudl-usage-metrics db to superset cloud run - Create a new bucket for raw usage metrics archives - Give github action service account permision to write to the bucket --- terraform/main.tf | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/terraform/main.tf b/terraform/main.tf index 7ba0b9ee48..4ad8e7bf57 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -228,7 +228,7 @@ resource "google_cloud_run_v2_service" "pudl-superset" { volumes { name = "cloudsql" cloud_sql_instance { - instances = ["catalyst-cooperative-pudl:us-central1:superset-database"] + instances = ["catalyst-cooperative-pudl:us-central1:superset-database", "catalyst-cooperative-pudl:us-central1:pudl-usage-metrics-db"] } } } @@ -396,3 +396,31 @@ resource "google_service_account_iam_member" "gce-default-account-iam" { role = "roles/iam.serviceAccountUser" member = "serviceAccount:345950277072@cloudbuild.gserviceaccount.com" } + +resource "google_secret_manager_secret" "pudl_usage_metrics_db_connection_string" { + secret_id = "pudl-usage-metrics-db-connection-string" + replication { + auto {} + } +} + +resource "google_storage_bucket" "pudl_usage_metrics_archive_bucket" { + name = "pudl-usage-metrics-archives.catalyst.coop" + location = "US" + storage_class = "STANDARD" + + uniform_bucket_level_access = true +} + +resource "google_service_account" "usage_metrics_archiver" { + account_id = "usage-metrics-archiver" + display_name = "PUDL usage metrics archiver github action service account" +} + +resource "google_storage_bucket_iam_member" "usage_metrics_archiver_gcs_iam" { + for_each = toset(["roles/storage.objectCreator", "roles/storage.objectViewer"]) + + bucket = google_storage_bucket.pudl_usage_metrics_archive_bucket.name + role = each.key + member = "serviceAccount:${google_service_account.usage_metrics_archiver.email}" +} From 1aaac08c7c3fcd5bc59434677fce3dd0fcb7732d Mon Sep 17 00:00:00 2001 From: bendnorman Date: Wed, 11 Sep 2024 10:27:47 -0800 Subject: [PATCH 2/4] Give pudl usage metrics etl service account permission to read from archive bucket --- terraform/main.tf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/terraform/main.tf b/terraform/main.tf index 4ad8e7bf57..4778108ce7 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -424,3 +424,10 @@ resource "google_storage_bucket_iam_member" "usage_metrics_archiver_gcs_iam" { role = each.key member = "serviceAccount:${google_service_account.usage_metrics_archiver.email}" } + +resource "google_storage_bucket_iam_member" "usage_metrics_etl_gcs_iam" { + + bucket = google_storage_bucket.pudl_usage_metrics_archive_bucket.name + role = "roles/storage.legacyBucketReader" + member = "serviceAccount:pudl-usage-metrics-etl@catalyst-cooperative-pudl.iam.gserviceaccount.com" +} From fe095b98644a816415ff4831bee680281cca9f41 Mon Sep 17 00:00:00 2001 From: bendnorman Date: Wed, 11 Sep 2024 10:40:39 -0800 Subject: [PATCH 3/4] Give pudl usage metrics etl service account permission to read from archive bucket and list buckets, previous commit was missing a role --- terraform/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/main.tf b/terraform/main.tf index 4778108ce7..e5406b9821 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -426,8 +426,9 @@ resource "google_storage_bucket_iam_member" "usage_metrics_archiver_gcs_iam" { } resource "google_storage_bucket_iam_member" "usage_metrics_etl_gcs_iam" { + for_each = toset(["roles/storage.legacyBucketReader", "roles/storage.objectViewer"]) bucket = google_storage_bucket.pudl_usage_metrics_archive_bucket.name - role = "roles/storage.legacyBucketReader" + role = each.key member = "serviceAccount:pudl-usage-metrics-etl@catalyst-cooperative-pudl.iam.gserviceaccount.com" } From 64234bae1aeb40db6402ef571987ab35fc0559c5 Mon Sep 17 00:00:00 2001 From: bendnorman Date: Wed, 11 Sep 2024 12:22:17 -0800 Subject: [PATCH 4/4] Give usage metrics service account permission on s3 logs bucket --- terraform/main.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/terraform/main.tf b/terraform/main.tf index e5406b9821..b0ec60ac9c 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -432,3 +432,11 @@ resource "google_storage_bucket_iam_member" "usage_metrics_etl_gcs_iam" { role = each.key member = "serviceAccount:pudl-usage-metrics-etl@catalyst-cooperative-pudl.iam.gserviceaccount.com" } + +resource "google_storage_bucket_iam_member" "usage_metrics_etl_s3_logs_gcs_iam" { + for_each = toset(["roles/storage.legacyBucketReader", "roles/storage.objectViewer"]) + + bucket = "pudl-s3-logs.catalyst.coop" + role = each.key + member = "serviceAccount:pudl-usage-metrics-etl@catalyst-cooperative-pudl.iam.gserviceaccount.com" +}