feat(module): Add ML Infra Projects Module and 4-projects refactor (#40)

* first commit * restore files * add new files * add copies * remove old files * add sa * add newline to EOF * add outputs.tf * remove remote.tff from inside module * chore(format): terraform fmt on dns module * add env kms project id as variable * generate docs * Revert "Merge branch 'format/fmt-dns-module' into refactor/ml-infra-projects" This reverts commit be0ba50, reversing changes made to 0d4f692. * update to 2024 * add title and description to ml single project * add project_name as variable on ml_single_project * add missing locals.tf * feat: parametrize prevent_destroy for project key * Update 4-projects/modules/ml_single_project/outputs.tf Co-authored-by: Daniel Andrade <[email protected]> * Update 4-projects/modules/ml_single_project/outputs.tf Co-authored-by: Daniel Andrade <[email protected]> * Update 4-projects/modules/ml_infra_projects/variables.tf Co-authored-by: Daniel Andrade <[email protected]> * chore: enabled_apis to locals * add space * Update 4-projects/modules/ml_single_project/outputs.tf Co-authored-by: Daniel Andrade <[email protected]> * Update 4-projects/modules/ml_single_project/variables.tf Co-authored-by: Daniel Andrade <[email protected]> * Update 4-projects/modules/ml_single_project/variables.tf Co-authored-by: Daniel Andrade <[email protected]> * Update 4-projects/modules/ml_single_project/variables.tf Co-authored-by: Daniel Andrade <[email protected]> * update description * chore: add trailing dots to where missing * remove default value * restore dns file * externalize project suffixes * externalize application name * add files again * remove files again --------- Co-authored-by: Daniel Andrade <[email protected]>
GoogleCloudPlatform · May 22, 2024 · fda06e1 · fda06e1
1 parent fe3b1b4
commit fda06e1
Show file tree

Hide file tree

Showing 19 changed files with 963 additions and 80 deletions.
diff --git a/4-projects/business_unit_3/shared/README.md b/4-projects/business_unit_3/shared/README.md
@@ -11,6 +11,7 @@
 | keyring\_name | Name to be used for KMS Keyring | `string` | `"sample-keyring"` | no |
 | location\_gcs | Case-Sensitive Location for GCS Bucket | `string` | `"US"` | no |
 | location\_kms | Case-Sensitive Location for KMS Keyring | `string` | `"us"` | no |
+| prevent\_destroy | Prevent Project Key destruction. | `bool` | `true` | no |
 | project\_budget | Budget configuration.<br>  budget\_amount: The amount to use as the budget.<br>  alert\_spent\_percents: A list of percentages of the budget to alert on when threshold is exceeded.<br>  alert\_pubsub\_topic: The name of the Cloud Pub/Sub topic where budget related messages will be published, in the form of `projects/{project_id}/topics/{topic_id}`.<br>  alert\_spend\_basis: The type of basis used to determine if spend has passed the threshold. Possible choices are `CURRENT_SPEND` or `FORECASTED_SPEND` (default). | <pre>object({<br>    budget_amount        = optional(number, 1000)<br>    alert_spent_percents = optional(list(number), [1.2])<br>    alert_pubsub_topic   = optional(string, null)<br>    alert_spend_basis    = optional(string, "FORECASTED_SPEND")<br>  })</pre> | `{}` | no |
 | remote\_state\_bucket | Backend bucket to load Terraform Remote State Data from previous steps. | `string` | n/a | yes |
 | tfc\_org\_name | Name of the TFC organization | `string` | `""` | no |

diff --git a/4-projects/business_unit_3/shared/ml_infra_projects.tf b/4-projects/business_unit_3/shared/ml_infra_projects.tf
@@ -0,0 +1,36 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "ml_infra_project" {
+  source = "../../modules/ml_infra_projects"
+
+  org_id                                 = local.org_id
+  folder_id                              = local.common_folder_name
+  billing_account                        = local.billing_account
+  environment                            = "common"
+  key_rings                              = local.shared_kms_key_ring
+  business_code                          = "bu3"
+  billing_code                           = "1234"
+  primary_contact                        = "[email protected]"
+  secondary_contact                      = "[email protected]"
+  cloud_source_artifacts_repo_name       = var.cloud_source_artifacts_repo_name
+  cloud_source_service_catalog_repo_name = var.cloud_source_service_catalog_repo_name
+  remote_state_bucket                    = var.remote_state_bucket
+  artifacts_infra_pipeline_sa            = module.infra_pipelines[0].terraform_service_accounts["bu3-artifact-publish"]
+  service_catalog_infra_pipeline_sa      = module.infra_pipelines[0].terraform_service_accounts["bu3-service-catalog"]
+  environment_kms_project_id             = ""
+  prevent_destroy                        = var.prevent_destroy
+}
diff --git a/4-projects/business_unit_3/shared/outputs.tf b/4-projects/business_unit_3/shared/outputs.tf
@@ -65,30 +65,30 @@ output "enable_cloudbuild_deploy" {
 
 output "service_catalog_project_id" {
   description = "Service Catalog Project ID."
-  value       = try(module.app_service_catalog_project[0].project_id, "")
+  value       = module.ml_infra_project.service_catalog_project_id
 }
 
 output "common_artifacts_project_id" {
   description = "App Infra Artifacts Project ID"
-  value       = try(module.app_infra_artifacts_project[0].project_id, "")
+  value       = module.ml_infra_project.common_artifacts_project_id
 }
 
 output "service_catalog_repo_name" {
   description = "The name of the Service Catalog repository"
-  value       = google_sourcerepo_repository.service_catalog.name
+  value       = module.ml_infra_project.service_catalog_repo_name
 }
 
 output "service_catalog_repo_id" {
   description = "ID of the Service Catalog repository"
-  value       = google_sourcerepo_repository.service_catalog.id
+  value       = module.ml_infra_project.service_catalog_repo_id
 }
 
 output "artifacts_repo_name" {
   description = "The name of the Artifacts repository"
-  value       = google_sourcerepo_repository.artifact_repo.name
+  value       = module.ml_infra_project.artifacts_repo_name
 }
 
 output "artifacts_repo_id" {
   description = "ID of the Artifacts repository"
-  value       = google_sourcerepo_repository.artifact_repo.id
+  value       = module.ml_infra_project.artifacts_repo_id
 }
diff --git a/4-projects/business_unit_3/shared/variables.tf b/4-projects/business_unit_3/shared/variables.tf
@@ -87,3 +87,9 @@ variable "cloud_source_artifacts_repo_name" {
   description = "Name to give the could source repository for Artifacts"
   type        = string
 }
+
+variable "prevent_destroy" {
+  description = "Prevent Project Key destruction."
+  type        = bool
+  default     = true
+}
diff --git a/...siness_unit_3/shared/example_artifacts.tf → ...es/ml_infra_projects/artifacts_project.tf b/...siness_unit_3/shared/example_artifacts.tf → ...es/ml_infra_projects/artifacts_project.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2021 Google LLC
+ * Copyright 2024 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,16 +27,15 @@ locals {
 
 }
 module "app_infra_artifacts_project" {
-  source = "../../modules/single_project"
-  # count  = local.enable_cloudbuild_deploy ? 1 : 0
+  source = "../ml_single_project"
 
-  org_id              = local.org_id
-  billing_account     = local.billing_account
-  folder_id           = local.common_folder_name
-  environment         = "common"
+  org_id              = var.org_id
+  billing_account     = var.billing_account
+  folder_id           = var.folder_id
+  environment         = var.environment
   project_budget      = var.project_budget
-  project_prefix      = local.project_prefix
-  key_rings           = local.shared_kms_key_ring
+  project_prefix      = var.project_prefix
+  key_rings           = var.key_rings
   remote_state_bucket = var.remote_state_bucket
   activate_apis = [
     "artifactregistry.googleapis.com",
@@ -49,48 +48,40 @@ module "app_infra_artifacts_project" {
     "sourcerepo.googleapis.com",
   ]
   # Metadata
-  project_suffix    = "artifacts"
-  application_name  = "app-infra-artifacts"
-  billing_code      = "1234"
-  primary_contact   = "[email protected]"
-  secondary_contact = "[email protected]"
-  business_code     = "bu3"
+  project_suffix             = var.artifacts_project_suffix
+  application_name           = var.artifacts_application_name
+  billing_code               = var.billing_code
+  primary_contact            = var.primary_contact
+  secondary_contact          = var.secondary_contact
+  business_code              = var.business_code
+  environment_kms_project_id = var.environment_kms_project_id
+  project_name               = "${var.project_prefix}-${local.env_code}-${var.business_code}${var.artifacts_project_suffix}"
+  prevent_destroy            = var.prevent_destroy
 }
 
-# resource "google_kms_crypto_key" "ml_key" {
-#   for_each        = toset(local.shared_kms_key_ring)
-#   name            = module.app_infra_artifacts_project[0].project_name
-#   key_ring        = each.key
-#   rotation_period = var.key_rotation_period
-#   lifecycle {
-#     prevent_destroy = false
-#   }
-# }
-
 resource "google_kms_crypto_key_iam_member" "ml_key" {
-  for_each      = module.app_infra_cloudbuild_project[0].kms_keys
+  for_each      = module.app_infra_artifacts_project.kms_keys
   crypto_key_id = each.value.id
   role          = "roles/cloudkms.admin"
-  member        = "serviceAccount:${module.infra_pipelines[0].terraform_service_accounts["bu3-artifact-publish"]}"
+  member        = "serviceAccount:${var.artifacts_infra_pipeline_sa}"
 }
 
 resource "google_project_iam_member" "artifact_tf_sa_roles" {
   for_each = toset(local.artifact_tf_sa_roles)
-  project  = module.app_infra_artifacts_project[0].project_id
+  project  = module.app_infra_artifacts_project.project_id
   role     = each.key
-  member   = "serviceAccount:${module.infra_pipelines[0].terraform_service_accounts["bu3-artifact-publish"]}"
+  member   = "serviceAccount:${var.artifacts_infra_pipeline_sa}"
 }
 
 // Add Service Agent for Cloud Build
 resource "google_project_iam_member" "artifact_cloudbuild_agent" {
-  project = module.app_infra_artifacts_project[0].project_id
+  project = module.app_infra_artifacts_project.project_id
   role    = "roles/secretmanager.secretAccessor"
-  member  = "serviceAccount:${module.app_infra_artifacts_project[0].project_number}@cloudbuild.gserviceaccount.com"
+  member  = "serviceAccount:${module.app_infra_artifacts_project.project_number}@cloudbuild.gserviceaccount.com"
 }
 
 // Add Repository for Artifact repo
-
 resource "google_sourcerepo_repository" "artifact_repo" {
-  project = module.app_infra_artifacts_project[0].project_id
+  project = module.app_infra_artifacts_project.project_id
   name    = var.cloud_source_artifacts_repo_name
 }
diff --git a/4-projects/modules/ml_infra_projects/locals.tf b/4-projects/modules/ml_infra_projects/locals.tf
@@ -0,0 +1,19 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  env_code = element(split("", var.environment), 0)
+}
diff --git a/4-projects/modules/ml_infra_projects/outputs.tf b/4-projects/modules/ml_infra_projects/outputs.tf
@@ -0,0 +1,45 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "service_catalog_project_id" {
+  description = "Service Catalog Project ID."
+  value       = try(module.app_service_catalog_project.project_id, "")
+}
+
+output "common_artifacts_project_id" {
+  description = "App Infra Artifacts Project ID."
+  value       = try(module.app_infra_artifacts_project.project_id, "")
+}
+
+output "service_catalog_repo_name" {
+  description = "The name of the Service Catalog repository."
+  value       = google_sourcerepo_repository.service_catalog.name
+}
+
+output "service_catalog_repo_id" {
+  description = "ID of the Service Catalog repository."
+  value       = google_sourcerepo_repository.service_catalog.id
+}
+
+output "artifacts_repo_name" {
+  description = "The name of the Artifacts repository."
+  value       = google_sourcerepo_repository.artifact_repo.name
+}
+
+output "artifacts_repo_id" {
+  description = "ID of the Artifacts repository."
+  value       = google_sourcerepo_repository.artifact_repo.id
+}
diff --git a/..._unit_3/shared/example_service_catalog.tf → ...infra_projects/service_catalog_project.tf b/..._unit_3/shared/example_service_catalog.tf → ...infra_projects/service_catalog_project.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2021 Google LLC
+ * Copyright 2024 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,16 +26,15 @@ locals {
 }
 
 module "app_service_catalog_project" {
-  source = "../../modules/single_project"
-  # count  = local.enable_cloudbuild_deploy ? 1 : 0
+  source = "../ml_single_project"
 
-  org_id              = local.org_id
-  billing_account     = local.billing_account
-  folder_id           = local.common_folder_name
-  environment         = "common"
+  org_id              = var.org_id
+  billing_account     = var.billing_account
+  folder_id           = var.folder_id
+  environment         = var.environment
   project_budget      = var.project_budget
-  project_prefix      = local.project_prefix
-  key_rings           = local.shared_kms_key_ring
+  project_prefix      = var.project_prefix
+  key_rings           = var.key_rings
   remote_state_bucket = var.remote_state_bucket
   activate_apis = [
     "logging.googleapis.com",
@@ -47,31 +46,34 @@ module "app_service_catalog_project" {
     "sourcerepo.googleapis.com",
   ]
   # Metadata
-  project_suffix    = var.cloud_source_service_catalog_repo_name
-  application_name  = "app-infra-ml"
-  billing_code      = "1234"
-  primary_contact   = "[email protected]"
-  secondary_contact = "[email protected]"
-  business_code     = "bu3"
+  project_suffix             = var.service_catalog_project_suffix
+  application_name           = var.service_catalog_application_name
+  billing_code               = var.billing_code
+  primary_contact            = var.primary_contact
+  secondary_contact          = var.secondary_contact
+  business_code              = var.business_code
+  environment_kms_project_id = var.environment_kms_project_id
+  project_name               = "${var.project_prefix}-${local.env_code}-${var.business_code}${var.service_catalog_project_suffix}"
+  prevent_destroy            = var.prevent_destroy
 }
 
 resource "google_kms_crypto_key_iam_member" "sc_key" {
-  for_each      = module.app_service_catalog_project[0].kms_keys
+  for_each      = module.app_service_catalog_project.kms_keys
   crypto_key_id = each.value.id
   role          = "roles/cloudkms.admin"
-  member        = "serviceAccount:${module.infra_pipelines[0].terraform_service_accounts["bu3-service-catalog"]}"
+  member        = "serviceAccount:${var.service_catalog_infra_pipeline_sa}"
 }
 
 // Grab Service Agent for Secret Manager
 resource "google_project_service_identity" "secretmanager_agent" {
   provider = google-beta
-  project  = module.app_service_catalog_project[0].project_id
+  project  = module.app_service_catalog_project.project_id
   service  = "secretmanager.googleapis.com"
 }
 
 // Add Secret Manager Service Agent to key with encrypt/decrypt permissions
 resource "google_kms_crypto_key_iam_member" "secretmanager_agent" {
-  for_each      = module.app_service_catalog_project[0].kms_keys
+  for_each      = module.app_service_catalog_project.kms_keys
   crypto_key_id = each.value.id
   role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
   member        = "serviceAccount:${google_project_service_identity.secretmanager_agent.email}"
@@ -80,55 +82,43 @@ resource "google_kms_crypto_key_iam_member" "secretmanager_agent" {
 // Grab Service Agent for Storage
 resource "google_project_service_identity" "storage" {
   provider = google-beta
-  project  = module.app_service_catalog_project[0].project_id
+  project  = module.app_service_catalog_project.project_id
   service  = "storage.googleapis.com"
 }
 // Add Service Agent for Storage
 resource "google_kms_crypto_key_iam_member" "storage_agent" {
-  for_each      = module.app_service_catalog_project[0].kms_keys
+  for_each      = module.app_service_catalog_project.kms_keys
   crypto_key_id = each.value.id
   role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
-  member        = "serviceAccount:service-${module.app_service_catalog_project[0].project_number}@gs-project-accounts.iam.gserviceaccount.com"
+  member        = "serviceAccount:service-${module.app_service_catalog_project.project_number}@gs-project-accounts.iam.gserviceaccount.com"
 
   depends_on = [google_project_service_identity.storage]
 }
 
 // Add infra pipeline SA encrypt/decrypt permissions
 resource "google_kms_crypto_key_iam_member" "storage-kms-key-binding" {
-  for_each      = module.app_service_catalog_project[0].kms_keys
+  for_each      = module.app_service_catalog_project.kms_keys
   crypto_key_id = each.value.id
   role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
-  member        = "serviceAccount:${module.infra_pipelines[0].terraform_service_accounts["bu3-service-catalog"]}"
+  member        = "serviceAccount:${var.service_catalog_infra_pipeline_sa}"
 }
 
 resource "google_project_iam_member" "service_catalog_tf_sa_roles" {
   for_each = toset(local.service_catalog_tf_sa_roles)
-  project  = module.app_service_catalog_project[0].project_id
+  project  = module.app_service_catalog_project.project_id
   role     = each.key
-  member   = "serviceAccount:${module.infra_pipelines[0].terraform_service_accounts["bu3-service-catalog"]}"
+  member   = "serviceAccount:${var.service_catalog_infra_pipeline_sa}"
 }
 
 // Add Service Agent for Cloud Build
 resource "google_project_iam_member" "cloudbuild_agent" {
-  project = module.app_service_catalog_project[0].project_id
+  project = module.app_service_catalog_project.project_id
   role    = "roles/secretmanager.secretAccessor"
-  member  = "serviceAccount:${module.app_service_catalog_project[0].project_number}@cloudbuild.gserviceaccount.com"
+  member  = "serviceAccount:${module.app_service_catalog_project.project_number}@cloudbuild.gserviceaccount.com"
 }
 
 // Add Service Catalog Source Repository
-
 resource "google_sourcerepo_repository" "service_catalog" {
-  project = module.app_service_catalog_project[0].project_id
+  project = module.app_service_catalog_project.project_id
   name    = var.cloud_source_service_catalog_repo_name
 }
-
-/**
- * When Jenkins CICD is used for deployment this resource
- * is created to terraform validation works.
- * Without this resource, this module creates zero resources
- * and it breaks terraform validation throwing the error below:
- * ERROR: [Terraform plan json does not contain resource_changes key]
- */
-resource "null_resource" "jenkins_cicd_service_catalog" {
-  count = !local.enable_cloudbuild_deploy ? 1 : 0
-}