From c6ce99e4acbb4b39f853116238fa85b0f637f71c Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Thu, 20 Jun 2024 14:18:09 -0300 Subject: [PATCH 1/2] refactor: update notebook modules, remove filter by labels --- .../modules/notebook/README.md | 22 ++++--- .../service-catalog/modules/notebook/data.tf | 27 ++------ .../service-catalog/modules/notebook/main.tf | 1 + .../modules/notebook/variables.tf | 62 ++++++++++--------- 4 files changed, 52 insertions(+), 60 deletions(-) diff --git a/5-app-infra/source_repos/service-catalog/modules/notebook/README.md b/5-app-infra/source_repos/service-catalog/modules/notebook/README.md index 4c445f0d..bd38aee5 100644 --- a/5-app-infra/source_repos/service-catalog/modules/notebook/README.md +++ b/5-app-infra/source_repos/service-catalog/modules/notebook/README.md @@ -103,24 +103,26 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| accelerator\_type | The type of accelerator to use | `string` | `"NVIDIA_TESLA_K80"` | no | -| boot\_disk\_size\_gb | (Optional) The size of the boot disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB) | `string` | `"150"` | no | -| boot\_disk\_type | Possible disk types for notebook instances | `string` | `"PD_SSD"` | no | +| accelerator\_type | The type of accelerator to use. | `string` | `"NVIDIA_TESLA_K80"` | no | +| boot\_disk\_size\_gb | (Optional) The size of the boot disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB). | `string` | `"150"` | no | +| boot\_disk\_type | Possible disk types for notebook instances. | `string` | `"PD_SSD"` | no | | boundry\_code | The boundry code for the tenant | `string` | `"001"` | no | -| core\_count | number of accelerators to use | `number` | `1` | no | +| core\_count | Number of accelerators to use. | `number` | `1` | no | | data\_disk\_size\_gb | (Optional) The size of the data disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB) | `string` | `"150"` | no | -| data\_disk\_type | Optional. Input only. Indicates the type of the disk. Possible values are: PD\_STANDARD, PD\_SSD, PD\_BALANCED, PD\_EXTREME. | `string` | `"PD_SSD"` | no | +| data\_disk\_type | (Optional) Input only. Indicates the type of the disk. Possible values are: PD\_STANDARD, PD\_SSD, PD\_BALANCED, PD\_EXTREME. | `string` | `"PD_SSD"` | no | | disable\_proxy\_access | (Optional) The notebook instance will not register with the proxy | `bool` | `false` | no | | image\_family | Use this VM image family to find the image; the newest image in this family will be used. | `string` | `"workbench-instances"` | no | | image\_name | Use VM image name to find the image. | `string` | `""` | no | -| image\_project | The name of the Google Cloud project that this VM image belongs to. Format: projects/{project\_id} | `string` | `"cloud-notebooks-managed"` | no | +| image\_project | The name of the Google Cloud project that this VM image belongs to. Format: projects/{project\_id}. | `string` | `"cloud-notebooks-managed"` | no | | install\_gpu\_driver | Whether the end user authorizes Google Cloud to install GPU driver on this instance. Only applicable to instances with GPUs. | `bool` | `false` | no | -| instance\_owners | email of the owner of the instance, e.g. alias@example.com. Only one owner is supported! | `set(string)` | n/a | yes | +| instance\_owners | Email of the owner of the instance, e.g. alias@example.com. Only one owner is supported! | `set(string)` | n/a | yes | +| kms\_keyring | The KMS keyring that will be used when selecting the KMS key, preferably this should be on the same region as var.location and the same environment.
This value can be obtained by running "gcloud kms keyrings list --project=KMS\_PROJECT\_ID --location=REGION". | `string` | n/a | yes | | location | Notebook instance location (zone). | `string` | `"us-central1-a"` | no | -| machine\_type | type of the machine to spin up for the notebook | `string` | `"e2-standard-4"` | no | -| name | name of the notebook instance | `string` | n/a | yes | -| project\_id | Optional Project ID. | `string` | `null` | no | +| machine\_type | Type of the machine to spin up for the notebook. | `string` | `"e2-standard-4"` | no | +| name | Name of the notebook instance. | `string` | n/a | yes | +| project\_id | Project ID to deploy the instance. | `string` | n/a | yes | | tags | The Compute Engine tags to add to instance. | `list(string)` | 
[
  "egress-internet"
]
| no | +| vpc\_project | This is the project id of the Restricted Shared VPC Host Project for your environment.
This value can be obtained by running "gcloud projects list --filter='labels.application\_name:restricted-shared-vpc-host lifecycleState:ACTIVE'" and selecting the project. | `string` | n/a | yes | ## Outputs diff --git a/5-app-infra/source_repos/service-catalog/modules/notebook/data.tf b/5-app-infra/source_repos/service-catalog/modules/notebook/data.tf index 64654ba2..8a245691 100644 --- a/5-app-infra/source_repos/service-catalog/modules/notebook/data.tf +++ b/5-app-infra/source_repos/service-catalog/modules/notebook/data.tf @@ -18,37 +18,20 @@ data "google_project" "project" { project_id = var.project_id } -data "google_projects" "kms" { - filter = "labels.application_name:env-kms labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - -data "google_projects" "vpc" { - filter = "labels.application_name:restricted-shared-vpc-host labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" - # filter = "labels.application_name:base-shared-vpc-host labels.environment:${data.google_project.project.labels.environment} lifecycleState:ACTIVE" -} - data "google_compute_network" "shared_vpc" { - name = "vpc-${data.google_project.project.labels.env_code}-shared-restricted" - # name = "vpc-${data.google_project.project.labels.env_code}-shared-base" - project = data.google_projects.vpc.projects.0.project_id + name = "vpc-${data.google_project.project.labels.env_code}-shared-restricted" + project = var.vpc_project } data "google_compute_subnetwork" "subnet" { - name = "sb-${data.google_project.project.labels.env_code}-shared-restricted-${local.region}" - # name = "sb-${data.google_project.project.labels.env_code}-shared-base-${local.region}" - project = data.google_projects.vpc.projects.0.project_id + name = "sb-${data.google_project.project.labels.env_code}-shared-restricted-${local.region}" + project = var.vpc_project region = local.region } -data "google_kms_key_ring" "kms" { - name = "sample-keyring" - location = local.region - project = data.google_projects.kms.projects.0.project_id -} - data "google_kms_crypto_key" "key" { name = data.google_project.project.name - key_ring = data.google_kms_key_ring.kms.id + key_ring = var.kms_keyring } data "google_netblock_ip_ranges" "legacy_health_checkers" { diff --git a/5-app-infra/source_repos/service-catalog/modules/notebook/main.tf b/5-app-infra/source_repos/service-catalog/modules/notebook/main.tf index 6686081e..5184745f 100644 --- a/5-app-infra/source_repos/service-catalog/modules/notebook/main.tf +++ b/5-app-infra/source_repos/service-catalog/modules/notebook/main.tf @@ -17,6 +17,7 @@ resource "google_workbench_instance" "instance" { name = var.name location = var.location + project = var.project_id gce_setup { machine_type = var.machine_type diff --git a/5-app-infra/source_repos/service-catalog/modules/notebook/variables.tf b/5-app-infra/source_repos/service-catalog/modules/notebook/variables.tf index ab4c2c28..cb8c8d4a 100644 --- a/5-app-infra/source_repos/service-catalog/modules/notebook/variables.tf +++ b/5-app-infra/source_repos/service-catalog/modules/notebook/variables.tf @@ -15,8 +15,8 @@ */ variable "name" { - description = "name of the notebook instance" type = string + description = "Name of the notebook instance." } variable "location" { @@ -30,20 +30,20 @@ variable "location" { } variable "machine_type" { - description = "type of the machine to spin up for the notebook" type = string + description = "Type of the machine to spin up for the notebook." default = "e2-standard-4" } variable "instance_owners" { - description = "email of the owner of the instance, e.g. alias@example.com. Only one owner is supported!" type = set(string) + description = "Email of the owner of the instance, e.g. alias@example.com. Only one owner is supported!" } variable "accelerator_type" { - description = "The type of accelerator to use" type = string + description = "The type of accelerator to use." default = "NVIDIA_TESLA_K80" validation { condition = contains(["ACCELERATOR_TYPE_UNSPECIFIED", "NVIDIA_TESLA_K80", @@ -55,39 +55,37 @@ variable "accelerator_type" { } variable "core_count" { type = number + description = "Number of accelerators to use." default = 1 - description = "number of accelerators to use" } variable "image_project" { - description = "The name of the Google Cloud project that this VM image belongs to. Format: projects/{project_id}" type = string - # default = "deeplearning-platform-release" - default = "cloud-notebooks-managed" + description = "The name of the Google Cloud project that this VM image belongs to. Format: projects/{project_id}." + default = "cloud-notebooks-managed" } variable "image_family" { - description = "Use this VM image family to find the image; the newest image in this family will be used." type = string - # default = "common-cpu-notebooks" - default = "workbench-instances" + description = "Use this VM image family to find the image; the newest image in this family will be used." + default = "workbench-instances" } variable "image_name" { - description = "Use VM image name to find the image." type = string + description = "Use VM image name to find the image." default = "" } variable "install_gpu_driver" { - description = "Whether the end user authorizes Google Cloud to install GPU driver on this instance. Only applicable to instances with GPUs." type = bool + description = "Whether the end user authorizes Google Cloud to install GPU driver on this instance. Only applicable to instances with GPUs." default = false } variable "boot_disk_type" { - description = "Possible disk types for notebook instances" type = string + description = "Possible disk types for notebook instances." default = "PD_SSD" validation { condition = contains(["DISK_TYPE_UNSPECIFIED", "PD_STANDARD", "PD_SSD", "PD_BALANCED", "PD_EXTREME"], var.boot_disk_type) @@ -96,14 +94,14 @@ variable "boot_disk_type" { } variable "boot_disk_size_gb" { - description = "(Optional) The size of the boot disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB)" type = string + description = "(Optional) The size of the boot disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB)." default = "150" } variable "data_disk_type" { - description = "Optional. Input only. Indicates the type of the disk. Possible values are: PD_STANDARD, PD_SSD, PD_BALANCED, PD_EXTREME." type = string + description = "(Optional) Input only. Indicates the type of the disk. Possible values are: PD_STANDARD, PD_SSD, PD_BALANCED, PD_EXTREME." default = "PD_SSD" validation { condition = contains(["PD_STANDARD", "PD_SSD", "PD_BALANCED", "PD_EXTREME"], var.data_disk_type) @@ -112,35 +110,27 @@ variable "data_disk_type" { } variable "data_disk_size_gb" { - description = "(Optional) The size of the data disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB)" type = string + description = "(Optional) The size of the data disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB)" default = "150" } variable "disable_proxy_access" { - description = "(Optional) The notebook instance will not register with the proxy" type = bool + description = "(Optional) The notebook instance will not register with the proxy" default = false } -# variable "dataproc_kernel_access" { -# description = "(Optional) Enables access to Dataproc kernels." -# type = bool -# default = true -# } - variable "boundry_code" { - description = "The boundry code for the tenant" type = string + description = "The boundry code for the tenant" default = "001" } - variable "project_id" { type = string - description = "Optional Project ID." - default = null + description = "Project ID to deploy the instance." } variable "tags" { @@ -148,3 +138,19 @@ variable "tags" { description = "The Compute Engine tags to add to instance." default = ["egress-internet"] } + +variable "kms_keyring" { + type = string + description = < Date: Thu, 20 Jun 2024 14:22:56 -0300 Subject: [PATCH 2/2] update module README.md --- .../modules/notebook/README.md | 92 +------------------ 1 file changed, 4 insertions(+), 88 deletions(-) diff --git a/5-app-infra/source_repos/service-catalog/modules/notebook/README.md b/5-app-infra/source_repos/service-catalog/modules/notebook/README.md index bd38aee5..a3cad803 100644 --- a/5-app-infra/source_repos/service-catalog/modules/notebook/README.md +++ b/5-app-infra/source_repos/service-catalog/modules/notebook/README.md @@ -1,103 +1,19 @@ ## Prerequisites -#### IAM Permissions +### IAM Permissions | Service Account | Scope | Role | |-----------------|-------|------| -| PROJECT_NUMBER@cloudbuild.gserviceaccount.com | Project | Browser | -| | Project | Service Usage Consumer | -| | Project | Notebooks Admin | -| | Project | Compute Network Admin | -| | Project | Compute Security Admin | +| service-ML_PRJ_NUMBER@compute-system.iam.gserviceaccount.com | Key | roles/cloudkms.cryptoKeyEncrypterDecrypter | +| service-ML_PRJ_NUMBER@gcp-sa-notebooks.iam.gserviceaccount.com | Key | roles/cloudkms.cryptoKeyEncrypterDecrypter | -#### Organizational policies +### Organizational policies | Policy constraint | Scope | Value | |-------------------|-------|-------| | constraints/ainotebooks.requireAutoUpgradeSchedule | Project | Google-managed | | constraints/ainotebooks.environmentOptions | Project | Google-managed | - - -Copyright 2023 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -## Requirements - -No requirements. - -## Providers - -| Name | Version | -|------|---------| -| [google](#provider\_google) | 5.14.0 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [google_workbench_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/workbench_instance) | resource | -| [google_compute_network.shared_vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_network) | data source | -| [google_compute_subnetwork.subnet](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_subnetwork) | data source | -| [google_kms_crypto_key.key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_crypto_key) | data source | -| [google_kms_key_ring.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/kms_key_ring) | data source | -| [google_netblock_ip_ranges.health_checkers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/netblock_ip_ranges) | data source | -| [google_netblock_ip_ranges.iap_forwarders](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/netblock_ip_ranges) | data source | -| [google_netblock_ip_ranges.legacy_health_checkers](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/netblock_ip_ranges) | data source | -| [google_project.project](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project) | data source | -| [google_projects.kms](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | -| [google_projects.vpc](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [accelerator\_type](#input\_accelerator\_type) | The type of accelerator to use | `string` | `"NVIDIA_TESLA_K80"` | no | -| [boot\_disk\_size\_gb](#input\_boot\_disk\_size\_gb) | (Optional) The size of the boot disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB) | `string` | `"100"` | no | -| [boot\_disk\_type](#input\_boot\_disk\_type) | Possible disk types for notebook instances | `string` | `"PD_SSD"` | no | -| [boundry\_code](#input\_boundry\_code) | The boundry code for the tenant | `string` | `"001"` | no | -| [core\_count](#input\_core\_count) | number of accelerators to use | `number` | `1` | no | -| [data\_disk\_size\_gb](#input\_data\_disk\_size\_gb) | (Optional) The size of the data disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB) | `string` | `"100"` | no | -| [data\_disk\_type](#input\_data\_disk\_type) | Optional. Input only. Indicates the type of the disk. Possible values are: PD\_STANDARD, PD\_SSD, PD\_BALANCED, PD\_EXTREME. | `string` | `"PD_SSD"` | no | -| [disable\_proxy\_access](#input\_disable\_proxy\_access) | (Optional) The notebook instance will not register with the proxy | `bool` | `false` | no | -| [image\_family](#input\_image\_family) | Use this VM image family to find the image; the newest image in this family will be used. | `string` | `"workbench-instances"` | no | -| [image\_name](#input\_image\_name) | Use VM image name to find the image. | `string` | `""` | no | -| [image\_project](#input\_image\_project) | The name of the Google Cloud project that this VM image belongs to. Format: projects/{project\_id} | `string` | `"cloud-notebooks-managed"` | no | -| [install\_gpu\_driver](#input\_install\_gpu\_driver) | Whether the end user authorizes Google Cloud to install GPU driver on this instance. Only applicable to instances with GPUs. | `bool` | `false` | no | -| [instance\_owners](#input\_instance\_owners) | email of the owner of the instance, e.g. alias@example.com. Only one owner is supported! | `set(string)` | n/a | yes | -| [location](#input\_location) | Notebook instance location (zone). | `string` | `"us-central1-a"` | no | -| [machine\_type](#input\_machine\_type) | type of the machine to spin up for the notebook | `string` | `"e2-standard-4"` | no | -| [name](#input\_name) | name of the notebook instance | `string` | n/a | yes | -| [project\_id](#input\_project\_id) | Optional Project ID. | `string` | `null` | no | -| [tags](#input\_tags) | The Compute Engine tags to add to instance. | `list(string)` | 
[
  "egress-internet"
]
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [create\_time](#output\_create\_time) | Instance creation time | -| [effective\_labels](#output\_effective\_labels) | All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Terraform, other clients and services. | -| [id](#output\_id) | an identifier for the resource with format projects/{{project}}/locations/{{location}}/instances/{{name}} | -| [proxy\_uri](#output\_proxy\_uri) | The proxy endpoint that is used to access the Jupyter notebook. Only returned when the resource is in a PROVISIONED state. If needed you can utilize terraform apply -refresh-only to await the population of this value. | -| [state](#output\_state) | The state of this instance. | -| [terraform\_labels](#output\_terraform\_labels) | The combination of labels configured directly on the resource and default labels configured on the provider. | -| [update\_time](#output\_update\_time) | Instance update time. | - ## Inputs