Merge pull request #677 from GoogleCloudPlatform/release-candidate

Release v1.8.0
GoogleCloudPlatform · Nov 1, 2022 · 78bb2bd · 78bb2bd
2 parents f4ed7c1 + 01320c7
commit 78bb2bd
Show file tree

Hide file tree

Showing 113 changed files with 1,109 additions and 432 deletions.
diff --git a/.ansible-lint b/.ansible-lint
@@ -1,5 +1,15 @@
 skip_list:
 - yaml
+- jinja[invalid]
+
+exclude_paths:
+  - .cache/ # implicit unless exclude_paths is defined in config
+  - .github/
+  - cmd/
+  - docs/
+  - examples/
+  - community/examples/
+  - pkg/
 
 mock_roles:
 - googlecloudplatform.google_cloud_ops_agents

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -15,7 +15,7 @@
 ---
 repos:
 - repo: https://github.com/antonbabenko/pre-commit-terraform
-  rev: v1.75.0
+  rev: v1.76.0
   hooks:
   - id: terraform_fmt
   - id: terraform_tflint
@@ -28,7 +28,7 @@ repos:
     name: terraform-readme
     entry: tools/autodoc/terraform_docs.sh
     language: script
-    types: ['terraform']
+    types: [terraform]
     exclude: \.terraform\/.*$
     pass_filenames: true
     require_serial: true
@@ -39,8 +39,16 @@ repos:
     files: ^.*\.pkr\.hcl$
     pass_filenames: true
     require_serial: true
+  - id: duplicate-diff
+    name: duplicate-diff
+    entry: python3 tools/duplicate-diff.py
+    language: python
+    language_version: python3
+    types: [shell]
+    pass_filenames: true
+    require_serial: true
 - repo: https://github.com/dnephin/pre-commit-golang
-  rev: v0.5.0
+  rev: v0.5.1
   hooks:
   - id: go-fmt
   - id: go-vet
@@ -57,16 +65,21 @@ repos:
   - id: go-critic
     args: [-disable, "#experimental,sloppyTypeAssert"]
 - repo: https://github.com/ansible/ansible-lint.git
-  rev: v5.4.0
+  rev: v6.8.2
   hooks:
   - id: ansible-lint
+    always_run: false
+    exclude: '^(docs/|examples/|community/examples/)'
+    types: [yaml]
+    additional_dependencies:
+    - ansible==6.*
 - repo: https://github.com/adrienverge/yamllint.git
   rev: v1.28.0
   hooks:
   - id: yamllint
     args: [-c=.yamllint]
 - repo: https://github.com/jackdewinter/pymarkdown
-  rev: v0.9.7
+  rev: v0.9.8
   hooks:
   - id: pymarkdown
     # Rules at https://github.com/jackdewinter/pymarkdown/tree/main/docs/rules
@@ -78,9 +91,13 @@ repos:
     # MD046 - Code block style
     args: [--disable-rules, "MD013,MD022,MD033,MD034,MD041,MD046", scan]
 - repo: https://github.com/jumanjihouse/pre-commit-hooks
-  rev: "3.0.0"  # or specific git tag
+  rev: "3.0.0"
   hooks:
   - id: script-must-have-extension
   - id: shellcheck
   - id: shfmt
     exclude: ".*tpl"
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.3.0
+  hooks:
+  - id: end-of-file-fixer
diff --git a/LICENSE b/LICENSE
@@ -199,4 +199,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.
diff --git a/Makefile b/Makefile
@@ -223,4 +223,3 @@ packer-format:
 endif
 endif
 # END OF PACKER SECTION
-
diff --git a/cmd/root.go b/cmd/root.go
@@ -42,7 +42,7 @@ HPC deployments on the Google Cloud Platform.`,
 				log.Fatalf("cmd.Help function failed: %s", err)
 			}
 		},
-		Version:     "v1.7.0",
+		Version:     "v1.8.0",
 		Annotations: annotation,
 	}
 )

diff --git a/community/examples/AMD/hpc-cluster-amd-slurmv5.yaml b/community/examples/AMD/hpc-cluster-amd-slurmv5.yaml
@@ -115,9 +115,6 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       runners:
-      - type: shell
-        source: modules/startup-script/examples/install_ansible.sh
-        destination: install_ansible.sh
       - $(swfs.install_nfs_client_runner)
       - $(swfs.mount_runner)
       - $(spack.install_spack_deps_runner)

diff --git a/community/examples/cloud-batch.yaml b/community/examples/cloud-batch.yaml
@@ -39,9 +39,6 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       runners:
-      - type: shell
-        source: modules/startup-script/examples/install_ansible.sh
-        destination: install_ansible.sh
       - $(appfs.install_nfs_client_runner)
       - $(appfs.mount_runner)
       - type: shell

diff --git a/community/examples/htcondor-pool.yaml b/community/examples/htcondor-pool.yaml
@@ -50,9 +50,6 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       runners:
-      - type: shell
-        source: modules/startup-script/examples/install_ansible.sh
-        destination: install_ansible.sh
       - $(htcondor_install.install_htcondor_runner)
       - $(htcondor_configure.central_manager_runner)
 
@@ -76,9 +73,6 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       runners:
-      - type: shell
-        source: modules/startup-script/examples/install_ansible.sh
-        destination: install_ansible.sh
       - $(htcondor_install.install_htcondor_runner)
       - $(htcondor_configure.execute_point_runner)
 
@@ -100,9 +94,6 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       runners:
-      - type: shell
-        source: modules/startup-script/examples/install_ansible.sh
-        destination: install_ansible.sh
       - $(htcondor_install.install_htcondor_runner)
       - $(htcondor_install.install_autoscaler_deps_runner)
       - $(htcondor_install.install_autoscaler_runner)

diff --git a/community/examples/omnia-cluster.yaml b/community/examples/omnia-cluster.yaml
@@ -55,9 +55,6 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       runners:
-      - type: shell
-        source: "modules/startup-script/examples/install_ansible.sh"
-        destination: "install_ansible.sh"
       - $(homefs.install_nfs_client_runner)
       - $(homefs.mount_runner)
       - $(omnia.setup_omnia_node_runner)
@@ -68,9 +65,6 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       runners:
-      - type: shell
-        source: "modules/startup-script/examples/install_ansible.sh"
-        destination: "install_ansible.sh"
       - $(homefs.install_nfs_client_runner)
       - $(homefs.mount_runner)
       - $(omnia.setup_omnia_node_runner)

diff --git a/community/examples/spack-gromacs.yaml b/community/examples/spack-gromacs.yaml
@@ -83,9 +83,6 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       runners:
-      - type: shell
-        source: modules/startup-script/examples/install_ansible.sh
-        destination: install_ansible.sh
       - $(spack.install_spack_deps_runner)
       - $(spack.install_spack_runner)
 

diff --git a/community/examples/starccm-tutorial.yaml b/community/examples/starccm-tutorial.yaml
@@ -38,10 +38,6 @@ deployment_groups:
           sudo yum -y install libExt libXext.x86_64 nmap
         destination: /tmp/install-deps.sh
 
-      - type: shell
-        source: modules/startup-script/examples/install_ansible.sh
-        destination: install_ansible.sh
-
       - type: shell
         content: $(homefs.install_nfs_client)
         destination: "/tmp/nfs_client.sh"

diff --git a/community/modules/compute/SchedMD-slurm-on-gcp-partition/versions.tf b/community/modules/compute/SchedMD-slurm-on-gcp-partition/versions.tf
@@ -22,7 +22,7 @@ terraform {
     }
   }
   provider_meta "google" {
-    module_name = "blueprints/terraform/hpc-toolkit:SchedMD-slurm-on-gcp-partition/v1.7.0"
+    module_name = "blueprints/terraform/hpc-toolkit:SchedMD-slurm-on-gcp-partition/v1.8.0"
   }
 
   required_version = ">= 0.14.0"

diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-partition/README.md b/community/modules/compute/schedmd-slurm-gcp-v5-partition/README.md
@@ -61,10 +61,13 @@ limitations under the License.
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | >= 3.83 |
 
 ## Providers
 
-No providers.
+| Name | Version |
+|------|---------|
+| <a name="provider_google"></a> [google](#provider\_google) | >= 3.83 |
 
 ## Modules
 
@@ -74,7 +77,9 @@ No providers.
 
 ## Resources
 
-No resources.
+| Name | Type |
+|------|------|
+| [google_compute_default_service_account.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_default_service_account) | data source |
 
 ## Inputs
 
@@ -110,7 +115,7 @@ No resources.
 | <a name="input_preemptible"></a> [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `string` | `false` | no |
 | <a name="input_project_id"></a> [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes |
 | <a name="input_region"></a> [region](#input\_region) | The default region for Cloud resources. | `string` | n/a | yes |
-| <a name="input_service_account"></a> [service\_account](#input\_service\_account) | Service account to attach to the instances. See<br>'main.tf:local.service\_account' for the default. | <pre>object({<br>    email  = string<br>    scopes = set(string)<br>  })</pre> | <pre>{<br>  "email": "default",<br>  "scopes": [<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ]<br>}</pre> | no |
+| <a name="input_service_account"></a> [service\_account](#input\_service\_account) | Service account to attach to the compute instances. If not set, the<br>default compute service account for the given project will be used with the<br>"https://www.googleapis.com/auth/cloud-platform" scope. | <pre>object({<br>    email  = string<br>    scopes = set(string)<br>  })</pre> | `null` | no |
 | <a name="input_shielded_instance_config"></a> [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless<br>enable\_shielded\_vm is 'true'.<br>* enable\_integrity\_monitoring : Compare the most recent boot measurements to the<br>  integrity policy baseline and return a pair of pass/fail results depending on<br>  whether they match or not.<br>* enable\_secure\_boot : Verify the digital signature of all boot components, and<br>  halt the boot process if signature verification fails.<br>* enable\_vtpm : Use a virtualized trusted platform module, which is a<br>  specialized computer chip you can use to encrypt objects like keys and<br>  certificates. | <pre>object({<br>    enable_integrity_monitoring = bool<br>    enable_secure_boot          = bool<br>    enable_vtpm                 = bool<br>  })</pre> | <pre>{<br>  "enable_integrity_monitoring": true,<br>  "enable_secure_boot": true,<br>  "enable_vtpm": true<br>}</pre> | no |
 | <a name="input_slurm_cluster_name"></a> [slurm\_cluster\_name](#input\_slurm\_cluster\_name) | Cluster name, used for resource naming and slurm accounting. If not provided it will default to the first 8 characters of the deployment name (removing any invalid characters). | `string` | `null` | no |
 | <a name="input_source_image"></a> [source\_image](#input\_source\_image) | Source disk image. By default, the image used will be the hpc-centos7<br>version of the slurm-gcp public images. More information can be found in the<br>slurm-gcp docs:<br>https://github.com/SchedMD/slurm-gcp/blob/v5.0.2/docs/images.md#public-image | `string` | `null` | no |

diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-partition/main.tf b/community/modules/compute/schedmd-slurm-gcp-v5-partition/main.tf
@@ -46,12 +46,15 @@ locals {
       min_cpu_platform         = var.min_cpu_platform
       on_host_maintenance      = var.on_host_maintenance
       preemptible              = var.preemptible
-      service_account          = var.service_account
       shielded_instance_config = var.shielded_instance_config
       source_image_family      = var.source_image_family == null ? "" : var.source_image_family
       source_image_project     = var.source_image_project == null ? "" : var.source_image_project
       source_image             = var.source_image == null ? "" : var.source_image
       tags                     = var.tags
+      service_account = var.service_account != null ? var.service_account : {
+        email  = data.google_compute_default_service_account.default.email
+        scopes = ["https://www.googleapis.com/auth/cloud-platform"]
+      }
 
       # Spot VM settings
       enable_spot_vm       = var.enable_spot_vm
@@ -68,6 +71,9 @@ locals {
   slurm_cluster_name = var.slurm_cluster_name != null ? var.slurm_cluster_name : local.tmp_cluster_name
 }
 
+data "google_compute_default_service_account" "default" {
+  project = var.project_id
+}
 
 module "slurm_partition" {
   source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_partition?ref=v5.1.0"
@@ -85,4 +91,3 @@ module "slurm_partition" {
   subnetwork              = var.subnetwork_self_link == null ? "" : var.subnetwork_self_link
   partition_conf          = local.partition_conf
 }
-
diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf b/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf
@@ -292,15 +292,11 @@ variable "service_account" {
     scopes = set(string)
   })
   description = <<-EOD
-    Service account to attach to the instances. See
-    'main.tf:local.service_account' for the default.
+    Service account to attach to the compute instances. If not set, the
+    default compute service account for the given project will be used with the
+    "https://www.googleapis.com/auth/cloud-platform" scope.
     EOD
-  default = {
-    email = "default"
-    scopes = [
-      "https://www.googleapis.com/auth/cloud-platform",
-    ]
-  }
+  default     = null
 }
 
 variable "shielded_instance_config" {

diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-partition/versions.tf b/community/modules/compute/schedmd-slurm-gcp-v5-partition/versions.tf
@@ -15,5 +15,14 @@
 */
 
 terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = ">= 3.83"
+    }
+  }
+  provider_meta "google" {
+    module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-partition/v1.6.0"
+  }
   required_version = ">= 0.13.0"
 }
diff --git a/community/modules/database/slurm-cloudsql-federation/versions.tf b/community/modules/database/slurm-cloudsql-federation/versions.tf
@@ -30,10 +30,10 @@ terraform {
     }
   }
   provider_meta "google" {
-    module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.7.0"
+    module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.8.0"
   }
   provider_meta "google-beta" {
-    module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.7.0"
+    module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.8.0"
   }
 
   required_version = ">= 0.13.0"

diff --git a/community/modules/file-system/DDN-EXAScaler/outputs.tf b/community/modules/file-system/DDN-EXAScaler/outputs.tf
@@ -31,18 +31,26 @@ output "client_config_script" {
 
 output "install_ddn_lustre_client_runner" {
   description = "Runner that encapsulates the `client_config_script` output on this module."
-  value = {
+  value       = local.client_install_runner
+}
+
+locals {
+  client_install_runner = {
     "type"        = "shell"
     "content"     = module.ddn_exascaler.client_config
     "destination" = "install_ddn_lustre_client.sh"
   }
-}
 
-locals {
+  # Mount command provided by DDN does not support custom local mount
   split_mount_cmd               = split(" ", module.ddn_exascaler.mount_command)
   split_mount_cmd_wo_mountpoint = slice(local.split_mount_cmd, 0, length(local.split_mount_cmd) - 1)
   mount_cmd                     = "${join(" ", local.split_mount_cmd_wo_mountpoint)} ${var.local_mount}"
   mount_cmd_w_mkdir             = "mkdir -p ${var.local_mount} && ${local.mount_cmd}"
+  mount_runner = {
+    "type"        = "shell"
+    "content"     = local.mount_cmd_w_mkdir
+    "destination" = "mount-ddn-lustre.sh"
+  }
 }
 
 output "mount_command" {
@@ -52,11 +60,7 @@ output "mount_command" {
 
 output "mount_runner" {
   description = "Runner to mount the DDN EXAScaler Lustre file system"
-  value = {
-    "type"        = "shell"
-    "content"     = local.mount_cmd_w_mkdir
-    "destination" = "mount-ddn-lustre.sh"
-  }
+  value       = local.mount_runner
 }
 
 output "http_console" {
@@ -67,10 +71,12 @@ output "http_console" {
 output "network_storage" {
   description = "Describes a EXAScaler system to be mounted by other systems."
   value = {
-    server_ip     = split(":", split(" ", module.ddn_exascaler.mount_command)[3])[0]
-    remote_mount  = length(regexall("^/.*", var.fsname)) > 0 ? var.fsname : format("/%s", var.fsname)
-    local_mount   = var.local_mount != null ? var.local_mount : format("/mnt/%s", var.fsname)
-    fs_type       = "lustre"
-    mount_options = ""
+    server_ip             = split(":", split(" ", module.ddn_exascaler.mount_command)[3])[0]
+    remote_mount          = length(regexall("^/.*", var.fsname)) > 0 ? var.fsname : format("/%s", var.fsname)
+    local_mount           = var.local_mount != null ? var.local_mount : format("/mnt/%s", var.fsname)
+    fs_type               = "lustre"
+    mount_options         = ""
+    client_install_runner = local.client_install_runner
+    mount_runner          = local.mount_runner
   }
 }
diff --git a/community/modules/file-system/nfs-server/README.md b/community/modules/file-system/nfs-server/README.md
@@ -106,6 +106,6 @@ No modules.
 |------|-------------|
 | <a name="output_install_nfs_client"></a> [install\_nfs\_client](#output\_install\_nfs\_client) | Script for installing NFS client |
 | <a name="output_install_nfs_client_runner"></a> [install\_nfs\_client\_runner](#output\_install\_nfs\_client\_runner) | Runner to install NFS client using the startup-script module |
-| <a name="output_mount_runner"></a> [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using the startup-script module.<br>This runner requires ansible to be installed. This can be achieved using the<br>install\_ansible.sh script as a prior runner in the startup-script module:<br>runners:<br>- type: shell<br>  source: modules/startup-script/examples/install\_ansible.sh<br>  destination: install\_ansible.sh<br>- $(your-fs-id.mount\_runner)<br>... |
+| <a name="output_mount_runner"></a> [mount\_runner](#output\_mount\_runner) | Runner to mount the file-system using an ansible playbook. The startup-script<br>module will automatically handle installation of ansible.<br>- id: example-startup-script<br>  source: modules/scripts/startup-script<br>  settings:<br>    runners:<br>    - $(your-fs-id.mount\_runner)<br>... |
 | <a name="output_network_storage"></a> [network\_storage](#output\_network\_storage) | export of all desired folder directories |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
Original file line number	Diff line number	Diff line change
Expand Up		@@ -223,4 +223,3 @@ packer-format:
		endif
		endif
		# END OF PACKER SECTION