From 2600c0a56aa7a396752246bab556524aee63a437 Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Wed, 21 Feb 2024 12:30:30 -0700
Subject: [PATCH 01/13] Add SAs and roles for Vault server

---
 environment/deployments/roundtable/main.tf | 36 ++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/environment/deployments/roundtable/main.tf b/environment/deployments/roundtable/main.tf
index 34acb3b5..fdddf2be 100644
--- a/environment/deployments/roundtable/main.tf
+++ b/environment/deployments/roundtable/main.tf
@@ -77,6 +77,42 @@ resource "google_service_account_iam_binding" "git-lfs-ro-gcs-binding" {
   ]
 }
 
+# Service account for Vault Server
+resource "google_service_account" "vault_server_sa" {
+  account_id   = "vault-server"
+  display_name = "Vault Server"
+  description  = "Terraform-managed service account for Vault server"
+  project      = module.project_factory.project_id
+}
+
+# Use Workload Identity to have the service run as the appropriate service
+# account (bound to a Kubernetes service account)
+resource "google_service_account_iam_binding" "vault-server-sa-wi" {
+  service_account_id = google_service_account.vault_server_sa.name
+  role               = "roles/iam.workloadIdentityUser"
+  members = [
+    "serviceAccount:${module.project_factory.project_id}.svc.id.goog[vault/vault]"
+  ]
+}
+
+# The Vault service account must be granted the roles Cloud KMS Viewer and
+# Cloud KMS CryptoKey Encrypter/Decrypter
+resource "google_service_account_iam_binding" "vault-server-viewer-binding" {
+  service_account_id = google_service_account.vault_server_sa.name
+  role               = "roles/cloudkms.viewer"
+  members = [
+    "serviceAccount:vault-server@${module.project_factory.project_id}.iam.gserviceaccount.com"
+  ]
+}
+
+resource "google_service_account_iam_binding" "vault-server-cryptokey-binding" {
+  service_account_id = google_service_account.vault_server_sa.name
+  role               = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+  members = [
+    "serviceAccount:vault-server@${module.project_factory.project_id}.iam.gserviceaccount.com"
+  ]
+}
+
 module "service_account_cluster" {
   source     = "terraform-google-modules/service-accounts/google"
   version    = "~> 2.0"

From 61ca0086c2c054958a78f273ad28e56faf2021c4 Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Wed, 21 Feb 2024 12:44:58 -0700
Subject: [PATCH 02/13] bump roundtable environment serials

---
 environment/deployments/roundtable/env/dev.tfvars        | 2 +-
 environment/deployments/roundtable/env/production.tfvars | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/environment/deployments/roundtable/env/dev.tfvars b/environment/deployments/roundtable/env/dev.tfvars
index 3f8d798f..508b80ad 100644
--- a/environment/deployments/roundtable/env/dev.tfvars
+++ b/environment/deployments/roundtable/env/dev.tfvars
@@ -70,4 +70,4 @@ activate_apis = [
 ]
 
 # Increase this number to force Terraform to update the dev environment.
-# Serial: 5
+# Serial: 6
diff --git a/environment/deployments/roundtable/env/production.tfvars b/environment/deployments/roundtable/env/production.tfvars
index 31d35c78..8aa4cb1f 100644
--- a/environment/deployments/roundtable/env/production.tfvars
+++ b/environment/deployments/roundtable/env/production.tfvars
@@ -68,4 +68,5 @@ activate_apis = [
 ]
 
 # Increase this number to force Terraform to update the prod environment.
-# Serial: 5
+# Serial: 6
+

From d210f167c85505d5108e7cf8cc47ac6de970cea1 Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Wed, 21 Feb 2024 13:04:10 -0700
Subject: [PATCH 03/13] Add data-curation-prod buckets and SAs for vault server

---
 environment/deployments/data-curation/main.tf | 52 +++++++++++++++++++
 .../deployments/data-curation/variables.tf    | 14 +++++
 2 files changed, 66 insertions(+)

diff --git a/environment/deployments/data-curation/main.tf b/environment/deployments/data-curation/main.tf
index 803f18ec..7396556a 100644
--- a/environment/deployments/data-curation/main.tf
+++ b/environment/deployments/data-curation/main.tf
@@ -231,6 +231,58 @@ resource "google_storage_bucket_iam_binding" "git-lfs-bucket-dev-rw-iam-binding"
   members = var.git_lfs_rw_dev_service_accounts
 }
 
+// Vault Server Storage Bucket
+module "storage_bucket_7" {
+  source        = "../../../modules/bucket"
+  project_id    = module.project_factory.project_id
+  storage_class = "REGIONAL"
+  location      = "us-central1"
+  suffix_name   = ["vault-server"]
+  prefix_name   = "rubin"
+  versioning = {
+    vault-server = false
+  }
+  force_destroy = {
+    vault-server = false
+  }
+  labels = {
+    environment = var.environment
+    application = "vault"
+  }
+}
+// RW storage access to Vault Server bucket
+resource "google_storage_bucket_iam_binding" "vault-server-iam-binding" {
+  bucket  = module.storage_bucket_7.name
+  role    = "roles/storage.objectUser"
+  members = var.vault_server_service_accounts
+}
+
+// Vault Server Storage Bucket (Dev)
+module "storage_bucket_8" {
+  source        = "../../../modules/bucket"
+  project_id    = module.project_factory.project_id
+  storage_class = "REGIONAL"
+  location      = "us-central1"
+  suffix_name   = ["vault-server-dev"]
+  prefix_name   = "rubin"
+  versioning = {
+    vault-server-dev = false
+  }
+  force_destroy = {
+    vault-server-dev = false
+  }
+  labels = {
+    environment = var.environment
+    application = "vault"
+  }
+}
+// RW storage access to Vault Server Dev bucket
+resource "google_storage_bucket_iam_binding" "vault-server-dev-iam-binding" {
+  bucket  = module.storage_bucket_8.name
+  role    = "roles/storage.objectUser"
+  members = var.vault_server_dev_service_accounts
+}
+
 #---------------------------------------------------------------
 // Data Curation Prod
 #---------------------------------------------------------------
diff --git a/environment/deployments/data-curation/variables.tf b/environment/deployments/data-curation/variables.tf
index 28186a5d..8f07f8bc 100644
--- a/environment/deployments/data-curation/variables.tf
+++ b/environment/deployments/data-curation/variables.tf
@@ -246,3 +246,17 @@ variable "git_lfs_rw_dev_service_accounts" {
   description = "Service accounts used for Git-LFS Giftless Dev access (RW)"
   default     = []
 }
+
+// Vault Server
+variable "vault_server_service_accounts" {
+  type        = list(string)
+  description = "Service accounts used for Vault-Server access"
+  default     = []
+}
+
+// Vault Server
+variable "vault_server_dev_service_accounts" {
+  type        = list(string)
+  description = "Service accounts used for Vault-Server Dev access"
+  default     = []
+}

From 10d21167a23702a8abe86e5a3c9e8ba612d16b9b Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Wed, 21 Feb 2024 13:26:34 -0700
Subject: [PATCH 04/13] Move storage for Vault Server from data-curation-prod
 to roundtable-[dev|prod]

---
 environment/deployments/data-curation/main.tf | 52 ------------------
 .../deployments/data-curation/variables.tf    | 14 -----
 environment/deployments/roundtable/main.tf    | 54 +++++++++++++++++++
 .../deployments/roundtable/variables.tf       | 16 ++++++
 4 files changed, 70 insertions(+), 66 deletions(-)

diff --git a/environment/deployments/data-curation/main.tf b/environment/deployments/data-curation/main.tf
index 7396556a..803f18ec 100644
--- a/environment/deployments/data-curation/main.tf
+++ b/environment/deployments/data-curation/main.tf
@@ -231,58 +231,6 @@ resource "google_storage_bucket_iam_binding" "git-lfs-bucket-dev-rw-iam-binding"
   members = var.git_lfs_rw_dev_service_accounts
 }
 
-// Vault Server Storage Bucket
-module "storage_bucket_7" {
-  source        = "../../../modules/bucket"
-  project_id    = module.project_factory.project_id
-  storage_class = "REGIONAL"
-  location      = "us-central1"
-  suffix_name   = ["vault-server"]
-  prefix_name   = "rubin"
-  versioning = {
-    vault-server = false
-  }
-  force_destroy = {
-    vault-server = false
-  }
-  labels = {
-    environment = var.environment
-    application = "vault"
-  }
-}
-// RW storage access to Vault Server bucket
-resource "google_storage_bucket_iam_binding" "vault-server-iam-binding" {
-  bucket  = module.storage_bucket_7.name
-  role    = "roles/storage.objectUser"
-  members = var.vault_server_service_accounts
-}
-
-// Vault Server Storage Bucket (Dev)
-module "storage_bucket_8" {
-  source        = "../../../modules/bucket"
-  project_id    = module.project_factory.project_id
-  storage_class = "REGIONAL"
-  location      = "us-central1"
-  suffix_name   = ["vault-server-dev"]
-  prefix_name   = "rubin"
-  versioning = {
-    vault-server-dev = false
-  }
-  force_destroy = {
-    vault-server-dev = false
-  }
-  labels = {
-    environment = var.environment
-    application = "vault"
-  }
-}
-// RW storage access to Vault Server Dev bucket
-resource "google_storage_bucket_iam_binding" "vault-server-dev-iam-binding" {
-  bucket  = module.storage_bucket_8.name
-  role    = "roles/storage.objectUser"
-  members = var.vault_server_dev_service_accounts
-}
-
 #---------------------------------------------------------------
 // Data Curation Prod
 #---------------------------------------------------------------
diff --git a/environment/deployments/data-curation/variables.tf b/environment/deployments/data-curation/variables.tf
index 8f07f8bc..28186a5d 100644
--- a/environment/deployments/data-curation/variables.tf
+++ b/environment/deployments/data-curation/variables.tf
@@ -246,17 +246,3 @@ variable "git_lfs_rw_dev_service_accounts" {
   description = "Service accounts used for Git-LFS Giftless Dev access (RW)"
   default     = []
 }
-
-// Vault Server
-variable "vault_server_service_accounts" {
-  type        = list(string)
-  description = "Service accounts used for Vault-Server access"
-  default     = []
-}
-
-// Vault Server
-variable "vault_server_dev_service_accounts" {
-  type        = list(string)
-  description = "Service accounts used for Vault-Server Dev access"
-  default     = []
-}
diff --git a/environment/deployments/roundtable/main.tf b/environment/deployments/roundtable/main.tf
index fdddf2be..61fc62a9 100644
--- a/environment/deployments/roundtable/main.tf
+++ b/environment/deployments/roundtable/main.tf
@@ -21,6 +21,60 @@ module "iam_admin" {
   member                  = "gcp-${var.application_name}-administrators@lsst.cloud"
 }
 
+
+// Vault Server Storage Bucket
+module "storage_bucket" {
+  source        = "../../../modules/bucket"
+  project_id    = module.project_factory.project_id
+  storage_class = "REGIONAL"
+  location      = "us-central1"
+  suffix_name   = ["vault-server"]
+  prefix_name   = "rubin"
+  versioning = {
+    vault-server = false
+  }
+  force_destroy = {
+    vault-server = false
+  }
+  labels = {
+    environment = var.environment
+    application = "vault"
+  }
+}
+// RW storage access to Vault Server bucket
+resource "google_storage_bucket_iam_binding" "vault-server-iam-binding" {
+  bucket  = module.storage_bucket.name
+  role    = "roles/storage.objectUser"
+  members = var.vault_server_service_accounts
+}
+
+// Vault Server Storage Bucket (Dev)
+module "storage_bucket_2" {
+  source        = "../../../modules/bucket"
+  project_id    = module.project_factory.project_id
+  storage_class = "REGIONAL"
+  location      = "us-central1"
+  suffix_name   = ["vault-server-dev"]
+  prefix_name   = "rubin"
+  versioning = {
+    vault-server-dev = false
+  }
+  force_destroy = {
+    vault-server-dev = false
+  }
+  labels = {
+    environment = var.environment
+    application = "vault"
+  }
+}
+// RW storage access to Vault Server Dev bucket
+resource "google_storage_bucket_iam_binding" "vault-server-dev-iam-binding" {
+  bucket  = module.storage_bucket_2.name
+  role    = "roles/storage.objectUser"
+  members = var.vault_server_dev_service_accounts
+}
+
+
 # Service account for Git LFS read/write
 resource "google_service_account" "git_lfs_rw_sa" {
   account_id   = "git-lfs-rw"
diff --git a/environment/deployments/roundtable/variables.tf b/environment/deployments/roundtable/variables.tf
index fc9dfd2e..bbc18310 100644
--- a/environment/deployments/roundtable/variables.tf
+++ b/environment/deployments/roundtable/variables.tf
@@ -176,3 +176,19 @@ variable "static_ip_name" {
   type        = string
   default     = "load-balancer"
 }
+
+# SERVICE ACCOUNTS
+
+// Vault Server
+variable "vault_server_service_accounts" {
+  type        = list(string)
+  description = "Service accounts used for Vault-Server access"
+  default     = []
+}
+
+// Vault Server
+variable "vault_server_dev_service_accounts" {
+  type        = list(string)
+  description = "Service accounts used for Vault-Server Dev access"
+  default     = []
+}

From 5e4132732c54ec81ea6b67f0fb11b2c3d776f469 Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Thu, 22 Feb 2024 14:07:27 -0700
Subject: [PATCH 05/13] glue in service accounts

---
 environment/deployments/roundtable/env/dev.tfvars        | 5 +++++
 environment/deployments/roundtable/env/production.tfvars | 5 +++++
 environment/deployments/roundtable/main.tf               | 4 ++--
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/environment/deployments/roundtable/env/dev.tfvars b/environment/deployments/roundtable/env/dev.tfvars
index 508b80ad..c47fe7e9 100644
--- a/environment/deployments/roundtable/env/dev.tfvars
+++ b/environment/deployments/roundtable/env/dev.tfvars
@@ -69,5 +69,10 @@ activate_apis = [
   "sqladmin.googleapis.com"
 ]
 
+# Vault service service account
+vault_server_dev_service_accounts = [
+  "serviceAccount:vault-server@roundtable-dev-abe2.iam.gserviceaccount.com"
+]
+
 # Increase this number to force Terraform to update the dev environment.
 # Serial: 6
diff --git a/environment/deployments/roundtable/env/production.tfvars b/environment/deployments/roundtable/env/production.tfvars
index 8aa4cb1f..05154696 100644
--- a/environment/deployments/roundtable/env/production.tfvars
+++ b/environment/deployments/roundtable/env/production.tfvars
@@ -67,6 +67,11 @@ activate_apis = [
   "sqladmin.googleapis.com"
 ]
 
+# Vault service service account
+vault_server_service_accounts = [
+  "serviceAccount:vault-server@roundtable-prod-f6fd.iam.gserviceaccount.com"
+]
+
 # Increase this number to force Terraform to update the prod environment.
 # Serial: 6
 
diff --git a/environment/deployments/roundtable/main.tf b/environment/deployments/roundtable/main.tf
index 61fc62a9..9ae8df77 100644
--- a/environment/deployments/roundtable/main.tf
+++ b/environment/deployments/roundtable/main.tf
@@ -42,7 +42,7 @@ module "storage_bucket" {
   }
 }
 // RW storage access to Vault Server bucket
-resource "google_storage_bucket_iam_binding" "vault-server-iam-binding" {
+resource "google_storage_bucket_iam_binding" "vault-server-storage-binding" {
   bucket  = module.storage_bucket.name
   role    = "roles/storage.objectUser"
   members = var.vault_server_service_accounts
@@ -68,7 +68,7 @@ module "storage_bucket_2" {
   }
 }
 // RW storage access to Vault Server Dev bucket
-resource "google_storage_bucket_iam_binding" "vault-server-dev-iam-binding" {
+resource "google_storage_bucket_iam_binding" "vault-server-dev-storage-binding" {
   bucket  = module.storage_bucket_2.name
   role    = "roles/storage.objectUser"
   members = var.vault_server_dev_service_accounts

From f29e52cd299f3e92884ccb6c472d708cfc9e7b06 Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Mon, 26 Feb 2024 10:26:11 -0700
Subject: [PATCH 06/13] Add KMS resources

---
 environment/deployments/roundtable/main.tf    | 26 +++++
 .../deployments/roundtable/variables.tf       |  1 +
 modules/kms/main.tf                           | 22 +++++
 modules/kms/outputs.tf                        | 19 ++++
 modules/kms/readme.md                         |  0
 modules/kms/variables.tf                      | 98 +++++++++++++++++++
 6 files changed, 166 insertions(+)
 create mode 100644 modules/kms/main.tf
 create mode 100644 modules/kms/outputs.tf
 create mode 100644 modules/kms/readme.md
 create mode 100644 modules/kms/variables.tf

diff --git a/environment/deployments/roundtable/main.tf b/environment/deployments/roundtable/main.tf
index 9ae8df77..761db1d2 100644
--- a/environment/deployments/roundtable/main.tf
+++ b/environment/deployments/roundtable/main.tf
@@ -21,6 +21,32 @@ module "iam_admin" {
   member                  = "gcp-${var.application_name}-administrators@lsst.cloud"
 }
 
+// Vault server key management
+// prod
+module "kms" {
+  source         = "../../../modules/kms"
+  project_id     = module.project_factory.project_id
+  location       = "us-central1"  
+  keyring        = "vault-server"
+  keys           = [ "vault-seal" ]
+  set_owners_for = [ "vault-seal" ]
+  decrypters     = var.vault_server_service_accounts
+  encrypters     = var.vault_server_service_accounts
+  owners         = var.vault_server_service_accounts
+}
+// dev
+module "kms_2" {
+  source         = "../../../modules/kms"
+  project_id     = module.project_factory.project_id
+  location       = "us-central1"  
+  keyring        = "vault-server-dev"
+  keys           = [ "vault-seal" ]
+  set_owners_for = [ "vault-seal" ]
+  decrypters     = var.vault_server_dev_service_accounts
+  encrypters     = var.vault_server_dev_service_accounts
+  owners         = var.vault_server_dev_service_accounts
+}
+
 
 // Vault Server Storage Bucket
 module "storage_bucket" {
diff --git a/environment/deployments/roundtable/variables.tf b/environment/deployments/roundtable/variables.tf
index bbc18310..f2815197 100644
--- a/environment/deployments/roundtable/variables.tf
+++ b/environment/deployments/roundtable/variables.tf
@@ -25,6 +25,7 @@ variable "activate_apis" {
   description = "The api to activate for the GCP project"
   type        = list(string)
   default = [
+    "cloudkms.googleapis.com",
     "compute.googleapis.com",
     "container.googleapis.com",
     "stackdriver.googleapis.com",
diff --git a/modules/kms/main.tf b/modules/kms/main.tf
new file mode 100644
index 00000000..6d99ca68
--- /dev/null
+++ b/modules/kms/main.tf
@@ -0,0 +1,22 @@
+module "kms" {
+  source  = "terraform-google-modules/kms/google"
+  version = "~> 2.0"
+
+  project_id                     = var.project_id
+  location                       = var.location
+  keyring                        = var.keyring
+  keys                           = var.keys
+  set_decrypters_for             = var.set_decrypters_for
+  set_encrypters_for             = var.set_encrypters_for
+  set_owners_for                 = var.set_owners_for
+  decrypters                     = var.decrypters
+  encrypters                     = var.encrypters
+  owners                         = var.owners
+  labels                         = var.labels
+  key_algorithm                  = var.key_algorithm
+  key_destroy_scheduled_duration = var.key_destroy_scheduled_duration
+  key_protection_level           = var.key_protection_level
+  key_rotation_period            = var.key_rotation_period
+  prevent_destroy                = var.prevent_destroy
+  purpose                        = var.purpose
+}
diff --git a/modules/kms/outputs.tf b/modules/kms/outputs.tf
new file mode 100644
index 00000000..eac3beed
--- /dev/null
+++ b/modules/kms/outputs.tf
@@ -0,0 +1,19 @@
+output "keyring" {
+  description = "Self link of the keyring."
+  value = module.kms.keyring
+}
+
+output "keyring_name" {
+  description = "Name of the keyring."
+  value = module.kms.keyring_name
+}
+
+output "keyring_resource" {
+  description = "Keyring resource."
+  value = module.kms.keyring_resource
+}
+
+output "keys" {
+  description = "Map of key name => key self link."
+  value = module.kms.keys
+}
diff --git a/modules/kms/readme.md b/modules/kms/readme.md
new file mode 100644
index 00000000..e69de29b
diff --git a/modules/kms/variables.tf b/modules/kms/variables.tf
new file mode 100644
index 00000000..36c3720c
--- /dev/null
+++ b/modules/kms/variables.tf
@@ -0,0 +1,98 @@
+variable "project_id" {
+  description = "Project id where the keyring will be created."
+  type        = string
+}
+
+variable "location" {
+  description = "Location for the keyring."
+  type        = string
+}
+
+variable "keyring" {
+  description = "Keyring name."
+  type        = string
+}
+
+variable "keys" {
+  description = "Key names."
+  type        = list(string)
+  default     = []
+}
+
+variable "set_decrypters_for" {
+  description = "Name of keys for which decrypters will be set."
+  type        = list(string)
+  default     = []
+}
+
+variable "set_encrypters_for" {
+  description = "Name of keys for which encrypters will be set."
+  type        = list(string)
+  default     = []
+}
+
+variable "set_owners_for" {
+  description = "Name of keys for which owners will be set."
+  type        = list(string)
+  default     = []
+}
+
+variable "decrypters" {
+  description = "List of comma-separated decrypters for each key declared in set_decrypters_for."
+  type        = list(string)
+  default     = []
+}
+
+variable "encrypters" {
+  description = "List of comma-separated encrypters for each key declared in set_encrypters_for."
+  type        = list(string)
+  default     = []
+}
+
+variable "owners" {
+  description = "List of comma-separated owners for each key declared in set_owners_for."
+  type        = list(string)
+  default     = []
+}
+
+variable "key_algorithm" {
+  description = "The algorithm to use when creating a version based on this template. See the https://cloud.google.com/kms/docs/reference/rest/v1/CryptoKeyVersionAlgorithm for possible inputs."
+  type                   = string
+  default                = "GOOGLE_SYMMETRIC_ENCRYPTION"
+}
+
+variable "key_destroy_scheduled_duration" {
+  description = "Set the period of time that versions of keys spend in the DESTROY_SCHEDULED state before transitioning to DESTROYED."
+  type        = string
+  default     = null
+}
+
+variable key_protection_level {
+  description = "The protection level to use when creating a version based on this template. Possible values are SOFTWARE and HSM."
+  type        = string
+  default     = "SOFTWARE"
+}
+
+variable "key_rotation_period" {
+  description = "Generate a new key every time this period passes."
+  type        = string
+  default     = "7776000s"
+}
+
+variable "labels" {
+  description = "Labels, provided as a map."
+  type        = map(string)
+  default     = {}
+}
+
+variable "prevent_destroy" {
+  description = "Set the prevent_destroy lifecycle attribute on keys."
+  type        = bool
+  default     = true
+}
+
+variable "purpose" {
+  description = "The immutable purpose of the CryptoKey. Possible values are ENCRYPT_DECRYPT, ASYMMETRIC_SIGN, and ASYMMETRIC_DECRYPT."
+  type        = string
+  default     = "ENCRYPT_DECRYPT"
+}

From bde78d1d47cfdef4dc2f6bececeb41280249cb55 Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Mon, 26 Feb 2024 11:38:16 -0700
Subject: [PATCH 07/13] Loosen google module dependencies for KMS

---
 environment/deployments/roundtable/backend.tf | 4 ++--
 modules/kms/main.tf                           | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/environment/deployments/roundtable/backend.tf b/environment/deployments/roundtable/backend.tf
index dea72ad8..e0c08822 100644
--- a/environment/deployments/roundtable/backend.tf
+++ b/environment/deployments/roundtable/backend.tf
@@ -6,7 +6,7 @@ terraform {
   backend "gcs" {
   }
   required_providers {
-    google      = "~> 3.51.0"
-    google-beta = "~> 3.51.0"
+    google      = ">= 3.51.0"
+    google-beta = ">= 3.51.0"
   }
 }
diff --git a/modules/kms/main.tf b/modules/kms/main.tf
index 6d99ca68..daa5c43d 100644
--- a/modules/kms/main.tf
+++ b/modules/kms/main.tf
@@ -1,6 +1,6 @@
 module "kms" {
   source  = "terraform-google-modules/kms/google"
-  version = "~> 2.0"
+  version = ">= 2.0"
 
   project_id                     = var.project_id
   location                       = var.location

From 0135ff171080c7c391be06d8e329d782e887d7a8 Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Mon, 26 Feb 2024 12:03:38 -0700
Subject: [PATCH 08/13] fill out kms readme

---
 modules/kms/readme.md | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/modules/kms/readme.md b/modules/kms/readme.md
index e69de29b..12b896bc 100644
--- a/modules/kms/readme.md
+++ b/modules/kms/readme.md
@@ -0,0 +1,37 @@
+# Terraform module for KMS
+
+This enables creation of a KMS keyring and one or more keys with linked owner/encrypter/decrypter service accounts.
+
+It is used by Rubin Observatory to store seal keys for the Vault server that backs K8s vault-secrets-operator.  In expected use, `project_id`, `location`, `keyring`, `keys`, and access information will be set, and everything else will be left at its default value.
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|----------|
+| project\_id | Project id where the keyring will be created. | `string` | n/a | yes |
+| location | Location for the keyring. | `string` | n/a | yes |
+| keyring | Keyring name. | `string` | n/a | yes |
+| keys | Key names. | `list(string)` | `[]` | no |
+| set\_decrypters\_for | Name of keys for which decrypters will be set. | `list(str)` | `[]` | no |
+| set\_encrypters\_for | Name of keys for which encrypters will be set. | `list(str)` | `[]` | no |
+| set\_owners\_for | Name of keys for which owners will be set. | `list(str)` | `[]` | no |
+| decrypters | List of comma-separated decrypters for each key declared in set\_decrypters\_for. | `list(str)` | `[]` | no |
+| encrypters | List of comma-separated encrypters for each key declared in set\_encrypters\_for. | `list(str)` | `[]` | no |
+| owners | List of comma-separated owners for each key declared in set\_owners\_for. | `list(str)` | `[]` | no |
+| key\_algorithm | The algorithm to use when creating a version based on this template. See the https://cloud.google.com/kms/docs/reference/rest/v1/CryptoKeyVersionAlgorithm for possible inputs. | `string` | `"GOOGLE_SYMMETRIC_ENCRYPTION"` | no |
+| key\_destroy\_scheduled\_duration | Set the period of time that versions of keys spend in the DESTROY_SCHEDULED state before transitioning to DESTROYED. | `string` | null | no |
+| key\_protection\_level | The protection level to use when creating a version based on this template. Possible values are SOFTWARE and HSM. | `string` | `"SOFTWARE"` | no |
+| key\_rotation\_peroid | Generate a new key every time this period passes. | `string` | `"7776000s"` | no |
+| labels | Labels, provided as a map. | `map(string)` | `{}` | no |
+| prevent\_destroy | Set the prevent\_destroy lifecycle attribute on keys. | `bool` | `true` | no |
+| purpose | The immutable purpose of the CryptoKey. Possible values are ENCRYPT\_DECRYPT, ASYMMETRIC\_SIGN, and ASYMMETRIC\_DECRYPT. | `string` | `"ENCRYPT_DECRYPT"` | no |
+
+## Outputs
+
+| Name | Description | Value |
+|------|-------------|-------|
+| keyring | Self link of the keyring. | `module.kms.keyring` |
+| keyring\_name | Name of the keyring. | `module.kms.keyring_name` |
+| keyring\_resource | Keyring resource. | `module.kms.keyring_resource` |
+| keys | Map of key name => key self link. | `module.kms.keys` |
+

From f282933385c05091032669da59d8aea5edc0f45d Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Mon, 26 Feb 2024 12:15:33 -0700
Subject: [PATCH 09/13] parameterize vault server resources more accurately

---
 .../deployments/roundtable/env/dev.tfvars     |  4 +-
 .../roundtable/env/production.tfvars          |  2 +
 environment/deployments/roundtable/main.tf    | 42 +------------------
 .../deployments/roundtable/variables.tf       | 10 ++---
 4 files changed, 11 insertions(+), 47 deletions(-)

diff --git a/environment/deployments/roundtable/env/dev.tfvars b/environment/deployments/roundtable/env/dev.tfvars
index c47fe7e9..cd690f9b 100644
--- a/environment/deployments/roundtable/env/dev.tfvars
+++ b/environment/deployments/roundtable/env/dev.tfvars
@@ -70,9 +70,11 @@ activate_apis = [
 ]
 
 # Vault service service account
-vault_server_dev_service_accounts = [
+vault_server_service_accounts = [
   "serviceAccount:vault-server@roundtable-dev-abe2.iam.gserviceaccount.com"
 ]
 
+vault_server_bucket_suffix = "vault-server-dev"
+
 # Increase this number to force Terraform to update the dev environment.
 # Serial: 6
diff --git a/environment/deployments/roundtable/env/production.tfvars b/environment/deployments/roundtable/env/production.tfvars
index 05154696..c64c3577 100644
--- a/environment/deployments/roundtable/env/production.tfvars
+++ b/environment/deployments/roundtable/env/production.tfvars
@@ -72,6 +72,8 @@ vault_server_service_accounts = [
   "serviceAccount:vault-server@roundtable-prod-f6fd.iam.gserviceaccount.com"
 ]
 
+vault_server_bucket_suffix = "vault-server"
+
 # Increase this number to force Terraform to update the prod environment.
 # Serial: 6
 
diff --git a/environment/deployments/roundtable/main.tf b/environment/deployments/roundtable/main.tf
index 761db1d2..7fb263cf 100644
--- a/environment/deployments/roundtable/main.tf
+++ b/environment/deployments/roundtable/main.tf
@@ -34,19 +34,6 @@ module "kms" {
   encrypters     = var.vault_server_service_accounts
   owners         = var.vault_server_service_accounts
 }
-// dev
-module "kms_2" {
-  source         = "../../../modules/kms"
-  project_id     = module.project_factory.project_id
-  location       = "us-central1"  
-  keyring        = "vault-server-dev"
-  keys           = [ "vault-seal" ]
-  set_owners_for = [ "vault-seal" ]
-  decrypters     = var.vault_server_dev_service_accounts
-  encrypters     = var.vault_server_dev_service_accounts
-  owners         = var.vault_server_dev_service_accounts
-}
-
 
 // Vault Server Storage Bucket
 module "storage_bucket" {
@@ -54,7 +41,7 @@ module "storage_bucket" {
   project_id    = module.project_factory.project_id
   storage_class = "REGIONAL"
   location      = "us-central1"
-  suffix_name   = ["vault-server"]
+  suffix_name   = [ var.vault_server_bucket_suffix ]
   prefix_name   = "rubin"
   versioning = {
     vault-server = false
@@ -74,33 +61,6 @@ resource "google_storage_bucket_iam_binding" "vault-server-storage-binding" {
   members = var.vault_server_service_accounts
 }
 
-// Vault Server Storage Bucket (Dev)
-module "storage_bucket_2" {
-  source        = "../../../modules/bucket"
-  project_id    = module.project_factory.project_id
-  storage_class = "REGIONAL"
-  location      = "us-central1"
-  suffix_name   = ["vault-server-dev"]
-  prefix_name   = "rubin"
-  versioning = {
-    vault-server-dev = false
-  }
-  force_destroy = {
-    vault-server-dev = false
-  }
-  labels = {
-    environment = var.environment
-    application = "vault"
-  }
-}
-// RW storage access to Vault Server Dev bucket
-resource "google_storage_bucket_iam_binding" "vault-server-dev-storage-binding" {
-  bucket  = module.storage_bucket_2.name
-  role    = "roles/storage.objectUser"
-  members = var.vault_server_dev_service_accounts
-}
-
-
 # Service account for Git LFS read/write
 resource "google_service_account" "git_lfs_rw_sa" {
   account_id   = "git-lfs-rw"
diff --git a/environment/deployments/roundtable/variables.tf b/environment/deployments/roundtable/variables.tf
index f2815197..3127347e 100644
--- a/environment/deployments/roundtable/variables.tf
+++ b/environment/deployments/roundtable/variables.tf
@@ -187,9 +187,9 @@ variable "vault_server_service_accounts" {
   default     = []
 }
 
-// Vault Server
-variable "vault_server_dev_service_accounts" {
-  type        = list(string)
-  description = "Service accounts used for Vault-Server Dev access"
-  default     = []
+# Buckets
+
+variable "vault_server_bucket_suffix" {
+  type        = string
+  description = "Suffix for bucket used for Vault server storage"
 }

From dd10c96b59d6e4f71e2b72c9166f80fc60d0de0c Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Mon, 26 Feb 2024 13:12:28 -0700
Subject: [PATCH 10/13] Add bucket for Vault server backups

---
 .../deployments/roundtable/env/dev.tfvars     |  1 +
 .../roundtable/env/production.tfvars          |  1 +
 environment/deployments/roundtable/main.tf    | 24 +++++++++++++++++++
 .../deployments/roundtable/variables.tf       |  5 ++++
 4 files changed, 31 insertions(+)

diff --git a/environment/deployments/roundtable/env/dev.tfvars b/environment/deployments/roundtable/env/dev.tfvars
index cd690f9b..72b5bb37 100644
--- a/environment/deployments/roundtable/env/dev.tfvars
+++ b/environment/deployments/roundtable/env/dev.tfvars
@@ -75,6 +75,7 @@ vault_server_service_accounts = [
 ]
 
 vault_server_bucket_suffix = "vault-server-dev"
+vault_server_backup_bucket_suffix = "vault-server-dev-backup"
 
 # Increase this number to force Terraform to update the dev environment.
 # Serial: 6
diff --git a/environment/deployments/roundtable/env/production.tfvars b/environment/deployments/roundtable/env/production.tfvars
index c64c3577..86ef4961 100644
--- a/environment/deployments/roundtable/env/production.tfvars
+++ b/environment/deployments/roundtable/env/production.tfvars
@@ -73,6 +73,7 @@ vault_server_service_accounts = [
 ]
 
 vault_server_bucket_suffix = "vault-server"
+vault_server_backup_bucket_suffix = "vault-server-backup"
 
 # Increase this number to force Terraform to update the prod environment.
 # Serial: 6
diff --git a/environment/deployments/roundtable/main.tf b/environment/deployments/roundtable/main.tf
index 7fb263cf..a6bd9ec4 100644
--- a/environment/deployments/roundtable/main.tf
+++ b/environment/deployments/roundtable/main.tf
@@ -54,6 +54,30 @@ module "storage_bucket" {
     application = "vault"
   }
 }
+
+// Vault Server Storage Bucket (Backup)
+// Note that we don't need all the SA/WI access to this: the only thing it's
+// going to be used for is a copy target.  We may need a different SA to
+// run the backups.
+module "storage_bucket_b" {
+  source        = "../../../modules/bucket"
+  project_id    = module.project_factory.project_id
+  storage_class = "REGIONAL"
+  location      = "us-central1"
+  suffix_name   = [ var.vault_server_backup_bucket_suffix ]
+  prefix_name   = "rubin"
+  versioning = {
+    vault-server = false
+  }
+  force_destroy = {
+    vault-server = false
+  }
+  labels = {
+    environment = var.environment
+    application = "vault"
+  }
+}
+
 // RW storage access to Vault Server bucket
 resource "google_storage_bucket_iam_binding" "vault-server-storage-binding" {
   bucket  = module.storage_bucket.name
diff --git a/environment/deployments/roundtable/variables.tf b/environment/deployments/roundtable/variables.tf
index 3127347e..e736996b 100644
--- a/environment/deployments/roundtable/variables.tf
+++ b/environment/deployments/roundtable/variables.tf
@@ -193,3 +193,8 @@ variable "vault_server_bucket_suffix" {
   type        = string
   description = "Suffix for bucket used for Vault server storage"
 }
+
+variable "vault_server_backup_bucket_suffix" {
+  type        = string
+  description = "Suffix for bucket used for Vault server storage backup"
+}

From e456cfe5e5283d3029d2b8faf7648b497320632a Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Mon, 26 Feb 2024 15:51:10 -0700
Subject: [PATCH 11/13] add lifecycle rules for buckets, parameterize some maps

---
 .../deployments/roundtable/env/dev.tfvars     |  1 -
 .../roundtable/env/production.tfvars          |  1 -
 environment/deployments/roundtable/main.tf    | 30 +++++++++++++++----
 .../deployments/roundtable/variables.tf       |  5 ----
 4 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/environment/deployments/roundtable/env/dev.tfvars b/environment/deployments/roundtable/env/dev.tfvars
index 72b5bb37..cd690f9b 100644
--- a/environment/deployments/roundtable/env/dev.tfvars
+++ b/environment/deployments/roundtable/env/dev.tfvars
@@ -75,7 +75,6 @@ vault_server_service_accounts = [
 ]
 
 vault_server_bucket_suffix = "vault-server-dev"
-vault_server_backup_bucket_suffix = "vault-server-dev-backup"
 
 # Increase this number to force Terraform to update the dev environment.
 # Serial: 6
diff --git a/environment/deployments/roundtable/env/production.tfvars b/environment/deployments/roundtable/env/production.tfvars
index 86ef4961..c64c3577 100644
--- a/environment/deployments/roundtable/env/production.tfvars
+++ b/environment/deployments/roundtable/env/production.tfvars
@@ -73,7 +73,6 @@ vault_server_service_accounts = [
 ]
 
 vault_server_bucket_suffix = "vault-server"
-vault_server_backup_bucket_suffix = "vault-server-backup"
 
 # Increase this number to force Terraform to update the prod environment.
 # Serial: 6
diff --git a/environment/deployments/roundtable/main.tf b/environment/deployments/roundtable/main.tf
index a6bd9ec4..b90ac8db 100644
--- a/environment/deployments/roundtable/main.tf
+++ b/environment/deployments/roundtable/main.tf
@@ -44,10 +44,20 @@ module "storage_bucket" {
   suffix_name   = [ var.vault_server_bucket_suffix ]
   prefix_name   = "rubin"
   versioning = {
-    vault-server = false
+    (var.vault_server_bucket_suffix) = true
   }
+  lifecycle_rules = [
+    {
+      action = {
+	type = "Delete"
+      }
+      condition = {
+        num_newer_versions = 3
+      }
+    }
+  ]
   force_destroy = {
-    vault-server = false
+    (var.vault_server_bucket_suffix) = false
   }
   labels = {
     environment = var.environment
@@ -64,13 +74,23 @@ module "storage_bucket_b" {
   project_id    = module.project_factory.project_id
   storage_class = "REGIONAL"
   location      = "us-central1"
-  suffix_name   = [ var.vault_server_backup_bucket_suffix ]
+  suffix_name   = [ "${var.vault_server_bucket_suffix}-backup" ]
   prefix_name   = "rubin"
   versioning = {
-    vault-server = false
+    "${var.vault_server_bucket_suffix}-backup" = true
   }
+  lifecycle_rules = [
+    {
+      action = {
+	type = "Delete"
+      }
+      condition = {
+	num_newer_versions = "20"
+      }
+    }
+  ]
   force_destroy = {
-    vault-server = false
+    "${var.vault_server_bucket_suffix}-backup" = false
   }
   labels = {
     environment = var.environment
diff --git a/environment/deployments/roundtable/variables.tf b/environment/deployments/roundtable/variables.tf
index e736996b..3127347e 100644
--- a/environment/deployments/roundtable/variables.tf
+++ b/environment/deployments/roundtable/variables.tf
@@ -193,8 +193,3 @@ variable "vault_server_bucket_suffix" {
   type        = string
   description = "Suffix for bucket used for Vault server storage"
 }
-
-variable "vault_server_backup_bucket_suffix" {
-  type        = string
-  description = "Suffix for bucket used for Vault server storage backup"
-}

From a70730eebbd4057e692839371491edbbf82ea1c7 Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Mon, 26 Feb 2024 16:35:29 -0700
Subject: [PATCH 12/13] Add backup schedule for Vault

---
 environment/deployments/roundtable/main.tf | 36 ++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/environment/deployments/roundtable/main.tf b/environment/deployments/roundtable/main.tf
index b90ac8db..59124455 100644
--- a/environment/deployments/roundtable/main.tf
+++ b/environment/deployments/roundtable/main.tf
@@ -98,6 +98,42 @@ module "storage_bucket_b" {
   }
 }
 
+// Resources for backups
+
+// Admin storage access to Vault Server backup bucket
+resource "google_storage_bucket_iam_binding" "vault-server-storage-backup-binding" {
+  bucket  = module.storage_bucket_b.name
+  role    = "roles/storage.admin"
+  members = var.vault_server_service_accounts
+}
+
+resource "google_storage_transfer_job" "vault-server-storage-backup" {
+  description  = "Nightly backup of Vault Server storage"
+  project      = module.project_factory.project_id
+  transfer_spec {
+    gcs_data_source {
+      bucket_name = module.storage_bucket.name
+    }
+    gcs_data_sink {
+      bucket_name = module.storage_bucket_b.name
+    }
+  }
+  schedule {
+    schedule_start_date {
+      year  = 2024
+      month = 1
+      day = 1
+    }
+    start_time_of_day { // UTC: 2 AM Pacific Standard Time
+      hours = 10
+      minutes = 0
+      seconds = 0
+      nanos = 0
+    }
+  }
+  depends_on = [ google_storage_bucket_iam_binding.vault-server-storage-backup-binding ]
+}
+
 // RW storage access to Vault Server bucket
 resource "google_storage_bucket_iam_binding" "vault-server-storage-binding" {
   bucket  = module.storage_bucket.name

From 5f14f10989ff91291d08e88ee0bd469a09bd2093 Mon Sep 17 00:00:00 2001
From: adam <athornton@lsst.org>
Date: Mon, 26 Feb 2024 16:52:27 -0700
Subject: [PATCH 13/13] Refactor Vault storage backup object layout

---
 environment/deployments/roundtable/main.tf | 92 +++++++++++-----------
 1 file changed, 46 insertions(+), 46 deletions(-)

diff --git a/environment/deployments/roundtable/main.tf b/environment/deployments/roundtable/main.tf
index 59124455..ed1e0057 100644
--- a/environment/deployments/roundtable/main.tf
+++ b/environment/deployments/roundtable/main.tf
@@ -66,9 +66,6 @@ module "storage_bucket" {
 }
 
 // Vault Server Storage Bucket (Backup)
-// Note that we don't need all the SA/WI access to this: the only thing it's
-// going to be used for is a copy target.  We may need a different SA to
-// run the backups.
 module "storage_bucket_b" {
   source        = "../../../modules/bucket"
   project_id    = module.project_factory.project_id
@@ -98,7 +95,50 @@ module "storage_bucket_b" {
   }
 }
 
-// Resources for backups
+// Service account and bindings for Vault Server
+
+// Service account for Vault Server
+resource "google_service_account" "vault_server_sa" {
+  account_id   = "vault-server"
+  display_name = "Vault Server"
+  description  = "Terraform-managed service account for Vault server"
+  project      = module.project_factory.project_id
+}
+
+// Use Workload Identity to have the service run as the appropriate service
+// account (bound to a Kubernetes service account)
+resource "google_service_account_iam_binding" "vault-server-sa-wi" {
+  service_account_id = google_service_account.vault_server_sa.name
+  role               = "roles/iam.workloadIdentityUser"
+  members = [
+    "serviceAccount:${module.project_factory.project_id}.svc.id.goog[vault/vault]"
+  ]
+}
+
+// The Vault service account must be granted the roles Cloud KMS Viewer and
+// Cloud KMS CryptoKey Encrypter/Decrypter
+resource "google_service_account_iam_binding" "vault-server-viewer-binding" {
+  service_account_id = google_service_account.vault_server_sa.name
+  role               = "roles/cloudkms.viewer"
+  members = [
+    "serviceAccount:vault-server@${module.project_factory.project_id}.iam.gserviceaccount.com"
+  ]
+}
+
+resource "google_service_account_iam_binding" "vault-server-cryptokey-binding" {
+  service_account_id = google_service_account.vault_server_sa.name
+  role               = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+  members = [
+    "serviceAccount:vault-server@${module.project_factory.project_id}.iam.gserviceaccount.com"
+  ]
+}
+
+// RW storage access to Vault Server bucket
+resource "google_storage_bucket_iam_binding" "vault-server-storage-binding" {
+  bucket  = module.storage_bucket.name
+  role    = "roles/storage.objectUser"
+  members = var.vault_server_service_accounts
+}
 
 // Admin storage access to Vault Server backup bucket
 resource "google_storage_bucket_iam_binding" "vault-server-storage-backup-binding" {
@@ -107,6 +147,8 @@ resource "google_storage_bucket_iam_binding" "vault-server-storage-backup-bindin
   members = var.vault_server_service_accounts
 }
 
+// Resources for Vault Server storage backups
+
 resource "google_storage_transfer_job" "vault-server-storage-backup" {
   description  = "Nightly backup of Vault Server storage"
   project      = module.project_factory.project_id
@@ -134,13 +176,6 @@ resource "google_storage_transfer_job" "vault-server-storage-backup" {
   depends_on = [ google_storage_bucket_iam_binding.vault-server-storage-backup-binding ]
 }
 
-// RW storage access to Vault Server bucket
-resource "google_storage_bucket_iam_binding" "vault-server-storage-binding" {
-  bucket  = module.storage_bucket.name
-  role    = "roles/storage.objectUser"
-  members = var.vault_server_service_accounts
-}
-
 # Service account for Git LFS read/write
 resource "google_service_account" "git_lfs_rw_sa" {
   account_id   = "git-lfs-rw"
@@ -197,41 +232,6 @@ resource "google_service_account_iam_binding" "git-lfs-ro-gcs-binding" {
   ]
 }
 
-# Service account for Vault Server
-resource "google_service_account" "vault_server_sa" {
-  account_id   = "vault-server"
-  display_name = "Vault Server"
-  description  = "Terraform-managed service account for Vault server"
-  project      = module.project_factory.project_id
-}
-
-# Use Workload Identity to have the service run as the appropriate service
-# account (bound to a Kubernetes service account)
-resource "google_service_account_iam_binding" "vault-server-sa-wi" {
-  service_account_id = google_service_account.vault_server_sa.name
-  role               = "roles/iam.workloadIdentityUser"
-  members = [
-    "serviceAccount:${module.project_factory.project_id}.svc.id.goog[vault/vault]"
-  ]
-}
-
-# The Vault service account must be granted the roles Cloud KMS Viewer and
-# Cloud KMS CryptoKey Encrypter/Decrypter
-resource "google_service_account_iam_binding" "vault-server-viewer-binding" {
-  service_account_id = google_service_account.vault_server_sa.name
-  role               = "roles/cloudkms.viewer"
-  members = [
-    "serviceAccount:vault-server@${module.project_factory.project_id}.iam.gserviceaccount.com"
-  ]
-}
-
-resource "google_service_account_iam_binding" "vault-server-cryptokey-binding" {
-  service_account_id = google_service_account.vault_server_sa.name
-  role               = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
-  members = [
-    "serviceAccount:vault-server@${module.project_factory.project_id}.iam.gserviceaccount.com"
-  ]
-}
 
 module "service_account_cluster" {
   source     = "terraform-google-modules/service-accounts/google"