PVC for Traefik Ingress (prevent LetsEncrypt throttling) (#2352)

src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf

@@ -9,7 +9,7 @@ locals {
       "--entrypoints.minio.http.tls.certResolver=letsencrypt",
       "--certificatesresolvers.letsencrypt.acme.tlschallenge",
       "--certificatesresolvers.letsencrypt.acme.email=${var.acme-email}",
-      "--certificatesresolvers.letsencrypt.acme.storage=acme.json",
+      "--certificatesresolvers.letsencrypt.acme.storage=/mnt/acme-certificates/acme.json",
       "--certificatesresolvers.letsencrypt.acme.caserver=${var.acme-server}",
     ]
     self-signed = local.default_cert
@@ -27,6 +27,22 @@ resource "kubernetes_service_account" "main" {
   }
 }
 
+resource "kubernetes_persistent_volume_claim" "traefik_certs_pvc" {
+  metadata {
+    name      = "traefik-ingress-certs"
+    namespace = var.namespace
+  }
+  spec {
+    access_modes = ["ReadWriteOnce"]
+    resources {
+      requests = {
+        storage = "5Gi"
+      }
+    }
+  }
+  wait_until_bound = false
+}
+
 
 resource "kubernetes_cluster_role" "main" {
   metadata {
@@ -215,6 +231,10 @@ resource "kubernetes_deployment" "main" {
           image = "${var.traefik-image.image}:${var.traefik-image.tag}"
           name  = var.name
 
+          volume_mount {
+            mount_path = "/mnt/acme-certificates"
+            name       = "acme-certificates"
+          }
           security_context {
             capabilities {
               drop = ["ALL"]
@@ -326,6 +346,12 @@ resource "kubernetes_deployment" "main" {
             success_threshold     = 1
           }
         }
+        volume {
+          name = "acme-certificates"
+          persistent_volume_claim {
+            claim_name = kubernetes_persistent_volume_claim.traefik_certs_pvc.metadata.0.name
+          }
+        }
       }
     }
   }