Merge branch 'main' into jaireddjawed-feature-cleanup-shadow-secrets

.github/actions/integration-test/action.yml

@@ -58,11 +58,11 @@ runs:
     # Checkout this repo.
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
     - name: Setup go
-      uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version-file: .go-version
     - name: Create Kind Cluster
-      uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+      uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
       with:
         cluster_name: ${{ inputs.kind-cluster-name }}
         config: test/integration/kind/config.yaml
@@ -117,13 +117,13 @@ runs:
         make $make_target VERSION=${{ inputs.version }} INTEGRATION_TESTS_PARALLEL=true SUPPRESS_TF_OUTPUT=true EXPORT_KIND_LOGS_ROOT=${{ steps.create_kind_export_log_root.outputs.log_root }}
     - name: Store kind cluster logs
       if: success()
-      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+      uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
       with:
         name: ${{ steps.create_kind_export_log_root.outputs.log_artifact_name }}
         path: ${{ steps.create_kind_export_log_root.outputs.log_root }}
     - name: Store kind cluster logs failure
       if: failure()
-      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+      uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
       with:
         name: ${{ steps.create_kind_export_log_root.outputs.log_artifact_name }}-failed
         path: ${{ steps.create_kind_export_log_root.outputs.log_root }}

.github/workflows/build.yaml

@@ -38,7 +38,7 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - id: setup-go
-        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
           go-version-file: .go-version
       - name: go mod download all
@@ -78,7 +78,7 @@ jobs:
       - build-pre-checks
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+      - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
           go-version-file: .go-version
       - run: make ci-test
@@ -108,7 +108,7 @@ jobs:
           version: ${{ needs.get-product-version.outputs.product-version }}
           product: ${{ env.PKG_NAME }}
           repositoryOwner: "hashicorp"
-      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+      - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: metadata.json
           path: ${{ steps.generate-metadata-file.outputs.filepath }}
@@ -127,7 +127,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Setup go
-        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
           go-version-file: .go-version
       - name: Build binary
@@ -149,7 +149,7 @@ jobs:
           echo "path=${ZIP_FILE}" >> $GITHUB_OUTPUT
           echo "name=$(basename ${ZIP_FILE})" >> $GITHUB_OUTPUT
       - name: Upload binary
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: ${{ steps.build-binary.outputs.name }}
           path: ${{ steps.build-binary.outputs.path }}
@@ -307,6 +307,7 @@ jobs:
         - "0.6.0"
         - "0.7.1"
         - "0.8.0"
+        - "0.9.0"
     steps:
       - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
@@ -316,7 +317,7 @@ jobs:
         run: |
             docker load --input ${{ github.event.repository.name }}_release-default_linux_amd64_${{ needs.get-product-version.outputs.product-version }}_${{ github.sha }}.docker.tar
       - name: Install kind
-        uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
         with:
           version: "v0.25.0"
           install_only: true
@@ -330,7 +331,7 @@ jobs:
           helm repo add hashicorp https://helm.releases.hashicorp.com
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Setup go
-        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
           go-version-file: .go-version
       - name: Run tests
@@ -345,9 +346,50 @@ jobs:
     outputs:
       # JSON encoded array of k8s versions
       K8S_VERSIONS: '["1.31.2", "1.30.6", "1.29.10", "1.28.15", "1.27.16"]'
-      VAULT_N: "1.17.2"
-      VAULT_N_1: "1.16.6"
-      VAULT_N_2: "1.15.12"
+      VAULT_N: "1.18.2"
+      VAULT_N_1: "1.17.9"
+      VAULT_N_2: "1.16.13"
+  oom-tests:
+    runs-on: ubuntu-latest
+    needs:
+      - get-product-version
+      - build-pre-checks
+      - build-docker
+      - versions
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s-version: ${{ fromJson(needs.versions.outputs.K8S_VERSIONS) }}
+    steps:
+      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: ${{ github.event.repository.name }}_release-default_linux_amd64_${{ needs.get-product-version.outputs.product-version }}_${{ github.sha }}.docker.tar
+      - name: Load docker image
+        shell: bash
+        run: |
+          docker load --input ${{ github.event.repository.name }}_release-default_linux_amd64_${{ needs.get-product-version.outputs.product-version }}_${{ github.sha }}.docker.tar
+      - name: Install kind
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          version: "v0.25.0"
+          install_only: true
+      - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
+        id: setup-helm
+        with:
+          version: "v3.15.1"
+      - name: Add repo
+        shell: bash
+        run: |
+          helm repo add hashicorp https://helm.releases.hashicorp.com
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Setup go
+        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
+        with:
+          go-version-file: .go-version
+      - name: Run tests
+        shell: bash
+        run: |
+          make integration-test-oom KIND_K8S_VERSION="v${{ matrix.k8s-version }}" VERSION=${{ needs.get-product-version.outputs.product-version }}
   latest-vault:
     name: vault:${{ matrix.vault-version }} kind:${{ matrix.k8s-version }} ${{ matrix.installation-method }} enterprise=${{ matrix.vault-enterprise }}
     needs:
@@ -432,6 +474,7 @@ jobs:
       - unit-tests
       - latest-vault
       - latest-k8s
+      - oom-tests
     steps:
     - name: cancelled
       if: ${{ (contains(needs.*.result, 'cancelled')) }}

.go-version

@@ -1 +1 @@
-1.22.8
+1.23.4

.release/security-scan.hcl

@@ -4,13 +4,28 @@
 container {
 	dependencies = true
 	alpine_secdb = true
-	secrets      = true
+	secrets {
+		all = true
+	}
 }
 
 binary {
-	secrets      = true
+	secrets {
+		all = true
+	}
 	go_modules   = true
 	osv          = true
 	oss_index    = false
 	nvd          = false
+	triage {
+		suppress {
+			vulnerabilities = [
+				// GO-2022-0635 is of low severity, and VSO isn't using the affected functionalities
+				// Upgrading to latest version of go-secure-stdlib is not possible at this time.
+				// The required functionality was inadvertently dropped from
+				// github.com/hashicorp/go-secure-stdlib/awsutil during the migration to aws-sdk-go-v2.
+				"GO-2022-0635"
+			]
+		}
+	}
 }

CHANGELOG.md

@@ -1,3 +1,24 @@
+## 0.9.1 (December 11th, 2024)
+
+Fix:
+* Memory: Prevent OOM due to large K8s Secrets cache: [GH-982](https://github.com/hashicorp/vault-secrets-operator/pull/982) [GH-984](https://github.com/hashicorp/vault-secrets-operator/pull/984)
+
+Improvements:
+* add events for HVS client failures: [GH-960](https://github.com/hashicorp/vault-secrets-operator/pull/960)
+* Memory: Use the mutex pool provided by K8s keymutex: [GH-975](https://github.com/hashicorp/vault-secrets-operator/pull/975)
+
+Build:
+* SEC-090: Automated trusted workflow pinning (2024-10-28): [GH-957](https://github.com/hashicorp/vault-secrets-operator/pull/957)
+* Bump K8s version: [GH-968](https://github.com/hashicorp/vault-secrets-operator/pull/968)
+
+Dependency Updates:
+* Bump the gomod-backward-compatible group with 2 updates: [GH-950](https://github.com/hashicorp/vault-secrets-operator/pull/950)
+* Bump the gomod-backward-compatible group across 1 directory with 9 updates: [GH-958](https://github.com/hashicorp/vault-secrets-operator/pull/958)
+* Bump ubi9/ubi-micro from 9.4-15 to 9.5: [GH-970](https://github.com/hashicorp/vault-secrets-operator/pull/970)
+* Bump ubi9/ubi-minimal from 9.4-1227.1726694542 to 9.5: [GH-971](https://github.com/hashicorp/vault-secrets-operator/pull/971)
+* Bump golang.org/x/crypto from 0.28.0 to 0.31.0: [GH-987](https://github.com/hashicorp/vault-secrets-operator/pull/987)
+
+
 ## 0.9.0 (October 8th, 2024)
 
 Features:

Makefile

@@ -345,6 +345,14 @@ integration-test-chart:
 	INTEGRATION_TESTS=true \
 	go test github.com/hashicorp/vault-secrets-operator/test/chart/... $(TESTARGS) -timeout=10m
 
+.PHONY: integration-test-oom
+integration-test-oom:
+	IMAGE_TAG_BASE=$(IMAGE_TAG_BASE) \
+	VERSION=$(VERSION) \
+	INTEGRATION_TESTS=true \
+	KIND_K8S_VERSION=$(KIND_K8S_VERSION) \
+	go test github.com/hashicorp/vault-secrets-operator/test/oom/... $(TESTARGS) -timeout=10m
+
 .PHONY: setup-kind
 setup-kind: ## create a kind cluster for running the acceptance tests locally
 	kind get clusters | grep --silent "^$(KIND_CLUSTER_NAME)$$" || \

api/v1beta1/vaultpkisecret_types.go

@@ -93,7 +93,7 @@ type VaultPKISecretSpec struct {
 	// not when generating a CSR for an intermediate CA.
 	// Should be in duration notation e.g. 120s, 2h, etc.
 	// +kubebuilder:validation:Type=string
-	// +kubebuilder:validation:Pattern=`^([0-9]+(\.[0-9]+)?(s|m|h))$`
+	// +kubebuilder:validation:Pattern=`^([0-9]+(\.[0-9]+)?(s|m|h|d))$`
 	TTL string `json:"ttl,omitempty"`
 
 	// Format for the certificate. Choices: "pem", "der", "pem_bundle".

api/v1beta1/vaultstaticsecret_types.go

@@ -48,7 +48,7 @@ type VaultStaticSecretSpec struct {
 	// not support dynamically reloading a rotated secret.
 	// In that case one, or more RolloutRestartTarget(s) can be configured here. The Operator will
 	// trigger a "rollout-restart" for each target whenever the Vault secret changes between reconciliation events.
-	// All configured targets wil be ignored if HMACSecretData is set to false.
+	// All configured targets will be ignored if HMACSecretData is set to false.
 	// See RolloutRestartTarget for more details.
 	RolloutRestartTargets []RolloutRestartTarget `json:"rolloutRestartTargets,omitempty"`
 	// Destination provides configuration necessary for syncing the Vault secret to Kubernetes.

chart/Chart.yaml

@@ -3,8 +3,8 @@
 
 apiVersion: v2
 name: vault-secrets-operator
-version: 0.9.0
-appVersion: "0.9.0"
+version: 0.9.1
+appVersion: "0.9.1"
 kubeVersion: ">=1.21.0-0"
 description: Official Vault Secrets Operator Chart
 type: application

chart/crds/secrets.hashicorp.com_vaultpkisecrets.yaml

@@ -316,7 +316,7 @@ spec:
                   Note: this only has an effect when generating a CA cert or signing a CA cert,
                   not when generating a CSR for an intermediate CA.
                   Should be in duration notation e.g. 120s, 2h, etc.
-                pattern: ^([0-9]+(\.[0-9]+)?(s|m|h))$
+                pattern: ^([0-9]+(\.[0-9]+)?(s|m|h|d))$
                 type: string
               uriSans:
                 description: The requested URI SANs.

chart/crds/secrets.hashicorp.com_vaultstaticsecrets.yaml

@@ -222,7 +222,7 @@ spec:
                   not support dynamically reloading a rotated secret.
                   In that case one, or more RolloutRestartTarget(s) can be configured here. The Operator will
                   trigger a "rollout-restart" for each target whenever the Vault secret changes between reconciliation events.
-                  All configured targets wil be ignored if HMACSecretData is set to false.
+                  All configured targets will be ignored if HMACSecretData is set to false.
                   See RolloutRestartTarget for more details.
                 items:
                   description: |-

chart/templates/_helpers.tpl

@@ -334,3 +334,14 @@ vaultAuthGlobalRef generates the default VaultAuth spec.vaultAuthGlobalRef.
 {{- $ret | toYaml | nindent 4 -}}
 {{- end -}}
 {{- end -}}
+
+{{/*
+clientCache numLocks
+*/}}
+{{- define "vso.clientCacheNumLocks" -}}
+{{- with .Values.controller.manager.clientCache -}}
+{{- if or .numLocks (eq .numLocks 0) -}}
+--client-cache-num-locks={{ .numLocks }}
+{{- end -}}
+{{- end -}}
+{{- end -}}

chart/templates/deployment.yaml

@@ -75,6 +75,9 @@ spec:
         {{- if .Values.controller.manager.clientCache.cacheSize }}
         - --client-cache-size={{ .Values.controller.manager.clientCache.cacheSize }}
         {{- end }}
+        {{- with include "vso.clientCacheNumLocks" . }}
+        - {{ . }}
+        {{- end }}
         {{- if .Values.controller.manager.maxConcurrentReconciles }}
         - --max-concurrent-reconciles={{ .Values.controller.manager.maxConcurrentReconciles }}
         {{- end }}

chart/values.yaml

@@ -161,7 +161,7 @@ controller:
     image:
       pullPolicy: IfNotPresent
       repository: hashicorp/vault-secrets-operator
-      tag: 0.9.0
+      tag: 0.9.1
 
     # logging
     logging:
@@ -267,6 +267,18 @@ controller:
       # @type: integer
       cacheSize:
 
+      # Defines the number of locks to use for the Vault client cache controller.
+      # May also be set via the `VSO_CLIENT_CACHE_NUM_LOCKS` environment variable.
+      #
+      # Setting this value less than 1 will cause the manager to set the number of locks equal
+      # to the number of logical CPUs of the run host.
+      #
+      # See the VSO help output for more information.
+      #
+      # default: 100
+      # @type: integer
+      numLocks:
+
       # StorageEncryption provides the necessary configuration to encrypt the client storage
       # cache within Kubernetes objects using (required) Vault Transit Engine.
       # This should only be configured when client cache persistence with encryption is enabled and

config/crd/bases/secrets.hashicorp.com_vaultpkisecrets.yaml

@@ -316,7 +316,7 @@ spec:
                   Note: this only has an effect when generating a CA cert or signing a CA cert,
                   not when generating a CSR for an intermediate CA.
                   Should be in duration notation e.g. 120s, 2h, etc.
-                pattern: ^([0-9]+(\.[0-9]+)?(s|m|h))$
+                pattern: ^([0-9]+(\.[0-9]+)?(s|m|h|d))$
                 type: string
               uriSans:
                 description: The requested URI SANs.

config/crd/bases/secrets.hashicorp.com_vaultstaticsecrets.yaml

@@ -222,7 +222,7 @@ spec:
                   not support dynamically reloading a rotated secret.
                   In that case one, or more RolloutRestartTarget(s) can be configured here. The Operator will
                   trigger a "rollout-restart" for each target whenever the Vault secret changes between reconciliation events.
-                  All configured targets wil be ignored if HMACSecretData is set to false.
+                  All configured targets will be ignored if HMACSecretData is set to false.
                   See RolloutRestartTarget for more details.
                 items:
                   description: |-

config/manager/kustomization.yaml

@@ -16,4 +16,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: hashicorp/vault-secrets-operator
-  newTag: 0.9.0
+  newTag: 0.9.1

controllers/hcpvaultsecretsapp_controller.go

@@ -389,7 +389,12 @@ func (r *HCPVaultSecretsAppReconciler) SetupWithManager(mgr ctrl.Manager, opts c
 			&secretsv1beta1.SecretTransformation{},
 			NewEnqueueRefRequestsHandlerST(r.referenceCache, nil),
 		).
-		Watches(
+		// In order to reduce the operator's memory usage, we only watch for the
+		// Secret's metadata. That is sufficient for us to know when a Secret is
+		// deleted. If we ever need to access to the Secret's data, we can always fetch
+		// it from the API server in a RequestHandler, selectively based on the Secret's
+		// labels.
+		WatchesMetadata(
 			&corev1.Secret{},
 			&enqueueOnDeletionRequestHandler{
 				gvk: secretsv1beta1.GroupVersion.WithKind(HCPVaultSecretsApp.String()),

controllers/vaultdynamicsecret_controller.go

@@ -617,7 +617,12 @@ func (r *VaultDynamicSecretReconciler) SetupWithManager(mgr ctrl.Manager, opts c
 			&secretsv1beta1.SecretTransformation{},
 			NewEnqueueRefRequestsHandlerST(r.referenceCache, r.SyncRegistry),
 		).
-		Watches(
+		// In order to reduce the operator's memory usage, we only watch for the
+		// Secret's metadata. That is sufficient for us to know when a Secret is
+		// deleted. If we ever need to access to the Secret's data, we can always fetch
+		// it from the API server in a RequestHandler, selectively based on the Secret's
+		// labels.
+		WatchesMetadata(
 			&corev1.Secret{},
 			&enqueueOnDeletionRequestHandler{
 				gvk: secretsv1beta1.GroupVersion.WithKind(VaultDynamicSecret.String()),