From d18a4b909de7e66a25b1e8ce2b2a4bdc07583de5 Mon Sep 17 00:00:00 2001
From: Shubhendu Ram Tripathi <shubhendu@minio.io>
Date: Wed, 29 Nov 2023 13:04:56 +0530
Subject: [PATCH 1/9] Add Vault CI test with deployed Vault with Transit

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>
---
 .github/workflows/ci-test.yml               | 29 ++++++
 kesconf/aws_test.go                         | 10 +--
 kesconf/azure_test.go                       | 10 +--
 kesconf/edge_test.go                        |  2 +-
 kesconf/fortanix_test.go                    | 10 +--
 kesconf/fs_test.go                          |  6 +-
 kesconf/gcp_test.go                         | 10 +--
 kesconf/gemalto_test.go                     | 10 +--
 kesconf/keycontrol_test.go                  | 10 +--
 kesconf/testdata/vault/deploy_vault.sh      | 97 +++++++++++++++++++++
 kesconf/testdata/vault/kes-config-vault.yml | 40 +++++++++
 kesconf/testdata/vault/kes-policy.hcl       |  9 ++
 kesconf/testdata/vault/vault-config.json    | 21 +++++
 kesconf/vault_ci_test.go                    | 39 +++++++++
 kesconf/vault_test.go                       | 10 +--
 15 files changed, 266 insertions(+), 47 deletions(-)
 create mode 100644 .github/workflows/ci-test.yml
 create mode 100755 kesconf/testdata/vault/deploy_vault.sh
 create mode 100644 kesconf/testdata/vault/kes-config-vault.yml
 create mode 100644 kesconf/testdata/vault/kes-policy.hcl
 create mode 100644 kesconf/testdata/vault/vault-config.json
 create mode 100644 kesconf/vault_ci_test.go

diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
new file mode 100644
index 00000000..80a96ff2
--- /dev/null
+++ b/.github/workflows/ci-test.yml
@@ -0,0 +1,29 @@
+name: CI tests
+
+on:
+  push:
+    branches:
+    - master
+
+jobs:
+  valut-ci-test:
+    name: Test Vault
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.21.4
+        check-latest: true
+      id: go
+    - name: Check out code
+      uses: actions/checkout@v3
+    - name: Deploy Vault
+      run: |
+        ${GITHUB_WORKSPACE}/kesconf/testdata/vault/deploy_vault.sh
+    - name: Test
+      env:
+        GO111MODULE: on
+        GOPROXY: "https://proxy.golang.org,direct"
+      run: |
+         go test ./kesconf -v -vault-ci.config=${GITHUB_WORKSPACE}/kesconf/testdata/vault/kes-config-vault.yml -run="TestVaultCI"
diff --git a/kesconf/aws_test.go b/kesconf/aws_test.go
index 07c20ba7..73306f05 100644
--- a/kesconf/aws_test.go
+++ b/kesconf/aws_test.go
@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var awsConfigFile = flag.String("aws.config", "", "Path to a KES config file with AWS SecretsManager config")
@@ -18,12 +16,12 @@ func TestAWS(t *testing.T) {
 		t.Skip("AWS SecretsManager tests disabled. Use -aws.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*awsConfigFile)
+	config, err := ReadFile(*awsConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, ok := config.KeyStore.(*kesconf.AWSSecretsManagerKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.AWSSecretsManagerKeyStore{})
+	if _, ok := config.KeyStore.(*AWSSecretsManagerKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &AWSSecretsManagerKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)
diff --git a/kesconf/azure_test.go b/kesconf/azure_test.go
index 06d7e8cf..b782df9a 100644
--- a/kesconf/azure_test.go
+++ b/kesconf/azure_test.go
@@ -2,14 +2,12 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"os"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var azureConfigFile = flag.String("azure.config", "", "Path to a KES config file with Azure KeyVault config")
@@ -24,12 +22,12 @@ func TestAzure(t *testing.T) {
 	}
 	defer file.Close()
 
-	config, err := kesconf.ReadFile(*azureConfigFile)
+	config, err := ReadFile(*azureConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, ok := config.KeyStore.(*kesconf.AzureKeyVaultKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.AzureKeyVaultKeyStore{})
+	if _, ok := config.KeyStore.(*AzureKeyVaultKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &AzureKeyVaultKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)
diff --git a/kesconf/edge_test.go b/kesconf/edge_test.go
index addb7fb4..5d3a8352 100644
--- a/kesconf/edge_test.go
+++ b/kesconf/edge_test.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"bytes"
diff --git a/kesconf/fortanix_test.go b/kesconf/fortanix_test.go
index 3e8ebe85..2496c7fb 100644
--- a/kesconf/fortanix_test.go
+++ b/kesconf/fortanix_test.go
@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var fortanixConfigFile = flag.String("fortanix.config", "", "Path to a KES config file with Fortanix SDKMS config")
@@ -18,13 +16,13 @@ func TestFortanix(t *testing.T) {
 		t.Skip("Fortanix tests disabled. Use -fortanix.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*fortanixConfigFile)
+	config, err := ReadFile(*fortanixConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.FortanixKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.FortanixKeyStore{})
+	if _, ok := config.KeyStore.(*FortanixKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &FortanixKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)
diff --git a/kesconf/fs_test.go b/kesconf/fs_test.go
index 2336b873..db70f1f7 100644
--- a/kesconf/fs_test.go
+++ b/kesconf/fs_test.go
@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var FSPath = flag.String("fs.path", "", "Path used for FS tests")
@@ -17,7 +15,7 @@ func TestFS(t *testing.T) {
 	if *FSPath == "" {
 		t.Skip("FS tests disabled. Use -fs.path=<path> to enable them")
 	}
-	config := kesconf.FSKeyStore{
+	config := FSKeyStore{
 		Path: *FSPath,
 	}
 
diff --git a/kesconf/gcp_test.go b/kesconf/gcp_test.go
index f3e6d131..910da96d 100644
--- a/kesconf/gcp_test.go
+++ b/kesconf/gcp_test.go
@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var gcpConfigFile = flag.String("gcp.config", "", "Path to a KES config file with GCP SecretManager config")
@@ -18,13 +16,13 @@ func TestGCP(t *testing.T) {
 		t.Skip("GCP tests disabled. Use -gcp.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*gcpConfigFile)
+	config, err := ReadFile(*gcpConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.GCPSecretManagerKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.GCPSecretManagerKeyStore{})
+	if _, ok := config.KeyStore.(*GCPSecretManagerKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &GCPSecretManagerKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)
diff --git a/kesconf/gemalto_test.go b/kesconf/gemalto_test.go
index 41785706..b22b0e07 100644
--- a/kesconf/gemalto_test.go
+++ b/kesconf/gemalto_test.go
@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var gemaltoConfigFile = flag.String("gemalto.config", "", "Path to a KES config file with Gemalto KeySecure config")
@@ -18,13 +16,13 @@ func TestGemalto(t *testing.T) {
 		t.Skip("Gemalto tests disabled. Use -gemalto.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*gemaltoConfigFile)
+	config, err := ReadFile(*gemaltoConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.KeySecureKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.KeySecureKeyStore{})
+	if _, ok := config.KeyStore.(*KeySecureKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &KeySecureKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)
diff --git a/kesconf/keycontrol_test.go b/kesconf/keycontrol_test.go
index e5d10a8d..d5a1cbc3 100644
--- a/kesconf/keycontrol_test.go
+++ b/kesconf/keycontrol_test.go
@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var keyControlConfigFile = flag.String("entrust.config", "", "Path to a KES config file with Entrust KeyControl config")
@@ -18,13 +16,13 @@ func TestKeyControl(t *testing.T) {
 		t.Skip("KeyControl tests disabled. Use -entrust.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*keyControlConfigFile)
+	config, err := ReadFile(*keyControlConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.EntrustKeyControlKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.EntrustKeyControlKeyStore{})
+	if _, ok := config.KeyStore.(*EntrustKeyControlKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &EntrustKeyControlKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)
diff --git a/kesconf/testdata/vault/deploy_vault.sh b/kesconf/testdata/vault/deploy_vault.sh
new file mode 100755
index 00000000..1f1d89f1
--- /dev/null
+++ b/kesconf/testdata/vault/deploy_vault.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+echo "${GITHUB_WORKSPACE=~/kes}"
+
+function main() {
+	# Initialize setup
+	init_setup
+
+	# Install HashiCorp vault
+	install_vault
+
+	# Install latest KES binary for cert etc
+	install_kes
+
+	# Setup vault
+	setup_vault
+}
+
+function init_setup() {
+	echo ""
+	echo "Initialize setup....."
+	echo ""
+	apt update -y && apt upgrade -y && apt install wget unzip
+	sudo chmod a+x /usr/local/bin/yq
+        wget https://releases.hashicorp.com/vault/1.15.2/vault_1.15.2_linux_amd64.zip
+
+	rm -rf /vault/file
+	pkill -9 vault
+	rm -f client.crt client.key private.key public.crt vault.crt vault.key
+}
+
+function install_vault() {
+	echo ""
+	echo "Installing HashiCorp vault....."
+	echo ""
+	sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+        unzip vault_1.15.2_linux_amd64.zip
+        chmod +x vault
+        mv vault /usr/local/bin || sudo mv vault /usr/local/bin
+        vault --version
+	rm -f vault_1.15.2_linux_amd64.zip
+}
+
+function install_kes() {
+	echo ""
+	echo "Installing latest KES binary for certificate etc....."
+	echo ""
+	wget -O kes https://github.com/minio/kes/releases/latest/download/kes-linux-amd64
+	chmod +x kes
+	mv kes /usr/local/bin/kes || sudo mv kes /usr/local/bin/kes
+	kes --version
+}
+
+function setup_vault() {
+	# Create vault certs
+	kes identity new --key vault.key --cert vault.crt --ip "127.0.0.1" localhost
+	mkdir -p /vault/file
+
+	# Start vaule server
+	vault server -config "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/vault-config.json &
+
+	# Generate certs for KES
+	kes identity new --ip "127.0.0.1" localhost --cert public.crt --key private.key
+
+	# Generate certs for client application (to be used by test)
+	kes identity new --key=client.key --cert=client.crt MyApp
+
+	client_id=$(kes identity of client.crt | awk '{print $1}')
+	id="${client_id}" yq e -i '.policy.my-app.identities += [strenv(id)] | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+
+	export VAULT_ADDR='https://127.0.0.1:8200'
+	export VAULT_SKIP_VERIFY=true
+	init_output=$(vault operator init)
+	vault_token=$(echo "$init_output" | grep "Initial Root Token:" | awk -F":" '{print $2}' | xargs)
+	unseal_key1=$(echo "$init_output" | grep "Unseal Key 1:" | awk -F":" '{print $2}' | xargs)
+	unseal_key2=$(echo "$init_output" | grep "Unseal Key 2:" | awk -F":" '{print $2}' | xargs)
+	unseal_key3=$(echo "$init_output" | grep "Unseal Key 3:" | awk -F":" '{print $2}' | xargs)
+	export VAULT_TOKEN=${vault_token}
+	vault operator unseal "${unseal_key1}"
+	vault operator unseal "${unseal_key2}"
+	vault operator unseal "${unseal_key3}"
+	vault secrets enable -version=1 kv
+	vault secrets enable transit
+	vault write -f transit/keys/my-key
+	vault policy write kes-policy kes-policy.hcl
+	vault auth enable approle
+	vault write auth/approle/role/kes-server token_num_uses=0 secret_id_num_uses=0 period=5m
+	vault write auth/approle/role/kes-server policies=kes-policy
+	roleid_output=$(vault read auth/approle/role/kes-server/role-id)
+	role_id=$(echo "$roleid_output" | grep "role_id" | awk -F" " '{print $2}')
+	secretid_output=$(vault write -f auth/approle/role/kes-server/secret-id)
+	secret_id=$(echo "$secretid_output" | grep "secret_id " | awk -F" " '{print $2}')
+	rlid="${role_id}" yq e -i '.keystore.vault.approle.id = strenv(rlid) | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+	sid="${secret_id}" yq e -i '.keystore.vault.approle.secret = strenv(sid) | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+}
+
+main "$@"
diff --git a/kesconf/testdata/vault/kes-config-vault.yml b/kesconf/testdata/vault/kes-config-vault.yml
new file mode 100644
index 00000000..71d45933
--- /dev/null
+++ b/kesconf/testdata/vault/kes-config-vault.yml
@@ -0,0 +1,40 @@
+address: 0.0.0.0:7373 # Listen on all network interfaces on port 7373
+
+admin:
+  identity: disabled  # We disable the admin identity since we don't need it in this guide 
+
+tls:
+  key: private.key    # The KES server TLS private key
+  cert: public.crt    # The KES server TLS certificate
+
+policy:
+  my-app:
+    allow:
+    - /v1/key/list/*
+    - /v1/key/generate/*
+    - /v1/key/encrypt/*
+    - /v1/key/decrypt/*
+    - /v1/key/create/*
+    - /v1/identity/describe/*
+    - /v1/identity/list/*
+    - /v1/policy/list/*
+    - /v1/metrics
+    identities:
+    - "" # Use the identity of your client.crt
+
+keystore:
+   vault:
+     endpoint: https://127.0.0.1:8200
+     version:  "v1" # The K/V engine version - either "v1" or "v2".
+     transit:
+       engine: "transit"
+       key: "my-key"
+     approle:
+       id:     "" # Your AppRole ID
+       secret: "" # Your AppRole Secret
+       retry:  15s
+     status:
+       ping: 10s
+     tls:
+       ca: vault.crt # Manually trust the vault certificate since we use self-signed certificates
+
diff --git a/kesconf/testdata/vault/kes-policy.hcl b/kesconf/testdata/vault/kes-policy.hcl
new file mode 100644
index 00000000..efb803c2
--- /dev/null
+++ b/kesconf/testdata/vault/kes-policy.hcl
@@ -0,0 +1,9 @@
+path "transit/encrypt/my-key" {
+   capabilities = [ "update" ]
+}
+path "transit/decrypt/my-key" {
+   capabilities = [ "update" ]
+}
+path "kv/*" {
+   capabilities = [ "create", "read", "delete", "list" ]
+}
diff --git a/kesconf/testdata/vault/vault-config.json b/kesconf/testdata/vault/vault-config.json
new file mode 100644
index 00000000..79055ed0
--- /dev/null
+++ b/kesconf/testdata/vault/vault-config.json
@@ -0,0 +1,21 @@
+{
+  "api_addr": "https://127.0.0.1:8200",
+  "backend": {
+    "file": {
+      "path": "/vault/file"
+    }
+  },
+
+  "default_lease_ttl": "168h",
+  "max_lease_ttl": "720h",
+
+  "listener": {
+    "tcp": {
+      "address": "0.0.0.0:8200",
+      "tls_cert_file": "vault.crt",
+      "tls_key_file": "vault.key",
+      "tls_min_version": "tls12"
+    }
+  }
+}
+
diff --git a/kesconf/vault_ci_test.go b/kesconf/vault_ci_test.go
new file mode 100644
index 00000000..1bd3527b
--- /dev/null
+++ b/kesconf/vault_ci_test.go
@@ -0,0 +1,39 @@
+// Copyright 2023 - MinIO, Inc. All rights reserved.
+// Use of this source code is governed by the AGPLv3
+// license that can be found in the LICENSE file.
+
+package kesconf
+
+import (
+	"flag"
+	"testing"
+)
+
+var vaultCfgFile = flag.String("vault-ci.config", "", "Path to a KES config file with Hashicorp Vault config")
+
+func TestVaultCI(t *testing.T) {
+	if *vaultCfgFile == "" {
+		t.Skip("Vault tests disabled. Use -vault-ci.config=<FILE> to enable them")
+	}
+
+	config, err := ReadFile(*vaultCfgFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, ok := config.KeyStore.(*VaultKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &VaultKeyStore{})
+	}
+
+	ctx, cancel := testingContext(t)
+	defer cancel()
+
+	store, err := config.KeyStore.Connect(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Create", func(t *testing.T) { testCreate(ctx, store, t, RandString(ranStringLength)) })
+	t.Run("Get", func(t *testing.T) { testGet(ctx, store, t, RandString(ranStringLength)) })
+	t.Run("Status", func(t *testing.T) { testStatus(ctx, store, t) })
+}
diff --git a/kesconf/vault_test.go b/kesconf/vault_test.go
index 00f15d8c..eb2d3d0b 100644
--- a/kesconf/vault_test.go
+++ b/kesconf/vault_test.go
@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var vaultConfigFile = flag.String("vault.config", "", "Path to a KES config file with Hashicorp Vault config")
@@ -18,13 +16,13 @@ func TestVault(t *testing.T) {
 		t.Skip("Vault tests disabled. Use -vault.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*vaultConfigFile)
+	config, err := ReadFile(*vaultConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.VaultKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.VaultKeyStore{})
+	if _, ok := config.KeyStore.(*VaultKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &VaultKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

From 7e43f4938dbe4a2f56a0cc9b3b757f1c22fff71b Mon Sep 17 00:00:00 2001
From: Shubhendu Ram Tripathi <shubhendu@minio.io>
Date: Fri, 1 Dec 2023 21:16:41 +0530
Subject: [PATCH 2/9] Set correct certificate paths

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>
---
 kesconf/testdata/vault/deploy_vault.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/kesconf/testdata/vault/deploy_vault.sh b/kesconf/testdata/vault/deploy_vault.sh
index 1f1d89f1..8e7786a2 100755
--- a/kesconf/testdata/vault/deploy_vault.sh
+++ b/kesconf/testdata/vault/deploy_vault.sh
@@ -92,6 +92,12 @@ function setup_vault() {
 	secret_id=$(echo "$secretid_output" | grep "secret_id " | awk -F" " '{print $2}')
 	rlid="${role_id}" yq e -i '.keystore.vault.approle.id = strenv(rlid) | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
 	sid="${secret_id}" yq e -i '.keystore.vault.approle.secret = strenv(sid) | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+	kes_private_key="${GITHUB_WORKSPACE}"/kesconf/testdata/vault/private.key
+	kes_public_cert="${GITHUB_WORKSPACE}"/kesconf/testdata/vault/public.crt
+	vault_public_cert="${GITHUB_WORKSPACE}"/kesconf/testdata/vault/vault.crt
+	kes_key="${kes_private_key}" yq e -i '.tls.key = strenv(kes_key)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+	kes_cert="${kes_public_cert}" yq e -i '.tls.cert = strenv(kes_cert)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+	vault_cert="${vault_public_cert}" yq e -i '.keystore.vault.tls.ca = strenv(vault_cert)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
 }
 
 main "$@"

From e0f30819921453a94c20bbe05634142cd451e46b Mon Sep 17 00:00:00 2001
From: Harshavardhana <harsha@minio.io>
Date: Tue, 9 Jan 2024 15:44:21 -0800
Subject: [PATCH 3/9] Apply suggestions from code review

Signed-off-by: Harshavardhana <harsha@minio.io>
---
 .github/workflows/ci-test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
index 80a96ff2..4285317a 100644
--- a/.github/workflows/ci-test.yml
+++ b/.github/workflows/ci-test.yml
@@ -1,7 +1,7 @@
-name: CI tests
+name: Vault Integration CI tests
 
 on:
-  push:
+  pull_request:
     branches:
     - master
 
@@ -13,7 +13,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.21.4
+        go-version: 1.21.5
         check-latest: true
       id: go
     - name: Check out code

From 486042087d8ddf215e3b00d55bfc1642af2f7fbb Mon Sep 17 00:00:00 2001
From: Shubhendu Ram Tripathi <shubhendu@minio.io>
Date: Wed, 10 Jan 2024 11:13:05 +0530
Subject: [PATCH 4/9] Run commands as sudo when needed

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>
---
 kesconf/testdata/vault/deploy_vault.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/kesconf/testdata/vault/deploy_vault.sh b/kesconf/testdata/vault/deploy_vault.sh
index 8e7786a2..a476ed55 100755
--- a/kesconf/testdata/vault/deploy_vault.sh
+++ b/kesconf/testdata/vault/deploy_vault.sh
@@ -20,12 +20,14 @@ function init_setup() {
 	echo ""
 	echo "Initialize setup....."
 	echo ""
-	apt update -y && apt upgrade -y && apt install wget unzip
+	apt update -y || sudo apt update -y
+	apt upgrade -y || sudo apt upgrade -y
+	apt install wget unzip || sudo apt install wget unzip
 	sudo chmod a+x /usr/local/bin/yq
         wget https://releases.hashicorp.com/vault/1.15.2/vault_1.15.2_linux_amd64.zip
 
-	rm -rf /vault/file
-	pkill -9 vault
+	rm -rf /vault/file || sudo rm -rf /vault/file
+	pkill -9 vault || sudo pkill -9 vault
 	rm -f client.crt client.key private.key public.crt vault.crt vault.key
 }
 

From 88caa9ca3effa0d7c7f55d8d107b7fd72a544fb9 Mon Sep 17 00:00:00 2001
From: Shubhendu Ram Tripathi <shubhendu@minio.io>
Date: Wed, 10 Jan 2024 11:30:07 +0530
Subject: [PATCH 5/9] Verbose logging of commands and config files

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>
---
 kesconf/testdata/vault/deploy_vault.sh   | 20 +++++++++++++++++---
 kesconf/testdata/vault/vault-config.json |  2 +-
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/kesconf/testdata/vault/deploy_vault.sh b/kesconf/testdata/vault/deploy_vault.sh
index a476ed55..b1cef753 100755
--- a/kesconf/testdata/vault/deploy_vault.sh
+++ b/kesconf/testdata/vault/deploy_vault.sh
@@ -26,7 +26,7 @@ function init_setup() {
 	sudo chmod a+x /usr/local/bin/yq
         wget https://releases.hashicorp.com/vault/1.15.2/vault_1.15.2_linux_amd64.zip
 
-	rm -rf /vault/file || sudo rm -rf /vault/file
+	rm -rf /tmp/vault/file || sudo rm -rf /tmp/vault/file
 	pkill -9 vault || sudo pkill -9 vault
 	rm -f client.crt client.key private.key public.crt vault.crt vault.key
 }
@@ -55,17 +55,27 @@ function install_kes() {
 
 function setup_vault() {
 	# Create vault certs
+	echo "Run: kes identity new --key vault.key --cert vault.crt --ip \"127.0.0.1\" localhost"
+	echi ""
 	kes identity new --key vault.key --cert vault.crt --ip "127.0.0.1" localhost
-	mkdir -p /vault/file
+	mkdir -p /tmp/vault/file || sudo mkdir -p /tmp/vault/file
+	echo ""
 
-	# Start vaule server
+	# Start vault server
+	echo "Starting vault server...."
+	echo "Run: vault server -config \"${GITHUB_WORKSPACE}\"/kesconf/testdata/vault/vault-config.json &"
 	vault server -config "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/vault-config.json &
+	echo ""
 
 	# Generate certs for KES
+	echo "Run: kes identity new --ip \"127.0.0.1\" localhost --cert public.crt --key private.key"
 	kes identity new --ip "127.0.0.1" localhost --cert public.crt --key private.key
+	echo ""
 
 	# Generate certs for client application (to be used by test)
+	echo "Run: kes identity new --key=client.key --cert=client.crt MyApp"
 	kes identity new --key=client.key --cert=client.crt MyApp
+	echo ""
 
 	client_id=$(kes identity of client.crt | awk '{print $1}')
 	id="${client_id}" yq e -i '.policy.my-app.identities += [strenv(id)] | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
@@ -100,6 +110,10 @@ function setup_vault() {
 	kes_key="${kes_private_key}" yq e -i '.tls.key = strenv(kes_key)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
 	kes_cert="${kes_public_cert}" yq e -i '.tls.cert = strenv(kes_cert)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
 	vault_cert="${vault_public_cert}" yq e -i '.keystore.vault.tls.ca = strenv(vault_cert)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+
+	echo "Content of \"${GITHUB_WORKSPACE}\"/kesconf/testdata/vault/kes-config-vault.yml"
+	cat "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+	echo ""
 }
 
 main "$@"
diff --git a/kesconf/testdata/vault/vault-config.json b/kesconf/testdata/vault/vault-config.json
index 79055ed0..951d230e 100644
--- a/kesconf/testdata/vault/vault-config.json
+++ b/kesconf/testdata/vault/vault-config.json
@@ -2,7 +2,7 @@
   "api_addr": "https://127.0.0.1:8200",
   "backend": {
     "file": {
-      "path": "/vault/file"
+      "path": "/tmp/vault/file"
     }
   },
 

From 4fef26a6024c7b956f6c959b0b721303e88e5560 Mon Sep 17 00:00:00 2001
From: Shubhendu Ram Tripathi <shubhendu@minio.io>
Date: Wed, 10 Jan 2024 11:51:26 +0530
Subject: [PATCH 6/9] Disable mlock by default

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>
---
 kesconf/testdata/vault/deploy_vault.sh   | 20 ++++++++++++++++----
 kesconf/testdata/vault/vault-config.json |  1 +
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/kesconf/testdata/vault/deploy_vault.sh b/kesconf/testdata/vault/deploy_vault.sh
index b1cef753..ba284f9c 100755
--- a/kesconf/testdata/vault/deploy_vault.sh
+++ b/kesconf/testdata/vault/deploy_vault.sh
@@ -55,26 +55,37 @@ function install_kes() {
 
 function setup_vault() {
 	# Create vault certs
+	echo "==================================================================================="
 	echo "Run: kes identity new --key vault.key --cert vault.crt --ip \"127.0.0.1\" localhost"
-	echi ""
+	echo ""
 	kes identity new --key vault.key --cert vault.crt --ip "127.0.0.1" localhost
+	realpath vault.key
+	realpath vault.crt
 	mkdir -p /tmp/vault/file || sudo mkdir -p /tmp/vault/file
 	echo ""
 
 	# Start vault server
+	echo "========================="
 	echo "Starting vault server...."
 	echo "Run: vault server -config \"${GITHUB_WORKSPACE}\"/kesconf/testdata/vault/vault-config.json &"
 	vault server -config "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/vault-config.json &
+	ps -ef | grep vault
 	echo ""
 
 	# Generate certs for KES
+	echo "======================================================================================"
 	echo "Run: kes identity new --ip \"127.0.0.1\" localhost --cert public.crt --key private.key"
 	kes identity new --ip "127.0.0.1" localhost --cert public.crt --key private.key
+	realpath public.crt
+	realpath private.key
 	echo ""
 
 	# Generate certs for client application (to be used by test)
+	echo "=============================================================="
 	echo "Run: kes identity new --key=client.key --cert=client.crt MyApp"
 	kes identity new --key=client.key --cert=client.crt MyApp
+	realpath client.key
+	realpath client.crt
 	echo ""
 
 	client_id=$(kes identity of client.crt | awk '{print $1}')
@@ -104,13 +115,14 @@ function setup_vault() {
 	secret_id=$(echo "$secretid_output" | grep "secret_id " | awk -F" " '{print $2}')
 	rlid="${role_id}" yq e -i '.keystore.vault.approle.id = strenv(rlid) | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
 	sid="${secret_id}" yq e -i '.keystore.vault.approle.secret = strenv(sid) | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
-	kes_private_key="${GITHUB_WORKSPACE}"/kesconf/testdata/vault/private.key
-	kes_public_cert="${GITHUB_WORKSPACE}"/kesconf/testdata/vault/public.crt
-	vault_public_cert="${GITHUB_WORKSPACE}"/kesconf/testdata/vault/vault.crt
+	kes_private_key="${GITHUB_WORKSPACE}"/private.key
+	kes_public_cert="${GITHUB_WORKSPACE}"/public.crt
+	vault_public_cert="${GITHUB_WORKSPACE}"/vault.crt
 	kes_key="${kes_private_key}" yq e -i '.tls.key = strenv(kes_key)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
 	kes_cert="${kes_public_cert}" yq e -i '.tls.cert = strenv(kes_cert)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
 	vault_cert="${vault_public_cert}" yq e -i '.keystore.vault.tls.ca = strenv(vault_cert)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
 
+	echo "=============================================================================="
 	echo "Content of \"${GITHUB_WORKSPACE}\"/kesconf/testdata/vault/kes-config-vault.yml"
 	cat "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
 	echo ""
diff --git a/kesconf/testdata/vault/vault-config.json b/kesconf/testdata/vault/vault-config.json
index 951d230e..e2c1d7d3 100644
--- a/kesconf/testdata/vault/vault-config.json
+++ b/kesconf/testdata/vault/vault-config.json
@@ -8,6 +8,7 @@
 
   "default_lease_ttl": "168h",
   "max_lease_ttl": "720h",
+  "disable_mlock": true,
 
   "listener": {
     "tcp": {

From d17bbeaa4c34ca653018b55131a4d9fdc174fa50 Mon Sep 17 00:00:00 2001
From: Shubhendu Ram Tripathi <shubhendu@minio.io>
Date: Wed, 10 Jan 2024 14:50:06 +0530
Subject: [PATCH 7/9] Trying to fix list issue

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>
---
 kesconf/testdata/vault/kes-policy.hcl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/kesconf/testdata/vault/kes-policy.hcl b/kesconf/testdata/vault/kes-policy.hcl
index efb803c2..daa8bf9d 100644
--- a/kesconf/testdata/vault/kes-policy.hcl
+++ b/kesconf/testdata/vault/kes-policy.hcl
@@ -7,3 +7,6 @@ path "transit/decrypt/my-key" {
 path "kv/*" {
    capabilities = [ "create", "read", "delete", "list" ]
 }
+path "secret/data/*" {
+  capabilities = ["list"]
+}

From 7e04980c6b7f0edc36e7616a0771c829b19086ec Mon Sep 17 00:00:00 2001
From: Shubhendu Ram Tripathi <shubhendu@minio.io>
Date: Wed, 10 Jan 2024 16:47:21 +0530
Subject: [PATCH 8/9] Simplify kes config

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>
---
 .github/workflows/ci-test.yml               |  2 +-
 kesconf/testdata/vault/deploy_vault.sh      | 37 +++++++--------------
 kesconf/testdata/vault/kes-config-vault.yml | 10 +++---
 3 files changed, 18 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
index 4285317a..ef31c28d 100644
--- a/.github/workflows/ci-test.yml
+++ b/.github/workflows/ci-test.yml
@@ -26,4 +26,4 @@ jobs:
         GO111MODULE: on
         GOPROXY: "https://proxy.golang.org,direct"
       run: |
-         go test ./kesconf -v -vault-ci.config=${GITHUB_WORKSPACE}/kesconf/testdata/vault/kes-config-vault.yml -run="TestVaultCI"
+         source ${GITHUB_WORKSPACE}/env.sh && go test ./kesconf -v -vault-ci.config=${GITHUB_WORKSPACE}/kesconf/testdata/vault/kes-config-vault.yml -run="TestVaultCI"
diff --git a/kesconf/testdata/vault/deploy_vault.sh b/kesconf/testdata/vault/deploy_vault.sh
index ba284f9c..53f5ef42 100755
--- a/kesconf/testdata/vault/deploy_vault.sh
+++ b/kesconf/testdata/vault/deploy_vault.sh
@@ -72,27 +72,9 @@ function setup_vault() {
 	ps -ef | grep vault
 	echo ""
 
-	# Generate certs for KES
-	echo "======================================================================================"
-	echo "Run: kes identity new --ip \"127.0.0.1\" localhost --cert public.crt --key private.key"
-	kes identity new --ip "127.0.0.1" localhost --cert public.crt --key private.key
-	realpath public.crt
-	realpath private.key
-	echo ""
-
-	# Generate certs for client application (to be used by test)
-	echo "=============================================================="
-	echo "Run: kes identity new --key=client.key --cert=client.crt MyApp"
-	kes identity new --key=client.key --cert=client.crt MyApp
-	realpath client.key
-	realpath client.crt
-	echo ""
-
-	client_id=$(kes identity of client.crt | awk '{print $1}')
-	id="${client_id}" yq e -i '.policy.my-app.identities += [strenv(id)] | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
-
 	export VAULT_ADDR='https://127.0.0.1:8200'
 	export VAULT_SKIP_VERIFY=true
+	export KES_API_KEY=kes:v1:AP6gQlUXjWj5iY1WkqeXKIR0OXTpyoiHa81XTY7ISy3l
 	init_output=$(vault operator init)
 	vault_token=$(echo "$init_output" | grep "Initial Root Token:" | awk -F":" '{print $2}' | xargs)
 	unseal_key1=$(echo "$init_output" | grep "Unseal Key 1:" | awk -F":" '{print $2}' | xargs)
@@ -113,19 +95,24 @@ function setup_vault() {
 	role_id=$(echo "$roleid_output" | grep "role_id" | awk -F" " '{print $2}')
 	secretid_output=$(vault write -f auth/approle/role/kes-server/secret-id)
 	secret_id=$(echo "$secretid_output" | grep "secret_id " | awk -F" " '{print $2}')
-	rlid="${role_id}" yq e -i '.keystore.vault.approle.id = strenv(rlid) | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
-	sid="${secret_id}" yq e -i '.keystore.vault.approle.secret = strenv(sid) | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
-	kes_private_key="${GITHUB_WORKSPACE}"/private.key
-	kes_public_cert="${GITHUB_WORKSPACE}"/public.crt
+	export VAULT_APPROLE_ID="${role_id}"
+	export VAULT_APPROLE_SECRET="${secret_id}"
 	vault_public_cert="${GITHUB_WORKSPACE}"/vault.crt
-	kes_key="${kes_private_key}" yq e -i '.tls.key = strenv(kes_key)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
-	kes_cert="${kes_public_cert}" yq e -i '.tls.cert = strenv(kes_cert)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
 	vault_cert="${vault_public_cert}" yq e -i '.keystore.vault.tls.ca = strenv(vault_cert)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
 
 	echo "=============================================================================="
 	echo "Content of \"${GITHUB_WORKSPACE}\"/kesconf/testdata/vault/kes-config-vault.yml"
 	cat "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
 	echo ""
+
+	cat > env.sh <<EOF
+#!/usr/bin/env bash
+
+export KES_API_KEY=kes:v1:AP6gQlUXjWj5iY1WkqeXKIR0OXTpyoiHa81XTY7ISy3l
+export VAULT_APPROLE_ID=${role_id}
+export VAULT_APPROLE_SECRET=${secret_id}
+EOF
+	cat env.sh
 }
 
 main "$@"
diff --git a/kesconf/testdata/vault/kes-config-vault.yml b/kesconf/testdata/vault/kes-config-vault.yml
index 71d45933..574dee67 100644
--- a/kesconf/testdata/vault/kes-config-vault.yml
+++ b/kesconf/testdata/vault/kes-config-vault.yml
@@ -4,8 +4,8 @@ admin:
   identity: disabled  # We disable the admin identity since we don't need it in this guide 
 
 tls:
-  key: private.key    # The KES server TLS private key
-  cert: public.crt    # The KES server TLS certificate
+  key: "_"    # The KES server TLS private key
+  cert: "_"   # The KES server TLS certificate
 
 policy:
   my-app:
@@ -20,7 +20,7 @@ policy:
     - /v1/policy/list/*
     - /v1/metrics
     identities:
-    - "" # Use the identity of your client.crt
+    - "51da9df6a350ea93c68d022a90421487d13897d1d808c88f2d03ad795cbdc609" # Use the identity of your client.crt
 
 keystore:
    vault:
@@ -30,8 +30,8 @@ keystore:
        engine: "transit"
        key: "my-key"
      approle:
-       id:     "" # Your AppRole ID
-       secret: "" # Your AppRole Secret
+       id:  ${VAULT_APPROLE_ID} # Your AppRole ID
+       secret: ${VAULT_APPROLE_SECRET} # Your AppRole Secret
        retry:  15s
      status:
        ping: 10s

From 086ed9515b310309b4d0627f2581057e8f1e1562 Mon Sep 17 00:00:00 2001
From: Shubhendu Ram Tripathi <shubhendu@minio.io>
Date: Wed, 10 Jan 2024 17:30:34 +0530
Subject: [PATCH 9/9] Debug CI failure

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>
---
 kesconf/testdata/vault/deploy_vault.sh | 2 ++
 kesconf/vault_ci_test.go               | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/kesconf/testdata/vault/deploy_vault.sh b/kesconf/testdata/vault/deploy_vault.sh
index 53f5ef42..c97059d0 100755
--- a/kesconf/testdata/vault/deploy_vault.sh
+++ b/kesconf/testdata/vault/deploy_vault.sh
@@ -111,6 +111,8 @@ function setup_vault() {
 export KES_API_KEY=kes:v1:AP6gQlUXjWj5iY1WkqeXKIR0OXTpyoiHa81XTY7ISy3l
 export VAULT_APPROLE_ID=${role_id}
 export VAULT_APPROLE_SECRET=${secret_id}
+export VAULT_ADDR=https://127.0.0.1:8200
+export VAULT_SKIP_VERIFY=true
 EOF
 	cat env.sh
 }
diff --git a/kesconf/vault_ci_test.go b/kesconf/vault_ci_test.go
index 1bd3527b..c2057311 100644
--- a/kesconf/vault_ci_test.go
+++ b/kesconf/vault_ci_test.go
@@ -6,6 +6,8 @@ package kesconf
 
 import (
 	"flag"
+	"fmt"
+	"os"
 	"testing"
 )
 
@@ -21,6 +23,9 @@ func TestVaultCI(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	fmt.Println("VAULT_APPROLE_ID=", os.Getenv("VAULT_APPROLE_ID"))
+	fmt.Println("VAULT_APPROLE_SECRET=", os.Getenv("VAULT_APPROLE_SECRET"))
+	fmt.Println("Keystore AppRole: ", config.KeyStore.(*VaultKeyStore).AppRole)
 	if _, ok := config.KeyStore.(*VaultKeyStore); !ok {
 		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &VaultKeyStore{})
 	}