Add Vault CI test with deployed Vault with Transit

.github/workflows/ci-test.yml

@@ -0,0 +1,29 @@
+name: CI tests
+
+on:
+  push:
+    branches:
+    - master
+
+jobs:
+  valut-ci-test:
+    name: Test Vault
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.21.4
+        check-latest: true
+      id: go
+    - name: Check out code
+      uses: actions/checkout@v3
+    - name: Deploy Vault
+      run: |
+        ${GITHUB_WORKSPACE}/kesconf/testdata/vault/deploy_vault.sh
+    - name: Test
+      env:
+        GO111MODULE: on
+        GOPROXY: "https://proxy.golang.org,direct"
+      run: |
+         go test ./kesconf -v -vault-ci.config=${GITHUB_WORKSPACE}/kesconf/testdata/vault/kes-config-vault.yml -run="TestVaultCI"

kesconf/aws_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var awsConfigFile = flag.String("aws.config", "", "Path to a KES config file with AWS SecretsManager config")
@@ -18,12 +16,12 @@ func TestAWS(t *testing.T) {
 		t.Skip("AWS SecretsManager tests disabled. Use -aws.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*awsConfigFile)
+	config, err := ReadFile(*awsConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, ok := config.KeyStore.(*kesconf.AWSSecretsManagerKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.AWSSecretsManagerKeyStore{})
+	if _, ok := config.KeyStore.(*AWSSecretsManagerKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &AWSSecretsManagerKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

kesconf/azure_test.go

@@ -2,14 +2,12 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"os"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var azureConfigFile = flag.String("azure.config", "", "Path to a KES config file with Azure KeyVault config")
@@ -24,12 +22,12 @@ func TestAzure(t *testing.T) {
 	}
 	defer file.Close()
 
-	config, err := kesconf.ReadFile(*azureConfigFile)
+	config, err := ReadFile(*azureConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, ok := config.KeyStore.(*kesconf.AzureKeyVaultKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.AzureKeyVaultKeyStore{})
+	if _, ok := config.KeyStore.(*AzureKeyVaultKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &AzureKeyVaultKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

kesconf/edge_test.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"bytes"

kesconf/fortanix_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var fortanixConfigFile = flag.String("fortanix.config", "", "Path to a KES config file with Fortanix SDKMS config")
@@ -18,13 +16,13 @@ func TestFortanix(t *testing.T) {
 		t.Skip("Fortanix tests disabled. Use -fortanix.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*fortanixConfigFile)
+	config, err := ReadFile(*fortanixConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.FortanixKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.FortanixKeyStore{})
+	if _, ok := config.KeyStore.(*FortanixKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &FortanixKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

kesconf/fs_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var FSPath = flag.String("fs.path", "", "Path used for FS tests")
@@ -17,7 +15,7 @@ func TestFS(t *testing.T) {
 	if *FSPath == "" {
 		t.Skip("FS tests disabled. Use -fs.path=<path> to enable them")
 	}
-	config := kesconf.FSKeyStore{
+	config := FSKeyStore{
 		Path: *FSPath,
 	}
 

kesconf/gcp_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var gcpConfigFile = flag.String("gcp.config", "", "Path to a KES config file with GCP SecretManager config")
@@ -18,13 +16,13 @@ func TestGCP(t *testing.T) {
 		t.Skip("GCP tests disabled. Use -gcp.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*gcpConfigFile)
+	config, err := ReadFile(*gcpConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.GCPSecretManagerKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.GCPSecretManagerKeyStore{})
+	if _, ok := config.KeyStore.(*GCPSecretManagerKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &GCPSecretManagerKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

kesconf/gemalto_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var gemaltoConfigFile = flag.String("gemalto.config", "", "Path to a KES config file with Gemalto KeySecure config")
@@ -18,13 +16,13 @@ func TestGemalto(t *testing.T) {
 		t.Skip("Gemalto tests disabled. Use -gemalto.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*gemaltoConfigFile)
+	config, err := ReadFile(*gemaltoConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.KeySecureKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.KeySecureKeyStore{})
+	if _, ok := config.KeyStore.(*KeySecureKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &KeySecureKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

kesconf/keycontrol_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var keyControlConfigFile = flag.String("entrust.config", "", "Path to a KES config file with Entrust KeyControl config")
@@ -18,13 +16,13 @@ func TestKeyControl(t *testing.T) {
 		t.Skip("KeyControl tests disabled. Use -entrust.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*keyControlConfigFile)
+	config, err := ReadFile(*keyControlConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.EntrustKeyControlKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.EntrustKeyControlKeyStore{})
+	if _, ok := config.KeyStore.(*EntrustKeyControlKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &EntrustKeyControlKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

kesconf/testdata/vault/deploy_vault.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+echo "${GITHUB_WORKSPACE=~/kes}"
+
+function main() {
+	# Initialize setup
+	init_setup
+
+	# Install HashiCorp vault
+	install_vault
+
+	# Install latest KES binary for cert etc
+	install_kes
+
+	# Setup vault
+	setup_vault
+}
+
+function init_setup() {
+	echo ""
+	echo "Initialize setup....."
+	echo ""
+	apt update -y && apt upgrade -y && apt install wget unzip
+	sudo chmod a+x /usr/local/bin/yq
+        wget https://releases.hashicorp.com/vault/1.15.2/vault_1.15.2_linux_amd64.zip
+
+	rm -rf /vault/file
+	pkill -9 vault
+	rm -f client.crt client.key private.key public.crt vault.crt vault.key
+}
+
+function install_vault() {
+	echo ""
+	echo "Installing HashiCorp vault....."
+	echo ""
+	sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+        unzip vault_1.15.2_linux_amd64.zip
+        chmod +x vault
+        mv vault /usr/local/bin || sudo mv vault /usr/local/bin
+        vault --version
+	rm -f vault_1.15.2_linux_amd64.zip
+}
+
+function install_kes() {
+	echo ""
+	echo "Installing latest KES binary for certificate etc....."
+	echo ""
+	wget -O kes https://github.com/minio/kes/releases/latest/download/kes-linux-amd64
+	chmod +x kes
+	mv kes /usr/local/bin/kes || sudo mv kes /usr/local/bin/kes
+	kes --version
+}
+
+function setup_vault() {
+	# Create vault certs
+	kes identity new --key vault.key --cert vault.crt --ip "127.0.0.1" localhost
+	mkdir -p /vault/file
+
+	# Start vaule server
+	vault server -config "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/vault-config.json &
+
+	# Generate certs for KES
+	kes identity new --ip "127.0.0.1" localhost --cert public.crt --key private.key
+
+	# Generate certs for client application (to be used by test)
+	kes identity new --key=client.key --cert=client.crt MyApp
+
+	client_id=$(kes identity of client.crt | awk '{print $1}')
+	id="${client_id}" yq e -i '.policy.my-app.identities += [strenv(id)] | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+
+	export VAULT_ADDR='https://127.0.0.1:8200'
+	export VAULT_SKIP_VERIFY=true
+	init_output=$(vault operator init)
+	vault_token=$(echo "$init_output" | grep "Initial Root Token:" | awk -F":" '{print $2}' | xargs)
+	unseal_key1=$(echo "$init_output" | grep "Unseal Key 1:" | awk -F":" '{print $2}' | xargs)
+	unseal_key2=$(echo "$init_output" | grep "Unseal Key 2:" | awk -F":" '{print $2}' | xargs)
+	unseal_key3=$(echo "$init_output" | grep "Unseal Key 3:" | awk -F":" '{print $2}' | xargs)
+	export VAULT_TOKEN=${vault_token}
+	vault operator unseal "${unseal_key1}"
+	vault operator unseal "${unseal_key2}"
+	vault operator unseal "${unseal_key3}"
+	vault secrets enable -version=1 kv
+	vault secrets enable transit
+	vault write -f transit/keys/my-key
+	vault policy write kes-policy kes-policy.hcl
+	vault auth enable approle
+	vault write auth/approle/role/kes-server token_num_uses=0 secret_id_num_uses=0 period=5m
+	vault write auth/approle/role/kes-server policies=kes-policy
+	roleid_output=$(vault read auth/approle/role/kes-server/role-id)
+	role_id=$(echo "$roleid_output" | grep "role_id" | awk -F" " '{print $2}')
+	secretid_output=$(vault write -f auth/approle/role/kes-server/secret-id)
+	secret_id=$(echo "$secretid_output" | grep "secret_id " | awk -F" " '{print $2}')
+	rlid="${role_id}" yq e -i '.keystore.vault.approle.id = strenv(rlid) | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+	sid="${secret_id}" yq e -i '.keystore.vault.approle.secret = strenv(sid) | ..style="double"' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+	kes_private_key="${GITHUB_WORKSPACE}"/kesconf/testdata/vault/private.key
+	kes_public_cert="${GITHUB_WORKSPACE}"/kesconf/testdata/vault/public.crt
+	vault_public_cert="${GITHUB_WORKSPACE}"/kesconf/testdata/vault/vault.crt
+	kes_key="${kes_private_key}" yq e -i '.tls.key = strenv(kes_key)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+	kes_cert="${kes_public_cert}" yq e -i '.tls.cert = strenv(kes_cert)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+	vault_cert="${vault_public_cert}" yq e -i '.keystore.vault.tls.ca = strenv(vault_cert)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+}
+
+main "$@"

kesconf/testdata/vault/kes-config-vault.yml

@@ -0,0 +1,40 @@
+address: 0.0.0.0:7373 # Listen on all network interfaces on port 7373
+
+admin:
+  identity: disabled  # We disable the admin identity since we don't need it in this guide 
+
+tls:
+  key: private.key    # The KES server TLS private key
+  cert: public.crt    # The KES server TLS certificate
+
+policy:
+  my-app:
+    allow:
+    - /v1/key/list/*
+    - /v1/key/generate/*
+    - /v1/key/encrypt/*
+    - /v1/key/decrypt/*
+    - /v1/key/create/*
+    - /v1/identity/describe/*
+    - /v1/identity/list/*
+    - /v1/policy/list/*
+    - /v1/metrics
+    identities:
+    - "" # Use the identity of your client.crt
+
+keystore:
+   vault:
+     endpoint: https://127.0.0.1:8200
+     version:  "v1" # The K/V engine version - either "v1" or "v2".
+     transit:
+       engine: "transit"
+       key: "my-key"
+     approle:
+       id:     "" # Your AppRole ID
+       secret: "" # Your AppRole Secret
+       retry:  15s
+     status:
+       ping: 10s
+     tls:
+       ca: vault.crt # Manually trust the vault certificate since we use self-signed certificates
+