Add Vault CI test with deployed Vault with Transit

.github/workflows/ci-test.yml

@@ -0,0 +1,29 @@
+name: Vault Integration CI tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  valut-ci-test:
+    name: Test Vault
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.21.5
+        check-latest: true
+      id: go
+    - name: Check out code
+      uses: actions/checkout@v3
+    - name: Deploy Vault
+      run: |
+        ${GITHUB_WORKSPACE}/kesconf/testdata/vault/deploy_vault.sh
+    - name: Test
+      env:
+        GO111MODULE: on
+        GOPROXY: "https://proxy.golang.org,direct"
+      run: |
+         source ${GITHUB_WORKSPACE}/env.sh && go test ./kesconf -v -vault-ci.config=${GITHUB_WORKSPACE}/kesconf/testdata/vault/kes-config-vault.yml -run="TestVaultCI"

kesconf/aws_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var awsConfigFile = flag.String("aws.config", "", "Path to a KES config file with AWS SecretsManager config")
@@ -18,12 +16,12 @@ func TestAWS(t *testing.T) {
 		t.Skip("AWS SecretsManager tests disabled. Use -aws.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*awsConfigFile)
+	config, err := ReadFile(*awsConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, ok := config.KeyStore.(*kesconf.AWSSecretsManagerKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.AWSSecretsManagerKeyStore{})
+	if _, ok := config.KeyStore.(*AWSSecretsManagerKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &AWSSecretsManagerKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

kesconf/azure_test.go

@@ -2,14 +2,12 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"os"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var azureConfigFile = flag.String("azure.config", "", "Path to a KES config file with Azure KeyVault config")
@@ -24,12 +22,12 @@ func TestAzure(t *testing.T) {
 	}
 	defer file.Close()
 
-	config, err := kesconf.ReadFile(*azureConfigFile)
+	config, err := ReadFile(*azureConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, ok := config.KeyStore.(*kesconf.AzureKeyVaultKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.AzureKeyVaultKeyStore{})
+	if _, ok := config.KeyStore.(*AzureKeyVaultKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &AzureKeyVaultKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

kesconf/edge_test.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"bytes"

kesconf/fortanix_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var fortanixConfigFile = flag.String("fortanix.config", "", "Path to a KES config file with Fortanix SDKMS config")
@@ -18,13 +16,13 @@ func TestFortanix(t *testing.T) {
 		t.Skip("Fortanix tests disabled. Use -fortanix.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*fortanixConfigFile)
+	config, err := ReadFile(*fortanixConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.FortanixKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.FortanixKeyStore{})
+	if _, ok := config.KeyStore.(*FortanixKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &FortanixKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

kesconf/fs_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var FSPath = flag.String("fs.path", "", "Path used for FS tests")
@@ -17,7 +15,7 @@ func TestFS(t *testing.T) {
 	if *FSPath == "" {
 		t.Skip("FS tests disabled. Use -fs.path=<path> to enable them")
 	}
-	config := kesconf.FSKeyStore{
+	config := FSKeyStore{
 		Path: *FSPath,
 	}
 

kesconf/gcp_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var gcpConfigFile = flag.String("gcp.config", "", "Path to a KES config file with GCP SecretManager config")
@@ -18,13 +16,13 @@ func TestGCP(t *testing.T) {
 		t.Skip("GCP tests disabled. Use -gcp.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*gcpConfigFile)
+	config, err := ReadFile(*gcpConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.GCPSecretManagerKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.GCPSecretManagerKeyStore{})
+	if _, ok := config.KeyStore.(*GCPSecretManagerKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &GCPSecretManagerKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

kesconf/gemalto_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var gemaltoConfigFile = flag.String("gemalto.config", "", "Path to a KES config file with Gemalto KeySecure config")
@@ -18,13 +16,13 @@ func TestGemalto(t *testing.T) {
 		t.Skip("Gemalto tests disabled. Use -gemalto.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*gemaltoConfigFile)
+	config, err := ReadFile(*gemaltoConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.KeySecureKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.KeySecureKeyStore{})
+	if _, ok := config.KeyStore.(*KeySecureKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &KeySecureKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

kesconf/keycontrol_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var keyControlConfigFile = flag.String("entrust.config", "", "Path to a KES config file with Entrust KeyControl config")
@@ -18,13 +16,13 @@ func TestKeyControl(t *testing.T) {
 		t.Skip("KeyControl tests disabled. Use -entrust.config=<FILE> to enable them")
 	}
 
-	config, err := kesconf.ReadFile(*keyControlConfigFile)
+	config, err := ReadFile(*keyControlConfigFile)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if _, ok := config.KeyStore.(*kesconf.EntrustKeyControlKeyStore); !ok {
-		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.EntrustKeyControlKeyStore{})
+	if _, ok := config.KeyStore.(*EntrustKeyControlKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &EntrustKeyControlKeyStore{})
 	}
 
 	ctx, cancel := testingContext(t)

kesconf/testdata/vault/deploy_vault.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+echo "${GITHUB_WORKSPACE=~/kes}"
+
+function main() {
+	# Initialize setup
+	init_setup
+
+	# Install HashiCorp vault
+	install_vault
+
+	# Install latest KES binary for cert etc
+	install_kes
+
+	# Setup vault
+	setup_vault
+}
+
+function init_setup() {
+	echo ""
+	echo "Initialize setup....."
+	echo ""
+	apt update -y || sudo apt update -y
+	apt upgrade -y || sudo apt upgrade -y
+	apt install wget unzip || sudo apt install wget unzip
+	sudo chmod a+x /usr/local/bin/yq
+        wget https://releases.hashicorp.com/vault/1.15.2/vault_1.15.2_linux_amd64.zip
+
+	rm -rf /tmp/vault/file || sudo rm -rf /tmp/vault/file
+	pkill -9 vault || sudo pkill -9 vault
+	rm -f client.crt client.key private.key public.crt vault.crt vault.key
+}
+
+function install_vault() {
+	echo ""
+	echo "Installing HashiCorp vault....."
+	echo ""
+	sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+        unzip vault_1.15.2_linux_amd64.zip
+        chmod +x vault
+        mv vault /usr/local/bin || sudo mv vault /usr/local/bin
+        vault --version
+	rm -f vault_1.15.2_linux_amd64.zip
+}
+
+function install_kes() {
+	echo ""
+	echo "Installing latest KES binary for certificate etc....."
+	echo ""
+	wget -O kes https://github.com/minio/kes/releases/latest/download/kes-linux-amd64
+	chmod +x kes
+	mv kes /usr/local/bin/kes || sudo mv kes /usr/local/bin/kes
+	kes --version
+}
+
+function setup_vault() {
+	# Create vault certs
+	echo "==================================================================================="
+	echo "Run: kes identity new --key vault.key --cert vault.crt --ip \"127.0.0.1\" localhost"
+	echo ""
+	kes identity new --key vault.key --cert vault.crt --ip "127.0.0.1" localhost
+	realpath vault.key
+	realpath vault.crt
+	mkdir -p /tmp/vault/file || sudo mkdir -p /tmp/vault/file
+	echo ""
+
+	# Start vault server
+	echo "========================="
+	echo "Starting vault server...."
+	echo "Run: vault server -config \"${GITHUB_WORKSPACE}\"/kesconf/testdata/vault/vault-config.json &"
+	vault server -config "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/vault-config.json &
+	ps -ef | grep vault
+	echo ""
+
+	export VAULT_ADDR='https://127.0.0.1:8200'
+	export VAULT_SKIP_VERIFY=true
+	export KES_API_KEY=kes:v1:AP6gQlUXjWj5iY1WkqeXKIR0OXTpyoiHa81XTY7ISy3l
+	init_output=$(vault operator init)
+	vault_token=$(echo "$init_output" | grep "Initial Root Token:" | awk -F":" '{print $2}' | xargs)
+	unseal_key1=$(echo "$init_output" | grep "Unseal Key 1:" | awk -F":" '{print $2}' | xargs)
+	unseal_key2=$(echo "$init_output" | grep "Unseal Key 2:" | awk -F":" '{print $2}' | xargs)
+	unseal_key3=$(echo "$init_output" | grep "Unseal Key 3:" | awk -F":" '{print $2}' | xargs)
+	export VAULT_TOKEN=${vault_token}
+	vault operator unseal "${unseal_key1}"
+	vault operator unseal "${unseal_key2}"
+	vault operator unseal "${unseal_key3}"
+	vault secrets enable -version=1 kv
+	vault secrets enable transit
+	vault write -f transit/keys/my-key
+	vault policy write kes-policy kes-policy.hcl
+	vault auth enable approle
+	vault write auth/approle/role/kes-server token_num_uses=0 secret_id_num_uses=0 period=5m
+	vault write auth/approle/role/kes-server policies=kes-policy
+	roleid_output=$(vault read auth/approle/role/kes-server/role-id)
+	role_id=$(echo "$roleid_output" | grep "role_id" | awk -F" " '{print $2}')
+	secretid_output=$(vault write -f auth/approle/role/kes-server/secret-id)
+	secret_id=$(echo "$secretid_output" | grep "secret_id " | awk -F" " '{print $2}')
+	export VAULT_APPROLE_ID="${role_id}"
+	export VAULT_APPROLE_SECRET="${secret_id}"
+	vault_public_cert="${GITHUB_WORKSPACE}"/vault.crt
+	vault_cert="${vault_public_cert}" yq e -i '.keystore.vault.tls.ca = strenv(vault_cert)' "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+
+	echo "=============================================================================="
+	echo "Content of \"${GITHUB_WORKSPACE}\"/kesconf/testdata/vault/kes-config-vault.yml"
+	cat "${GITHUB_WORKSPACE}"/kesconf/testdata/vault/kes-config-vault.yml
+	echo ""
+
+	cat > env.sh <<EOF
+#!/usr/bin/env bash
+
+export KES_API_KEY=kes:v1:AP6gQlUXjWj5iY1WkqeXKIR0OXTpyoiHa81XTY7ISy3l
+export VAULT_APPROLE_ID=${role_id}
+export VAULT_APPROLE_SECRET=${secret_id}
+export VAULT_ADDR=https://127.0.0.1:8200
+export VAULT_SKIP_VERIFY=true
+EOF
+	cat env.sh
+}
+
+main "$@"