Add Vault CI test with deployed Vault with Transit

Signed-off-by: Shubhendu Ram Tripathi <[email protected]>

.github/workflows/ci-test.yml

@@ -0,0 +1,29 @@
+name: Edge tests
+
+on:
+  push:
+    branches:
+    - master
+
+jobs:
+  valut-ci-test:
+    name: Test Vault
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.21.4
+        check-latest: true
+      id: go
+    - name: Check out code
+      uses: actions/checkout@v3
+    - name: Deploy Vault
+      run: |
+        ${GITHUB_WORKSPACE}/kesconf/testdata/vault/deploy_vault.sh
+    - name: Test
+      env:
+        GO111MODULE: on
+        GOPROXY: "https://proxy.golang.org,direct"
+      run: |
+         go test ./kesconf -v -vault.config=${GITHUB_WORKSPACE}/kesconf/testdata/vault/kes-config-vault.yml -run="TestVaultCI"

kesconf/aws_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var awsConfigFile = flag.String("aws.config", "", "Path to a KES config file with AWS SecretsManager config")

kesconf/azure_test.go

@@ -2,14 +2,12 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"os"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var azureConfigFile = flag.String("azure.config", "", "Path to a KES config file with Azure KeyVault config")

kesconf/edge_test.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"bytes"

kesconf/fortanix_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var fortanixConfigFile = flag.String("fortanix.config", "", "Path to a KES config file with Fortanix SDKMS config")

kesconf/fs_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var FSPath = flag.String("fs.path", "", "Path used for FS tests")

kesconf/gcp_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var gcpConfigFile = flag.String("gcp.config", "", "Path to a KES config file with GCP SecretManager config")

kesconf/gemalto_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var gemaltoConfigFile = flag.String("gemalto.config", "", "Path to a KES config file with Gemalto KeySecure config")

kesconf/keycontrol_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var keyControlConfigFile = flag.String("entrust.config", "", "Path to a KES config file with Entrust KeyControl config")

kesconf/testdata/vault/deploy_vault.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+echo "${GITHUB_WORKSPACE=~/kes}"
+
+function main() {
+	# Install HashiCorp vault
+	install_vault
+
+	# Install latest KES binary for cert etc
+	install_kes
+
+	# Setup vault
+	setup_vault
+}
+
+function install_vault() {
+	echo ""
+	echo "Installing HashiCorp vault"
+	echo ""
+	wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+	echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+	sudo apt update -y && sudo apt install vault
+}
+
+function install_kes() {
+	echo ""
+	echo "Installing latest KES binary for certificate etc"
+	echo ""
+	wget -O kes https://github.com/minio/kes/releases/"${VER}"/download/kes-linux-amd64
+	chmod +x kes
+	mv kes /usr/local/bin/kes || sudo mv kes /usr/local/bin/kes
+	kes --version
+}
+
+function setup_vault() {
+	# Create vault certs
+	kes identity new --key vault.key --cert vault.crt --ip "127.0.0.1" localhost
+	mkdir -p /vault/file
+
+	# Start vaule server
+	vault server -config ${GITHUB_WORKSPACE}/kesconf/testdata/vault/vault-config.json &
+
+	# Generate certs for KES
+	kes identity new --ip "127.0.0.1" localhost --cert public.crt --key private.key
+
+	# Generate certs for client application (to be used by test)
+	kes identity new --key=client.key --cert=client.crt MyApp
+
+	client_id=$(kes identity of client.crt | awk '{print $1}')
+	yq e -i '.policy.my-app.identities += ["${client_id}"]' ${GITHUB_WORKSPACE}/kesconf/testdata/vault/kes-config-vault.yml
+
+	export VAULT_ADDR='https://127.0.0.1:8200'
+	export VAULT_SKIP_VERIFY=true
+	init_output=$(vault operator init)
+	vault_token=$(echo $init_output | grep "Initial Root Token:" | awk -F":" '{print $2}' | xargs)
+	unseal_key1=$(echo $init_output | grep "Unseal Key 1:" | awk -F":" '{print $2}' | xargs)
+	unseal_key2=$(echo $init_output | grep "Unseal Key 1:" | awk -F":" '{print $2}' | xargs)
+	unseal_key3=$(echo $init_output | grep "Unseal Key 1:" | awk -F":" '{print $2}' | xargs)
+	export VAULT_TOKEN=${vault_token}
+	vault operator unseal ${unseal_key1}
+	vault operator unseal ${unseal_key2}
+	vault operator unseal ${unseal_key3}
+	vault secrets enable -version=1 kv
+	vault secrets enable transit
+	vault write -f transit/keys/my-key
+	vault policy write kes-policy kes-policy.hcl
+	vault auth enable approle
+	vault write auth/approle/role/kes-server token_num_uses=0  secret_id_num_uses=0  period=5m
+	vault write auth/approle/role/kes-server policies=kes-policy
+	roleid_output=$(vault read auth/approle/role/kes-server/role-id)
+	role_id=$(echo $roleid_output | grep "role_id" | awk -F" " '{print $2}')
+	secretid_output=$(vault write -f auth/approle/role/kes-server/secret-id)
+	secret_id=$(echo $secretid_output | grep "secret_id " | awk -F" " '{print $2}')
+	yq e -i '.keystore.vault.approle.id = "${role_id}"' ${GITHUB_WORKSPACE}/kesconf/testdata/vault/kes-config-vault.yml
+	yq e -i '.keystore.vault.approle.secret = "${secret_id}"' ${GITHUB_WORKSPACE}/kesconf/testdata/vault/kes-config-vault.yml
+}
+
+main "$@"

kesconf/testdata/vault/kes-config-vault.yml

@@ -0,0 +1,40 @@
+address: 0.0.0.0:7373 # Listen on all network interfaces on port 7373
+
+admin:
+  identity: disabled  # We disable the admin identity since we don't need it in this guide 
+
+tls:
+  key: private.key    # The KES server TLS private key
+  cert: public.crt    # The KES server TLS certificate
+
+policy:
+  my-app:
+    allow:
+    - /v1/key/list/*
+    - /v1/key/generate/*
+    - /v1/key/encrypt/*
+    - /v1/key/decrypt/*
+    - /v1/key/create/*
+    - /v1/identity/describe/*
+    - /v1/identity/list/*
+    - /v1/policy/list/*
+    - /v1/metrics
+    identities:
+    - "" # Use the identity of your client.crt
+
+keystore:
+   vault:
+     endpoint: https://127.0.0.1:8200
+     version:  "v1" # The K/V engine version - either "v1" or "v2".
+     transit:
+       engine: "transit"
+       key: "my-key"
+     approle:
+       id:     "" # Your AppRole ID
+       secret: "" # Your AppRole Secret
+       retry:  15s
+     status:
+       ping: 10s
+     tls:
+       ca: vault.crt # Manually trust the vault certificate since we use self-signed certificates
+

kesconf/testdata/vault/kes-policy.hcl

@@ -0,0 +1,9 @@
+path "transit/encrypt/my-key" {
+   capabilities = [ "update" ]
+}
+path "transit/decrypt/my-key" {
+   capabilities = [ "update" ]
+}
+path "kv/*" {
+   capabilities = [ "create", "read", "delete", "list" ]
+}

kesconf/testdata/vault/vault-config.yml

@@ -0,0 +1,21 @@
+{
+  "api_addr": "https://127.0.0.1:8200",
+  "backend": {
+    "file": {
+      "path": "/vault/file"
+    }
+  },
+
+  "default_lease_ttl": "168h",
+  "max_lease_ttl": "720h",
+
+  "listener": {
+    "tcp": {
+      "address": "0.0.0.0:8200",
+      "tls_cert_file": "vault.crt",
+      "tls_key_file": "vault.key",
+      "tls_min_version": "tls12"
+    }
+  }
+}
+

kesconf/vault_ci_test.go

@@ -0,0 +1,39 @@
+// Copyright 2023 - MinIO, Inc. All rights reserved.
+// Use of this source code is governed by the AGPLv3
+// license that can be found in the LICENSE file.
+
+package kesconf
+
+import (
+	"flag"
+	"testing"
+)
+
+var vaultConfigFile = flag.String("vault.config", "", "Path to a KES config file with Hashicorp Vault config")
+
+func TestVaultCI(t *testing.T) {
+	if *vaultConfigFile == "" {
+		t.Skip("Vault tests disabled. Use -vault.config=<FILE> to enable them")
+	}
+
+	config, err := kesconf.ReadFile(*vaultConfigFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, ok := config.KeyStore.(*kesconf.VaultKeyStore); !ok {
+		t.Fatalf("Invalid Keystore: want %T - got %T", config.KeyStore, &kesconf.VaultKeyStore{})
+	}
+
+	ctx, cancel := testingContext(t)
+	defer cancel()
+
+	store, err := config.KeyStore.Connect(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Create", func(t *testing.T) { testCreate(ctx, store, t, RandString(ranStringLength)) })
+	t.Run("Get", func(t *testing.T) { testGet(ctx, store, t, RandString(ranStringLength)) })
+	t.Run("Status", func(t *testing.T) { testStatus(ctx, store, t) })
+}

kesconf/vault_test.go

@@ -2,13 +2,11 @@
 // Use of this source code is governed by the AGPLv3
 // license that can be found in the LICENSE file.
 
-package kesconf_test
+package kesconf
 
 import (
 	"flag"
 	"testing"
-
-	"github.com/minio/kes/kesconf"
 )
 
 var vaultConfigFile = flag.String("vault.config", "", "Path to a KES config file with Hashicorp Vault config")