From 1b99a340b695992679fc89b02495c0cdc976ab86 Mon Sep 17 00:00:00 2001 From: Andreas Auernhammer Date: Wed, 18 Oct 2023 16:20:46 +0200 Subject: [PATCH] refactor KES API and internals **Description:** This commit introduces a series of significant changes to various components within the KES project. Among other things it: 1. Exposes a top-level library API for running and customizing KES servers. 2. Improves logging by using structured logging (`log/slog`). 3. Removes unused code 4. Introduces a KES-specific framework for handling HTTP requests (`internal/api`). 5. Stabilizes the KES API and prepares the introduction of protobuf as serialization format (in addition to JSON). However, this commit does not refactor the `kv` package or the KES config file handling. While still required, this will be done in a separate commit. **Performance** A lot of effort has gone into designing and implementing an efficient KES library API. Since majority of KES operations are read-only, accessing a policy, encrypting a message, a.s.o., it can leverage and benefit from lock-free concurrency primitives. Hence, the `Server` type tries to avoid blocking on `sync.{RW}Mutex` as much as possible and instead uses atomic primitives, like `atomic.Pointer`. Further, the logging framework has been completely reworked to use structured logging using the `log/slog` standard library package. Now, error log messages are only generated when required (based on log levels). The audit logging framework (`AuditHandler` and `AuditRecord` type) works similar to the `slog` package and is also designed to be efficient. **Readability** The new `internal/api` package provides a small KES-specific framework for defining HTTP APIs and handling request. It tries to provide composable primitives to build HTTP APIs that are efficient, secure and easy to reason about. It provides a specific `Request` type that represents an authenticated HTTP request. This allows to separate buisness logic (e.g. handling a key creation request) from timeout handling, authentication, etc. Further, this commit tries to add more expressive documentation describing the intent. **Versioning** The KES library package will follow semantic versioning, like any other Go module. However, the KES server command and CLI (`cmd/kes`) will continue to use the rolling release timestamp versioning. A KES library release can be tagged independently from the KES CLI and vice versa. Users of the KES package will be able to import like any other Go module: `import "github.com/minio/kes@v0.24.0"`. Signed-off-by: Andreas Auernhammer --- api_test.go | 636 ++++++++++++++ audit.go | 188 +++++ auth.go | 173 ++++ auth_test.go | 126 +++ cmd/kes/gateway.go | 675 --------------- cmd/kes/identity.go | 11 +- cmd/kes/key.go | 24 +- cmd/kes/main.go | 20 +- cmd/kes/mlock_linux.go | 2 + cmd/kes/mlock_ref.go | 6 + cmd/kes/policy.go | 12 +- cmd/kes/server.go | 533 +++++++++++- cmd/kes/update.go | 3 +- config.go | 155 ++++ example_test.go | 44 + go.mod | 2 +- internal/api/api.go | 360 ++++---- internal/api/api_test.go | 195 ----- internal/api/error.go | 167 +++- internal/api/health.go | 55 -- internal/api/identity.go | 221 ----- internal/api/key.go | 555 ------------ internal/api/log.go | 93 -- internal/api/metric.go | 49 -- internal/api/multicast.go | 153 ++++ internal/api/policy.go | 204 ----- internal/api/proxy.go | 24 - internal/api/request.go | 28 + internal/api/response.go | 144 ++++ internal/api/router.go | 120 --- internal/api/status.go | 147 ---- internal/api/version.go | 42 - internal/audit/audit.go | 108 --- internal/auth/identity.go | 226 ----- internal/auth/policy.go | 112 --- internal/cache/cow.go | 15 + internal/headers/header.go | 64 ++ internal/headers/header_test.go | 40 + internal/{auth => https}/proxy.go | 39 +- internal/{auth => https}/proxy_test.go | 2 +- internal/https/server.go | 171 ---- internal/keystore/cache.go | 273 ------ internal/log/json_test.go | 73 -- internal/log/log.go | 167 ---- internal/log/writer.go | 148 ---- internal/secret/secret.go | 136 --- internal/sys/build.go | 54 +- kestest/example_test.go | 59 -- kestest/gateway.go | 218 ----- kestest/gateway_aws_test.go | 51 -- kestest/gateway_azure_test.go | 50 -- kestest/gateway_entrust_test.go | 50 -- kestest/gateway_fortanix_test.go | 50 -- kestest/gateway_fs_test.go | 39 - kestest/gateway_gcp_test.go | 51 -- kestest/gateway_gemalto_test.go | 50 -- kestest/gateway_mem_test.go | 30 - kestest/gateway_test.go | 546 ------------ kestest/gateway_vault_test.go | 50 -- kestest/policy.go | 256 ------ keystore.go | 360 ++++++++ log.go | 91 ++ server-config.yaml | 20 - server.go | 1075 ++++++++++++++++++++++++ server_test.go | 169 ++++ state.go | 267 ++++++ 66 files changed, 4678 insertions(+), 5599 deletions(-) create mode 100644 api_test.go create mode 100644 audit.go create mode 100644 auth.go create mode 100644 auth_test.go delete mode 100644 cmd/kes/gateway.go create mode 100644 config.go create mode 100644 example_test.go delete mode 100644 internal/api/api_test.go delete mode 100644 internal/api/health.go delete mode 100644 internal/api/identity.go delete mode 100644 internal/api/key.go delete mode 100644 internal/api/log.go delete mode 100644 internal/api/metric.go create mode 100644 internal/api/multicast.go delete mode 100644 internal/api/policy.go delete mode 100644 internal/api/proxy.go create mode 100644 internal/api/request.go create mode 100644 internal/api/response.go delete mode 100644 internal/api/router.go delete mode 100644 internal/api/status.go delete mode 100644 internal/api/version.go delete mode 100644 internal/audit/audit.go delete mode 100644 internal/auth/identity.go delete mode 100644 internal/auth/policy.go create mode 100644 internal/headers/header.go create mode 100644 internal/headers/header_test.go rename internal/{auth => https}/proxy.go (90%) rename internal/{auth => https}/proxy_test.go (99%) delete mode 100644 internal/https/server.go delete mode 100644 internal/keystore/cache.go delete mode 100644 internal/log/json_test.go delete mode 100644 internal/log/log.go delete mode 100644 internal/log/writer.go delete mode 100644 internal/secret/secret.go delete mode 100644 kestest/example_test.go delete mode 100644 kestest/gateway.go delete mode 100644 kestest/gateway_aws_test.go delete mode 100644 kestest/gateway_azure_test.go delete mode 100644 kestest/gateway_entrust_test.go delete mode 100644 kestest/gateway_fortanix_test.go delete mode 100644 kestest/gateway_fs_test.go delete mode 100644 kestest/gateway_gcp_test.go delete mode 100644 kestest/gateway_gemalto_test.go delete mode 100644 kestest/gateway_mem_test.go delete mode 100644 kestest/gateway_test.go delete mode 100644 kestest/gateway_vault_test.go delete mode 100644 kestest/policy.go create mode 100644 keystore.go create mode 100644 log.go create mode 100644 server.go create mode 100644 server_test.go create mode 100644 state.go diff --git a/api_test.go b/api_test.go new file mode 100644 index 00000000..db5c0085 --- /dev/null +++ b/api_test.go @@ -0,0 +1,636 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package kes + +import ( + "bytes" + "errors" + "net/http" + "runtime" + "slices" + "strconv" + "testing" + "time" + + "aead.dev/mem" + "github.com/minio/kes-go" +) + +func TestImportKey(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + client := defaultClient(url) + + const name = "my-key" + for i, test := range importKeyTests { + name := name + "-" + strconv.Itoa(i) + err := client.ImportKey(ctx, name, &kes.ImportKeyRequest{ + Key: test.Key, + Cipher: test.Cipher, + }) + if err == nil && test.ShouldFail { + t.Errorf("Test %d: setup: creating key '%s' should have failed", i, name) + } + if err != nil && !test.ShouldFail { + t.Errorf("Test %d: setup: failed to create key '%s': %v", i, name, err) + } + } +} + +func TestAPI(t *testing.T) { + t.Parallel() + + t.Run("v1/metrics", testMetrics) + t.Run("v1/api", testListAPIDefaults) + t.Run("v1/status", testStatus) + t.Run("v1/key/create", testCreateKey) + t.Run("v1/key/delete", testDeleteKey) + t.Run("v1/key/import", testImportKey) + t.Run("v1/key/describe", testDescribeKey) + t.Run("v1/key/generate", testGenerateKey) + t.Run("v1/key/encrypt", testEncryptDecryptKey) // also tests decryption + t.Run("v1/key/list", testListKeys) + t.Run("v1/identity/describe", testDescribeIdentity) + t.Run("v1/identity/list", testListIdentites) + t.Run("v1/identity/self/describe", testSelfDescribeIdentity) + t.Run("v1/policy/describe", testDescribePolicy) + t.Run("v1/policy/read", testReadPolicy) + t.Run("v1/policy/list", testListPolicies) +} + +func testMetrics(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + client := defaultClient(url) + metric, err := client.Metrics(ctx) + if err != nil { + t.Fatalf("Failed fetch server metrics: %v", err) + } + if n := metric.RequestOK + metric.RequestErr + metric.RequestFail; n != metric.RequestN() { + t.Fatalf("metrics request count differs: got %d - want %d", n, metric.RequestN()) + } + if metric.CPUs == 0 { + t.Fatalf("metrics contains no number of CPUs") + } + if metric.HeapAlloc == 0 { + t.Fatalf("metrics contains no heap allocations") + } + if metric.HeapObjects == 0 { + t.Fatalf("metrics contains no heap objects") + } + if metric.StackAlloc == 0 { + t.Fatalf("metrics contains no stack allocations") + } +} + +func testStatus(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + client := defaultClient(url) + stat, err := client.Status(ctx) + if err != nil { + t.Fatalf("Failed to fetch status information: %v", err) + } + if stat.Arch != runtime.GOARCH { + t.Fatalf("Invalid status: got '%s' - want '%s'", stat.Arch, runtime.GOARCH) + } + if stat.OS != runtime.GOOS { + t.Fatalf("Invalid status: got '%s' - want '%s'", stat.OS, runtime.GOOS) + } + if stat.StackAlloc == 0 { + t.Fatal("Invalid status: allocated stack memory cannot be 0") + } + if stat.HeapAlloc == 0 { + t.Fatal("Invalid status: allocated heap memory cannot be 0") + } +} + +func testListAPIDefaults(t *testing.T) { + defaults := map[string]struct { + Method string + MaxBody mem.Size + Timeout time.Duration + }{ + "/version": {Method: http.MethodGet, MaxBody: 0, Timeout: 10 * time.Second}, + "/v1/ready": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, + "/v1/status": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, + "/v1/metrics": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, + "/v1/api": {Method: http.MethodGet, MaxBody: 0, Timeout: 10 * time.Second}, + + "/v1/key/create/": {Method: http.MethodPut, MaxBody: 0, Timeout: 15 * time.Second}, + "/v1/key/import/": {Method: http.MethodPut, MaxBody: 1 * mem.MB, Timeout: 15 * time.Second}, + "/v1/key/describe/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, + "/v1/key/list/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, + "/v1/key/delete/": {Method: http.MethodDelete, MaxBody: 0, Timeout: 15 * time.Second}, + "/v1/key/generate/": {Method: http.MethodPut, MaxBody: 1 * mem.MB, Timeout: 15 * time.Second}, + "/v1/key/encrypt/": {Method: http.MethodPut, MaxBody: 1 * mem.MB, Timeout: 15 * time.Second}, + "/v1/key/decrypt/": {Method: http.MethodPut, MaxBody: 1 * mem.MB, Timeout: 15 * time.Second}, + + "/v1/policy/describe/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, + "/v1/policy/read/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, + "/v1/policy/list/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, + + "/v1/identity/describe/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, + "/v1/identity/self/describe": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, + "/v1/identity/list/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, + + "/v1/log/error": {Method: http.MethodGet, MaxBody: 0, Timeout: 0}, + "/v1/log/audit": {Method: http.MethodGet, MaxBody: 0, Timeout: 0}, + } + + t.Parallel() + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + client := defaultClient(url) + routes, err := client.APIs(ctx) + if err != nil { + t.Fatalf("Failed fetch server APIs: %v", err) + } + if len(routes) != len(defaults) { + t.Fatalf("Routes mismatch: got len '%d' - want len '%d'", len(routes), len(defaults)) + } + for i := range routes { + api, ok := defaults[routes[i].Path] + if !ok { + t.Fatalf("Route '%s': not found", routes[i].Path) + } + if routes[i].Method != api.Method { + t.Fatalf("Route '%s': method mismatch: got '%s' - want '%s'", routes[i].Path, routes[i].Method, api.Method) + } + if routes[i].MaxBody != int64(api.MaxBody) { + t.Fatalf("Route '%s': max body mismatch: got '%d' - want '%d'", routes[i].Path, routes[i].MaxBody, api.MaxBody) + } + if routes[i].Timeout != api.Timeout { + t.Fatalf("Route '%s': timeout mismatch: got '%v' - want '%v'", routes[i].Path, routes[i].Timeout, api.Timeout) + } + } +} + +func testCreateKey(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + client := defaultClient(url) + for i, test := range validNameTests { + err := client.CreateKey(ctx, test.Name) + if err == nil && test.ShouldFail { + t.Errorf("Test %d: creating key '%s' should have failed", i, test.Name) + } + if err != nil && !test.ShouldFail { + t.Errorf("Test %d: failed to create key '%s': %v", i, test.Name, err) + } + } +} + +func testDeleteKey(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + client := defaultClient(url) + for i, test := range validNameTests { + err := client.CreateKey(ctx, test.Name) + if err == nil && test.ShouldFail { + t.Errorf("Test %d: setup: creating key '%s' should have failed", i, test.Name) + } + if err != nil && !test.ShouldFail { + t.Errorf("Test %d: setup: failed to create key '%s': %v", i, test.Name, err) + } + + if test.ShouldFail { + continue + } + if err := client.DeleteKey(ctx, test.Name); err != nil { + t.Errorf("Test %d: failed to delete key '%s': %v", i, test.Name, err) + } + } +} + +func testImportKey(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + client := defaultClient(url) + for i, test := range validNameTests { + err := client.ImportKey(ctx, test.Name, &kes.ImportKeyRequest{ + Key: make([]byte, 32), + Cipher: kes.AES256, + }) + if err == nil && test.ShouldFail { + t.Errorf("Test %d: setup: creating key '%s' should have failed", i, test.Name) + } + if err != nil && !test.ShouldFail { + t.Errorf("Test %d: setup: failed to create key '%s': %v", i, test.Name, err) + } + } +} + +func testDescribeKey(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + client := defaultClient(url) + for i, test := range validNameTests { + err := client.CreateKey(ctx, test.Name) + if err == nil && test.ShouldFail { + t.Errorf("Test %d: setup: creating key '%s' should have failed", i, test.Name) + } + if err != nil && !test.ShouldFail { + t.Errorf("Test %d: setup: failed to create key '%s': %v", i, test.Name, err) + } + + if test.ShouldFail { + continue + } + + info, err := client.DescribeKey(ctx, test.Name) + if err != nil { + t.Errorf("Test %d: failed to describe key '%s': %v", i, test.Name, err) + } + if info.Algorithm > kes.ChaCha20 { + t.Errorf("Test %d: failed to describe key '%s': invalid algorithm '%d'", i, test.Name, info.Algorithm) + } + if info.CreatedAt.IsZero() { + t.Errorf("Test %d: failed to describe key '%s': created_at is zero", i, test.Name) + } + if info.CreatedBy.IsUnknown() { + t.Errorf("Test %d: failed to describe key '%s': created_by is empty", i, test.Name) + } + } +} + +func testGenerateKey(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + associatedData := make([]byte, 80) + + client := defaultClient(url) + for i, test := range validNameTests { + err := client.CreateKey(ctx, test.Name) + if err == nil && test.ShouldFail { + t.Errorf("Test %d: setup: creating key '%s' should have failed", i, test.Name) + } + if err != nil && !test.ShouldFail { + t.Errorf("Test %d: setup: failed to create key '%s': %v", i, test.Name, err) + } + + if test.ShouldFail { + continue + } + + dek, err := client.GenerateKey(ctx, test.Name, associatedData) + if err != nil { + t.Errorf("Test %d: failed to generate DEK with key '%s': %v", i, test.Name, err) + } + plaintext, err := client.Decrypt(ctx, test.Name, dek.Ciphertext, associatedData) + if err != nil { + t.Errorf("Test %d: failed to decrypt DEK with key '%s': %v", i, test.Name, err) + } + if !bytes.Equal(plaintext, dek.Plaintext) { + t.Errorf("Test %d: plaintext mismatch: got %v - want %v", i, plaintext, dek.Plaintext) + } + + dek2, err := client.GenerateKey(ctx, test.Name, associatedData) + if err != nil { + t.Errorf("Test %d: failed to generate DEK with key '%s': %v", i, test.Name, err) + } + if bytes.Equal(dek.Plaintext, dek2.Plaintext) { + t.Errorf("Test %d: generate key is deterministic and produces the same DEKs", i) + } + if bytes.Equal(dek.Ciphertext, dek2.Ciphertext) { + t.Errorf("Test %d: generate key is deterministic and produces the same DEK ciphertexts", i) + } + } +} + +func testEncryptDecryptKey(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + plaintext := make([]byte, mem.KB) + associatedData := make([]byte, 80) + + client := defaultClient(url) + for i, test := range validNameTests { + err := client.CreateKey(ctx, test.Name) + if err == nil && test.ShouldFail { + t.Errorf("Test %d: setup: creating key '%s' should have failed", i, test.Name) + } + if err != nil && !test.ShouldFail { + t.Errorf("Test %d: setup: failed to create key '%s': %v", i, test.Name, err) + } + + if test.ShouldFail { + continue + } + + ciphertext, err := client.Encrypt(ctx, test.Name, plaintext, associatedData) + if err != nil { + t.Errorf("Test %d: failed to encrypt with key '%s': %v", i, test.Name, err) + } + ptext, err := client.Decrypt(ctx, test.Name, ciphertext, associatedData) + if err != nil { + t.Errorf("Test %d: failed to decrypt with key '%s': %v", i, test.Name, err) + } + if !bytes.Equal(ptext, plaintext) { + t.Errorf("Test %d: plaintext mismatch: got %v - want %v", i, ptext, plaintext) + } + + ctext, err := client.Encrypt(ctx, test.Name, plaintext, associatedData) + if err != nil { + t.Errorf("Test %d: failed to encrypt with key '%s': %v", i, test.Name, err) + } + if bytes.Equal(ctext, ciphertext) { + t.Errorf("Test %d: encryption is deterministic and produces the same ciphertexts", i) + } + } +} + +func testListKeys(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + var names []string + client := defaultClient(url) + for i, test := range validNameTests { + err := client.CreateKey(ctx, test.Name) + if err == nil && test.ShouldFail { + t.Errorf("Test %d: setup: creating key '%s' should have failed", i, test.Name) + } + if err != nil && !test.ShouldFail { + t.Errorf("Test %d: setup: failed to create key '%s': %v", i, test.Name, err) + } + if !test.ShouldFail { + names = append(names, test.Name) + } + } + slices.Sort(names) + + keys, _, err := client.ListKeys(ctx, "", -1) + if err != nil { + t.Fatalf("Failed to list keys: %v", err) + } + if !slices.Equal(names, keys) { + t.Fatalf("Failed to list keys: got %v - want %v", keys, names) + } +} + +func testDescribeIdentity(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + var identities []kes.Identity + for _, test := range validNameTests { + if validName(test.Name) { + identities = append(identities, kes.Identity(test.Name)) + } + } + if err := srv.UpdatePolicies(map[string]Policy{"policy": {Identities: identities}}); err != nil { + t.Fatalf("Failed to update server policies: %v", err) + } + + client := defaultClient(url) + for i, id := range identities { + info, err := client.DescribeIdentity(ctx, id) + if err != nil { + t.Fatalf("Test %d: failed to describe identity '%s': %v", i, id, err) + } + if info.IsAdmin { + t.Errorf("Test %d: identity '%s' is admin", i, id) + } + if !info.ExpiresAt.IsZero() || info.TTL > 0 { + t.Errorf("Test %d: identity '%s' expires", i, id) + } + if info.CreatedBy != defaultIdentity { + t.Errorf("Test %d: identity '%s' was not created by '%s'", i, id, defaultIdentity) + } + + } +} + +func testListIdentites(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + names := []kes.Identity{ + "8ed87d812abbf280ffa760080873d0d503fdfa9c41c1bf32b4cffd1dc71b1d1c", + "34d90ce76fbc40ab8354ea8c42c17b3f20e1f63a10f6cef787a94394b02141c4", + "cd0dd4c3efab6a5744d1e9b1754dbe7f612bd759062d6f17a8ef25f47fc86c54", + "59ba6afc8e844ba36edcf8ed50c23cb62626ea420e9b9ca7509ab6fa6d13ad3a", + "disabled", + } + if err := srv.UpdatePolicies(map[string]Policy{"policy": {Identities: names}}); err != nil { + t.Fatalf("Failed to update server policies: %v", err) + } + names = append(names, defaultIdentity) // Listing identities always includes the admin identity + slices.Sort(names) + + client := defaultClient(url) + identities, _, err := client.ListIdentities(ctx, "", -1) + if err != nil { + t.Fatalf("Failed to list identities: %v", err) + } + if !slices.Equal(names, identities) { + t.Fatalf("Failed to list identities: got %v - want %v", identities, names) + } +} + +func testSelfDescribeIdentity(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + client := defaultClient(url) + info, _, err := client.DescribeSelf(ctx) + if err != nil { + t.Fatalf("Failed to self-describe identity: %v", err) + } + if !info.IsAdmin { + t.Error("Failed to self-describe identity: not the admin") + } + if info.Identity.String() != defaultIdentity { + t.Errorf("Failed to self-describe identity: got '%s' - want '%s'", info.Identity.String(), defaultIdentity) + } +} + +func testDescribePolicy(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + policies := make(map[string]Policy) + for _, test := range validNameTests { + if validName(test.Name) { + policies[test.Name] = Policy{} + } + } + if err := srv.UpdatePolicies(policies); err != nil { + t.Fatalf("Failed to update server policies: %v", err) + } + + client := defaultClient(url) + for i, test := range validNameTests { + info, err := client.DescribePolicy(ctx, test.Name) + if err == nil && test.ShouldFail { + t.Errorf("Test %d: describing policy '%s' should have failed", i, test.Name) + } + if err != nil && !test.ShouldFail { + t.Errorf("Test %d: failed to describe policy '%s': %v", i, test.Name, err) + } + if !validName(test.Name) && errors.Is(err, kes.ErrPolicyNotFound) { + t.Errorf("Test %d: received %v for invalid policy name '%s'", i, err, test.Name) + } + + if test.ShouldFail { + continue + } + + if info.Name != test.Name { + t.Errorf("Test %d: invalid name: got '%s' - want '%s'", i, info.Name, test.Name) + } + } +} + +func testReadPolicy(t *testing.T) { + t.Parallel() + + policy := kes.Policy{ + Allow: map[string]kes.Rule{ + "/v1/status": {}, + "/v1/ready": {}, + "/v1/key/create/*": {}, + "/v1/key/generate/*": {}, + "/v1/key/decrypt/*": {}, + }, + Deny: map[string]kes.Rule{"/v1/key/decrypt/internal*": {}}, + } + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + policies := make(map[string]Policy) + for _, test := range validNameTests { + if validName(test.Name) { + policies[test.Name] = Policy{ + Allow: policy.Allow, + Deny: policy.Deny, + } + } + } + if err := srv.UpdatePolicies(policies); err != nil { + t.Fatalf("Failed to update server policies: %v", err) + } + + client := defaultClient(url) + for i, test := range validNameTests { + p, err := client.GetPolicy(ctx, test.Name) + if err == nil && test.ShouldFail { + t.Errorf("Test %d: reading policy '%s' should have failed", i, test.Name) + } + if err != nil && !test.ShouldFail { + t.Errorf("Test %d: failed to read policy '%s': %v", i, test.Name, err) + } + if !validName(test.Name) && errors.Is(err, kes.ErrPolicyNotFound) { + t.Errorf("Test %d: received %v for invalid policy name '%s'", i, err, test.Name) + } + + if test.ShouldFail { + continue + } + + if !p.IsSubset(&policy) || !policy.IsSubset(p) { + t.Errorf("Test %d: policy mismatch: got '%v' - want '%v'", i, p, policy) + } + } +} + +func testListPolicies(t *testing.T) { + t.Parallel() + + ctx := testContext(t) + srv, url := startServer(ctx, nil) + defer srv.Close() + + var names []string + policies := make(map[string]Policy) + for _, test := range validNameTests { + if validName(test.Name) { + policies[test.Name] = Policy{} + names = append(names, test.Name) + } + } + if err := srv.UpdatePolicies(policies); err != nil { + t.Fatalf("Failed to update server policies: %v", err) + } + slices.Sort(names) + + client := defaultClient(url) + list, _, err := client.ListPolicies(ctx, "", -1) + if err != nil { + t.Fatalf("Failed to list policies: %v", err) + } + if !slices.Equal(names, list) { + t.Fatalf("Failed to list policies: got %v - want %v", list, names) + } +} + +var importKeyTests = []struct { + Key []byte + Cipher kes.KeyAlgorithm + ShouldFail bool +}{ + {Key: make([]byte, 32), Cipher: kes.AES256}, // 0 + {Key: make([]byte, 32), Cipher: kes.ChaCha20}, // 1 + + {Key: make([]byte, 16), Cipher: kes.AES256, ShouldFail: true}, // 2 + {Key: make([]byte, 24), Cipher: kes.ChaCha20, ShouldFail: true}, // 3 + {Key: make([]byte, 32), Cipher: kes.ChaCha20 + 1, ShouldFail: true}, // 4 +} diff --git a/audit.go b/audit.go new file mode 100644 index 00000000..bc50a7bd --- /dev/null +++ b/audit.go @@ -0,0 +1,188 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package kes + +import ( + "context" + "encoding/json" + "log/slog" + "net/netip" + "time" + + "github.com/minio/kes-go" + "github.com/minio/kes/internal/api" +) + +// AuditRecord describes an audit event logged by a KES server. +type AuditRecord struct { + // Point in time when the audit event happened. + Time time.Time + + // The request HTTP method. (GET, PUT, ...) + Method string + + // Request URL path. Always starts with a '/'. + Path string + + // Identity that send the request. + Identity kes.Identity + + // IP address of the client that sent the request. + RemoteIP netip.Addr + + // Status code the KES server responded with. + StatusCode int + + // Amount of time the server took to process the + // request and generate a response. + ResponseTime time.Duration + + // The log level of this event. + Level slog.Level + + // The log message describing the event. + Message string +} + +// An AuditHandler handles audit records produced by a Server. +// +// A typical handler may print audit records to standard error, +// or write them to a file or database. +// +// Any of the AuditHandler's methods may be called concurrently +// with itself or with other methods. It is the responsibility +// of the Handler to manage this concurrency. +type AuditHandler interface { + // Enabled reports whether the handler handles records at + // the given level. The handler ignores records whose level + // is lower. It is called early, before an audit record is + // created, to safe effort if the audit event should be + // discarded. + // + // The Server will pass the request context as the first + // argument, or context.Background() if no context is + // available. Enabled may use the context to make a + // decision. + Enabled(context.Context, slog.Level) bool + + // Handle handles the AuditRecord. It will only called when + // Enabled returns true. + // + // The context is present for providing AuditHandlers access + // to the context's values and to potentially pass it to an + // underlying slog.Handler. Canceling the context should not + // affect record processing. + Handle(context.Context, AuditRecord) error +} + +// AuditLogHandler is an AuditHandler adapter that wraps +// an slog.Handler. It converts AuditRecords to slog.Records +// and passes them to the slog.Handler. An AuditLogHandler +// acts as a bridge between AuditHandlers and slog.Handlers. +type AuditLogHandler struct { + Handler slog.Handler +} + +// Enabled reports whether the AuditLogHandler handles records +// at the given level. It returns true if the underlying handler +// returns true. +func (a *AuditLogHandler) Enabled(ctx context.Context, level slog.Level) bool { + return a.Handler.Enabled(ctx, level) +} + +// Handle converts the AuditRecord to an slog.Record and +// passes it to the underlying handler. +func (a *AuditLogHandler) Handle(ctx context.Context, r AuditRecord) error { + rec := slog.Record{ + Time: r.Time, + Message: r.Message, + Level: r.Level, + } + rec.AddAttrs( + slog.Attr{Key: "req", Value: slog.GroupValue( + slog.String("method", r.Method), + slog.String("path", r.Path), + slog.String("ip", r.RemoteIP.String()), + slog.String("identity", r.Identity.String()), + )}, + slog.Attr{Key: "res", Value: slog.GroupValue( + slog.Int("code", r.StatusCode), + slog.Duration("time", r.ResponseTime), + )}, + ) + return a.Handler.Handle(ctx, rec) +} + +// An auditLogger records information about a request/response +// handled by the Server. +// +// For each call of its Log method, it creates an AuditRecord and +// passes it to its AuditHandler. If clients have subscribed to +// the AuditLog API, the logger also sends the AuditRecord to these +// clients. +type auditLogger struct { + h AuditHandler + level slog.Leveler + + out *api.Multicast // clients subscribed to the AuditLog API +} + +// newAuditLogger returns a new auditLogger passing AuditRecords to h. +// A record is only sent to clients subscribed to the AuditLog API if +// its log level is >= level. +func newAuditLogger(h AuditHandler, level slog.Leveler) *auditLogger { + return &auditLogger{ + h: h, + level: level, + out: &api.Multicast{}, + } +} + +// Log emits an audit record with the current time, log message, +// response status code and request information. +func (a *auditLogger) Log(msg string, statusCode int, req *api.Request) { + const Level = slog.LevelInfo + if Level < a.level.Level() { + return + } + + hEnabled, oEnabled := a.h.Enabled(req.Context(), Level), a.out.Num() > 0 + if !hEnabled && !oEnabled { + return + } + + now := time.Now() + remoteIP, _ := netip.ParseAddrPort(req.RemoteAddr) + r := AuditRecord{ + Time: time.Now(), + Method: req.Method, + Path: req.URL.Path, + Identity: req.Identity, + RemoteIP: remoteIP.Addr(), + StatusCode: statusCode, + ResponseTime: now.Sub(req.Received), + Level: Level, + Message: msg, + } + if hEnabled { + a.h.Handle(req.Context(), r) + } + + if !oEnabled { + return + } + json.NewEncoder(a.out).Encode(api.AuditLogEvent{ + Time: r.Time, + Request: api.AuditLogRequest{ + IP: r.RemoteIP.String(), + APIPath: r.Path, + Identity: r.Identity.String(), + }, + Response: api.AuditLogResponse{ + StatusCode: r.StatusCode, + Time: r.ResponseTime.Milliseconds(), + }, + }) +} diff --git a/auth.go b/auth.go new file mode 100644 index 00000000..56f8e950 --- /dev/null +++ b/auth.go @@ -0,0 +1,173 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package kes + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "fmt" + "net/http" + "sync/atomic" + + "github.com/minio/kes-go" + "github.com/minio/kes/internal/api" +) + +// verifyIdentity authenticates client requests by verifying that +// the client provides a certificate during the TLS handshake (mTLS) +// and that the identity of the certificate public key matches either +// the admin identity or an identity with an assigned policy. +// +// A request is accepted if the identity matches the admin identity +// or the policy associated to the identity allows the request. The +// later is the case if none of the policy's deny rules and at least +// one of the policy's allow rules apply. Otherwise, the request is +// rejected. +type verifyIdentity atomic.Pointer[serverState] + +// Authenticate verifies that the request is either sent by the +// server admin or passes the policy assigned to the identity. +// Otherwise, it returns an error. +func (v *verifyIdentity) Authenticate(req *http.Request) (*api.Request, api.Error) { + identity, err := identifyRequest(req.TLS) + if err != nil { + s := (*atomic.Pointer[serverState])(v).Load() + s.Log.DebugContext(req.Context(), err.Error(), "req", req) + return nil, err + } + + s := (*atomic.Pointer[serverState])(v).Load() + if identity == s.Admin { + return &api.Request{ + Request: req, + Identity: identity, + }, nil + } + + policy, ok := s.Identities[identity] + if !ok { + s.Log.DebugContext(req.Context(), "access denied: identity not found", "req", req) + return nil, kes.ErrNotAllowed + } + if err := policy.Verify(req); err != nil { + s.Log.DebugContext(req.Context(), fmt.Sprintf("access denied: rejected by policy '%s'", policy.Name), "req", req) + return nil, kes.ErrNotAllowed + } + + return &api.Request{ + Request: req, + Identity: identity, + }, nil +} + +// insecureIdentifyOnly does not authenticate client requests but +// computes the certificate public key identity, if provided. +// It does not return an error if the client did not provide a +// certificate, or an invalid one, during the TLS handshake. In +// such a case, the identity of the returned request is empty. +type insecureIdentifyOnly struct{} + +func (insecureIdentifyOnly) Authenticate(req *http.Request) (*api.Request, api.Error) { + identity, _ := identifyRequest(req.TLS) + return &api.Request{ + Request: req, + Identity: identity, + }, nil +} + +func identifyRequest(state *tls.ConnectionState) (kes.Identity, api.Error) { + if state == nil { + return "", api.NewError(http.StatusBadRequest, "insecure connection: TLS is required") + } + + var cert *x509.Certificate + for _, c := range state.PeerCertificates { + if c.IsCA { + continue + } + if cert != nil { + return "", api.NewError(http.StatusBadRequest, "tls: received more than one client certificate") + } + cert = c + } + if cert == nil { + return "", api.NewError(http.StatusBadRequest, "tls: client certificate is required") + } + + h := sha256.Sum256(cert.RawSubjectPublicKeyInfo) + return kes.Identity(hex.EncodeToString(h[:])), nil +} + +// validName reports whether s is a valid {policy|identity|key} name. +// +// Valid names only contain the characters: +// - 0-9 +// - A-Z +// - a-z +// - '-' (hyphen, must not be first/last character) +// - '_' (underscore, must not be the only character) +// +// More characters may be allowed in the future. +func validName(s string) bool { + const MaxLength = 80 // Some arbitrary but reasonable limit + + if s == "" || s == "_" || len(s) > MaxLength { + return false + } + + n := len(s) - 1 + for i, r := range s { + switch { + case r >= '0' && r <= '9': + case r >= 'A' && r <= 'Z': + case r >= 'a' && r <= 'z': + case r == '-' && i > 0 && i < n: + case r == '_': + default: + return false + } + } + return true +} + +// validPattern reports whether s is a valid pattern for +// listing {policy|identity|key} names. +// +// Valid patterns only contain the characters: +// - 0-9 +// - A-Z +// - a-z +// - '-' (hyphen, must not be first/last character) +// - '_' (underscore, must not be the only character) +// - '*' (only as last character) +// +// More characters may be allowed in the future. +func validPattern(s string) bool { + const MaxLength = 80 // Some arbitrary but reasonable limit + + if s == "*" { // fast path + return true + } + if s == "_" || len(s) > MaxLength { + return false + } + + n := len(s) - 1 + for i, r := range s { + switch { + case r >= '0' && r <= '9': + case r >= 'A' && r <= 'Z': + case r >= 'a' && r <= 'z': + case r == '-' && i > 0 && i < n: + case r == '_': + case r == '*' && i == n: + default: + return false + } + } + return true +} diff --git a/auth_test.go b/auth_test.go new file mode 100644 index 00000000..da55890f --- /dev/null +++ b/auth_test.go @@ -0,0 +1,126 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package kes + +import ( + "strings" + "testing" +) + +func TestValidName(t *testing.T) { + t.Parallel() + for i, test := range validNameTests { + if valid := validName(test.Name); valid != !test.ShouldFail { + t.Errorf("Test %d: got 'valid=%v' - want 'fail=%v' for name '%s'", i, valid, test.ShouldFail, test.Name) + } + } +} + +func TestValidPattern(t *testing.T) { + t.Parallel() + for i, test := range validPatternTests { + if valid := validPattern(test.Pattern); valid != !test.ShouldFail { + t.Errorf("Test %d: got 'valid=%v' - want 'fail=%v' for pattern '%s'", i, valid, test.ShouldFail, test.Pattern) + } + } +} + +func BenchmarkValidName(b *testing.B) { + const ( + EmptyName = "" + ValidName = "my-minio-key" + InvalidName = "my-minio-key*" + ) + + b.Run("empty", func(b *testing.B) { + for i := 0; i < b.N; i++ { + validName(EmptyName) + } + }) + b.Run("valid", func(b *testing.B) { + for i := 0; i < b.N; i++ { + validName(ValidName) + } + }) + b.Run("invalid", func(b *testing.B) { + for i := 0; i < b.N; i++ { + validName(InvalidName) + } + }) +} + +func BenchmarkValidPattern(b *testing.B) { + const ( + MatchAll = "*" + ValidPattern = "my-minio-key*" + InvalidPattern = "my-minio-key/" + ) + + b.Run("matchall", func(b *testing.B) { + for i := 0; i < b.N; i++ { + validPattern(MatchAll) + } + }) + b.Run("valid", func(b *testing.B) { + for i := 0; i < b.N; i++ { + validPattern(ValidPattern) + } + }) + b.Run("invalid", func(b *testing.B) { + for i := 0; i < b.N; i++ { + validPattern(InvalidPattern) + } + }) +} + +var ( + validNameTests = []struct { + Name string + ShouldFail bool + }{ + {Name: "my-key"}, // 0 + {Name: "abc123"}, // 1 + {Name: "0"}, // 2 + {Name: "123ABC321"}, // 3 + {Name: "_-___---_"}, // 4 + {Name: "_0"}, // 5 + {Name: "0-Z"}, // 6 + {Name: "my_key-0"}, // 7 + + {Name: "", ShouldFail: true}, // 8 + {Name: "my.key", ShouldFail: true}, // 9 + {Name: "key/", ShouldFail: true}, // 10 + {Name: "", ShouldFail: true}, // 11 + {Name: "☰", ShouldFail: true}, // 12 + {Name: "hel 0 - if next { - i.current = i.values[0] - i.values = i.values[1:] - } - return next -} - -func (i *policyIterator) Name() string { return i.current } - -func (i *policyIterator) Close() error { return nil } - -// identitySetFromConfig returns an in-memory IdentitySet -// from the given ServerConfig. -func identitySetFromConfig(config *edge.ServerConfig) (auth.IdentitySet, error) { - identities := &identitySet{ - admin: config.Admin, - createdAt: time.Now().UTC(), - roles: map[kes.Identity]auth.IdentityInfo{}, - } - - for name, policy := range config.Policies { - for _, id := range policy.Identities { - if id.IsUnknown() { - continue - } - - if id == config.Admin { - return nil, fmt.Errorf("identity %q is already an admin identity", id) - } - if _, ok := identities.roles[id]; ok { - return nil, fmt.Errorf("identity %q is already assigned", id) - } - for _, proxyID := range config.TLS.Proxies { - if id == proxyID { - return nil, fmt.Errorf("identity %q is already a TLS proxy identity", id) - } - } - identities.roles[id] = auth.IdentityInfo{ - Policy: name, - CreatedAt: time.Now().UTC(), - CreatedBy: config.Admin, - } - } - } - return identities, nil -} - -type identitySet struct { - admin kes.Identity - createdAt time.Time - - lock sync.RWMutex - roles map[kes.Identity]auth.IdentityInfo -} - -var _ auth.IdentitySet = (*identitySet)(nil) // compiler check - -func (i *identitySet) Admin(context.Context) (kes.Identity, error) { return i.admin, nil } - -func (i *identitySet) SetAdmin(context.Context, kes.Identity) error { - return kes.NewError(http.StatusNotImplemented, "cannot set admin identity") -} - -func (i *identitySet) Assign(_ context.Context, policy string, identity kes.Identity) error { - if i.admin == identity { - return kes.NewError(http.StatusBadRequest, "identity is root") - } - i.lock.Lock() - defer i.lock.Unlock() - - i.roles[identity] = auth.IdentityInfo{ - Policy: policy, - CreatedAt: time.Now().UTC(), - CreatedBy: i.admin, - } - return nil -} - -func (i *identitySet) Get(_ context.Context, identity kes.Identity) (auth.IdentityInfo, error) { - if identity == i.admin { - return auth.IdentityInfo{ - IsAdmin: true, - CreatedAt: i.createdAt, - }, nil - } - i.lock.RLock() - defer i.lock.RUnlock() - - policy, ok := i.roles[identity] - if !ok { - return auth.IdentityInfo{}, kes.ErrIdentityNotFound - } - return policy, nil -} - -func (i *identitySet) Delete(_ context.Context, identity kes.Identity) error { - i.lock.Lock() - defer i.lock.Unlock() - - delete(i.roles, identity) - return nil -} - -func (i *identitySet) List(_ context.Context) (auth.IdentityIterator, error) { - i.lock.RLock() - defer i.lock.RUnlock() - - values := make([]kes.Identity, 0, len(i.roles)) - for identity := range i.roles { - values = append(values, identity) - } - return &identityIterator{ - values: values, - }, nil -} - -type identityIterator struct { - values []kes.Identity - current kes.Identity -} - -var _ auth.IdentityIterator = (*identityIterator)(nil) // compiler check - -func (i *identityIterator) Next() bool { - next := len(i.values) > 0 - if next { - i.current = i.values[0] - i.values = i.values[1:] - } - return next -} - -func (i *identityIterator) Identity() kes.Identity { return i.current } - -func (i *identityIterator) Close() error { return nil } - -func loadGatewayConfig(gConfig gatewayConfig) (*edge.ServerConfig, error) { - file, err := os.Open(gConfig.ConfigFile) - if err != nil { - return nil, err - } - defer file.Close() - - config, err := edge.ReadServerConfigYAML(file) - if err != nil { - return nil, fmt.Errorf("failed to read config file: %v", err) - } - if gConfig.Address != "" { - config.Addr = gConfig.Address - } - if gConfig.PrivateKey != "" { - config.TLS.PrivateKey = gConfig.PrivateKey - } - if gConfig.Certificate != "" { - config.TLS.Certificate = gConfig.Certificate - } - - // Set config defaults - if config.Addr == "" { - config.Addr = "0.0.0.0:7373" - } - if config.Cache.Expiry == 0 { - config.Cache.Expiry = 5 * time.Minute - } - if config.Cache.ExpiryUnused == 0 { - config.Cache.ExpiryUnused = 30 * time.Second - } - - // Verify config - if config.Admin.IsUnknown() { - return nil, errors.New("no admin identity specified") - } - if config.TLS.PrivateKey == "" { - return nil, errors.New("no TLS private key specified") - } - if config.TLS.Certificate == "" { - return nil, errors.New("no TLS certificate specified") - } - return config, nil -} - -func newTLSConfig(config *edge.ServerConfig, auth string) (*tls.Config, error) { - certificate, err := https.CertificateFromFile(config.TLS.Certificate, config.TLS.PrivateKey, config.TLS.Password) - if err != nil { - return nil, fmt.Errorf("failed to read TLS certificate: %v", err) - } - if certificate.Leaf != nil { - if len(certificate.Leaf.DNSNames) == 0 && len(certificate.Leaf.IPAddresses) == 0 { - // Support for TLS certificates with a subject CN but without any SAN - // has been removed in Go 1.15. Ref: https://go.dev/doc/go1.15#commonname - // Therefore, we require at least one SAN for the server certificate. - return nil, fmt.Errorf("invalid TLS certificate: certificate does not contain any DNS or IP address as SAN") - } - } - - var rootCAs *x509.CertPool - if config.TLS.CAPath != "" { - rootCAs, err = https.CertPoolFromFile(config.TLS.CAPath) - if err != nil { - return nil, fmt.Errorf("failed to read TLS CA certificates: %v", err) - } - } - var clientAuth tls.ClientAuthType - switch strings.ToLower(auth) { - case "", "on": - clientAuth = tls.RequireAndVerifyClientCert - if config.API != nil { - for _, api := range config.API.Paths { - if api.InsecureSkipAuth { - clientAuth = tls.VerifyClientCertIfGiven - break - } - } - } - case "off": - clientAuth = tls.RequireAnyClientCert - if config.API != nil { - for _, api := range config.API.Paths { - if api.InsecureSkipAuth { - clientAuth = tls.RequestClientCert - break - } - } - } - default: - return nil, fmt.Errorf("invalid option for --auth: %s", auth) - } - - return &tls.Config{ - Certificates: []tls.Certificate{certificate}, - ClientAuth: clientAuth, - RootCAs: rootCAs, - ClientCAs: rootCAs, - - MinVersion: tls.VersionTLS12, - CipherSuites: fips.TLSCiphers(), - CurvePreferences: fips.TLSCurveIDs(), - }, nil -} - -func newGatewayConfig(ctx context.Context, config *edge.ServerConfig, tlsConfig *tls.Config) (*api.EdgeRouterConfig, error) { - rConfig := &api.EdgeRouterConfig{} - - if config.Log.Error { - rConfig.ErrorLog = log.New(os.Stderr, "Error: ", log.Ldate|log.Ltime|log.Lmsgprefix) - } else { - rConfig.ErrorLog = log.New(io.Discard, "Error: ", log.Ldate|log.Ltime|log.Lmsgprefix) - } - if config.Log.Audit { - rConfig.AuditLog = log.New(os.Stdout, "", 0) - } else { - rConfig.AuditLog = log.New(io.Discard, "", 0) - } - - if len(config.TLS.Proxies) != 0 { - rConfig.Proxy = &auth.TLSProxy{ - CertHeader: http.CanonicalHeaderKey(config.TLS.ForwardCertHeader), - } - if tlsConfig.ClientAuth == tls.RequireAndVerifyClientCert { - rConfig.Proxy.VerifyOptions = &x509.VerifyOptions{ - Roots: tlsConfig.RootCAs, - } - } - for _, identity := range config.TLS.Proxies { - if !identity.IsUnknown() { - rConfig.Proxy.Add(identity) - } - } - } - - if config.API != nil && len(config.API.Paths) > 0 { - rConfig.APIConfig = make(map[string]api.Config, len(config.API.Paths)) - for k, v := range config.API.Paths { - k = strings.TrimSpace(k) // Ensure that the API path starts with a '/' - if !strings.HasPrefix(k, "/") { - k = "/" + k - } - - if _, ok := rConfig.APIConfig[k]; ok { - return nil, fmt.Errorf("ambiguous API configuration for '%s'", k) - } - rConfig.APIConfig[k] = api.Config{ - Timeout: v.Timeout, - InsecureSkipAuth: v.InsecureSkipAuth, - } - } - } - - var err error - rConfig.Policies, err = policySetFromConfig(config) - if err != nil { - return nil, err - } - rConfig.Identities, err = identitySetFromConfig(config) - if err != nil { - return nil, err - } - - conn, err := config.KeyStore.Connect(ctx) - if err != nil { - return nil, err - } - rConfig.Keys = keystore.NewCache(ctx, conn, &keystore.CacheConfig{ - Expiry: config.Cache.Expiry, - ExpiryUnused: config.Cache.ExpiryUnused, - ExpiryOffline: config.Cache.ExpiryOffline, - }) - - for _, k := range config.Keys { - var algorithm kes.KeyAlgorithm - if fips.Enabled || cpu.HasAESGCM() { - algorithm = kes.AES256 - } else { - algorithm = kes.ChaCha20 - } - - key, err := key.Random(algorithm, config.Admin) - if err != nil { - return nil, fmt.Errorf("failed to create key '%s': %v", k.Name, err) - } - if err = rConfig.Keys.Create(ctx, k.Name, key); err != nil && !errors.Is(err, kes.ErrKeyExists) { - return nil, fmt.Errorf("failed to create key '%s': %v", k.Name, err) - } - } - - rConfig.Metrics = metric.New() - rConfig.AuditLog.Add(rConfig.Metrics.AuditEventCounter()) - rConfig.ErrorLog.Add(rConfig.Metrics.ErrorEventCounter()) - return rConfig, nil -} - -func gatewayMessage(config *edge.ServerConfig, tlsConfig *tls.Config, mlock bool) (*cli.Buffer, error) { - ip, port := serverAddr(config.Addr) - ifaceIPs := listeningOnV4(ip) - if len(ifaceIPs) == 0 { - return nil, errors.New("failed to listen on network interfaces") - } - kmsKind, kmsEndpoints, err := description(config) - if err != nil { - return nil, err - } - - var faint, item, green, red tui.Style - if isTerm(os.Stdout) { - faint = faint.Faint(true) - item = item.Foreground(tui.Color("#2e42d1")).Bold(true) - green = green.Foreground(tui.Color("#00a700")) - red = red.Foreground(tui.Color("#a70000")) - } - - buffer := new(cli.Buffer) - buffer.Stylef(item, "%-12s", "Copyright").Sprintf("%-22s", "MinIO, Inc.").Styleln(faint, "https://min.io") - buffer.Stylef(item, "%-12s", "License").Sprintf("%-22s", "GNU AGPLv3").Styleln(faint, "https://www.gnu.org/licenses/agpl-3.0.html") - buffer.Stylef(item, "%-12s", "Version").Sprintf("%-22s", sys.BinaryInfo().Version).Stylef(faint, "%s/%s\n", runtime.GOOS, runtime.GOARCH) - buffer.Sprintln() - buffer.Stylef(item, "%-12s", "KMS").Sprintf("%s: %s\n", kmsKind, kmsEndpoints[0]) - for _, endpoint := range kmsEndpoints[1:] { - buffer.Sprintf("%-12s", " ").Sprint(strings.Repeat(" ", len(kmsKind))).Sprintf(" %s\n", endpoint) - } - buffer.Stylef(item, "%-12s", "Endpoints").Sprintf("https://%s:%s\n", ifaceIPs[0], port) - for _, ifaceIP := range ifaceIPs[1:] { - buffer.Sprintf("%-12s", " ").Sprintf("https://%s:%s\n", ifaceIP, port) - } - buffer.Sprintln() - if r, err := hex.DecodeString(config.Admin.String()); err == nil && len(r) == sha256.Size { - buffer.Stylef(item, "%-12s", "Admin").Sprintln(config.Admin) - } else { - buffer.Stylef(item, "%-12s", "Admin").Sprintf("%-22s", "_").Styleln(faint, "[ disabled ]") - } - if tlsConfig.ClientAuth == tls.RequireAndVerifyClientCert { - buffer.Stylef(item, "%-12s", "Mutual TLS").Sprint("on").Styleln(faint, "Verify client certificates") - } - switch { - case runtime.GOOS == "linux" && mlock: - buffer.Stylef(item, "%-12s", "Mem Lock").Stylef(green, "%-22s", "on").Styleln(faint, "RAM pages will not be swapped to disk") - case runtime.GOOS == "linux": - buffer.Stylef(item, "%-12s", "Mem Lock").Stylef(red, "%-22s", "off").Styleln(faint, "Failed to lock RAM pages. Consider granting CAP_IPC_LOCK") - default: - buffer.Stylef(item, "%-12s", "Mem Lock").Stylef(red, "%-22s", "off").Stylef(faint, "Not supported on %s/%s\n", runtime.GOOS, runtime.GOARCH) - } - return buffer, nil -} diff --git a/cmd/kes/identity.go b/cmd/kes/identity.go index e135f4dd..8171f5d9 100644 --- a/cmd/kes/identity.go +++ b/cmd/kes/identity.go @@ -340,7 +340,6 @@ Options: is detected - colors are automatically disabled if the output goes to a pipe. Possible values: *auto*, never, always. - -e, --enclave Operate within the specified enclave. -h, --help Print command line options. @@ -392,9 +391,9 @@ func infoIdentityCmd(args []string) { dotDenyStyle = dotDenyStyle.Foreground(ColorDotDeny) } - enclave := newEnclave(enclaveName, insecureSkipVerify) + client := newClient(insecureSkipVerify) if cmd.NArg() == 0 { - info, policy, err := enclave.DescribeSelf(ctx) + info, policy, err := client.DescribeSelf(ctx) if err != nil { cli.Fatal(err) } @@ -441,7 +440,7 @@ func infoIdentityCmd(args []string) { } } } else { - info, err := enclave.DescribeIdentity(ctx, kes.Identity(cmd.Arg(0))) + info, err := client.DescribeIdentity(ctx, kes.Identity(cmd.Arg(0))) if err != nil { cli.Fatal(err) } @@ -591,12 +590,12 @@ func rmIdentityCmd(args []string) { cli.Fatal("no identity specified. See 'kes identity rm --help'") } - enclave := newEnclave(enclaveName, insecureSkipVerify) + client := newClient(insecureSkipVerify) ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) defer cancel() for _, identity := range cmd.Args() { - if err := enclave.DeleteIdentity(ctx, kes.Identity(identity)); err != nil { + if err := client.DeleteIdentity(ctx, kes.Identity(identity)); err != nil { if errors.Is(err, context.Canceled) { os.Exit(1) } diff --git a/cmd/kes/key.go b/cmd/kes/key.go index 3440cfcf..caf04c50 100644 --- a/cmd/kes/key.go +++ b/cmd/kes/key.go @@ -116,9 +116,9 @@ func createKeyCmd(args []string) { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) defer cancel() - enclave := newEnclave(enclaveName, insecureSkipVerify) + client := newClient(insecureSkipVerify) for _, name := range cmd.Args() { - if err := enclave.CreateKey(ctx, name); err != nil { + if err := client.CreateKey(ctx, name); err != nil { if errors.Is(err, context.Canceled) { os.Exit(1) } @@ -234,8 +234,8 @@ func describeKeyCmd(args []string) { defer cancelCtx() name := cmd.Arg(0) - enclave := newEnclave(enclaveName, insecureSkipVerify) - info, err := enclave.DescribeKey(ctx, name) + client := newClient(insecureSkipVerify) + info, err := client.DescribeKey(ctx, name) if err != nil { if errors.Is(err, context.Canceled) { os.Exit(1) @@ -385,9 +385,9 @@ func rmKeyCmd(args []string) { ctx, cancelCtx := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) defer cancelCtx() - enclave := newEnclave(enclaveName, insecureSkipVerify) + client := newClient(insecureSkipVerify) for _, name := range cmd.Args() { - if err := enclave.DeleteKey(ctx, name); err != nil { + if err := client.DeleteKey(ctx, name); err != nil { if errors.Is(err, context.Canceled) { os.Exit(1) } @@ -441,8 +441,8 @@ func encryptKeyCmd(args []string) { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) defer cancel() - enclave := newEnclave(enclaveName, insecureSkipVerify) - ciphertext, err := enclave.Encrypt(ctx, name, []byte(message), nil) + client := newClient(insecureSkipVerify) + ciphertext, err := client.Encrypt(ctx, name, []byte(message), nil) if err != nil { if errors.Is(err, context.Canceled) { os.Exit(1) @@ -514,8 +514,8 @@ func decryptKeyCmd(args []string) { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) defer cancel() - enclave := newEnclave(enclaveName, insecureSkipVerify) - plaintext, err := enclave.Decrypt(ctx, name, ciphertext, associatedData) + client := newClient(insecureSkipVerify) + plaintext, err := client.Decrypt(ctx, name, ciphertext, associatedData) if err != nil { if errors.Is(err, context.Canceled) { os.Exit(1) @@ -580,8 +580,8 @@ func dekCmd(args []string) { ctx, cancelCtx := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) defer cancelCtx() - enclave := newEnclave(enclaveName, insecureSkipVerify) - key, err := enclave.GenerateKey(ctx, name, associatedData) + client := newClient(insecureSkipVerify) + key, err := client.GenerateKey(ctx, name, associatedData) if err != nil { if errors.Is(err, context.Canceled) { os.Exit(1) diff --git a/cmd/kes/main.go b/cmd/kes/main.go index 702ee17d..cda5ac1c 100644 --- a/cmd/kes/main.go +++ b/cmd/kes/main.go @@ -12,8 +12,11 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" + "time" + tui "github.com/charmbracelet/lipgloss" "github.com/minio/kes-go" "github.com/minio/kes/internal/cli" "github.com/minio/kes/internal/https" @@ -95,15 +98,28 @@ func main() { if cmd.NArg() > 1 { cli.Fatalf("%q is not a kes command. See 'kes --help'", cmd.Arg(1)) } + if showVersion { - buildInfo := sys.BinaryInfo() - cli.Printf("kes %s (commit=%s)\n", buildInfo.Version, buildInfo.CommitID) + info, err := sys.ReadBinaryInfo() + if err != nil { + cli.Fatal(err) + } + + faint := tui.NewStyle().Faint(true) + buf := &strings.Builder{} + fmt.Fprintf(buf, "Version %-22s %s\n", info.Version, faint.Render("commit="+info.CommitID)) + fmt.Fprintf(buf, "Runtime %-22s %s\n", fmt.Sprintf("%s %s/%s", info.Runtime, runtime.GOOS, runtime.GOARCH), faint.Render("compiler="+info.Compiler)) + fmt.Fprintf(buf, "License %-22s %s\n", "AGPLv3", faint.Render("https://www.gnu.org/licenses/agpl-3.0.html")) + fmt.Fprintf(buf, "Copyright %-22s %s\n", fmt.Sprintf("2015-%d MinIO Inc.", time.Now().Year()), faint.Render("https://min.io")) + fmt.Print(buf.String()) return } + if autoCompletion { installAutoCompletion() return } + cmd.Usage() os.Exit(2) } diff --git a/cmd/kes/mlock_linux.go b/cmd/kes/mlock_linux.go index 780f8e17..daa66f4e 100644 --- a/cmd/kes/mlock_linux.go +++ b/cmd/kes/mlock_linux.go @@ -11,3 +11,5 @@ import ( ) func mlockall() error { return unix.Mlockall(syscall.MCL_CURRENT | syscall.MCL_FUTURE) } + +func munlockall() error { return unix.Munlockall() } diff --git a/cmd/kes/mlock_ref.go b/cmd/kes/mlock_ref.go index 1cde210c..3a00c8f8 100644 --- a/cmd/kes/mlock_ref.go +++ b/cmd/kes/mlock_ref.go @@ -12,3 +12,9 @@ func mlockall() error { // on linux at the moment. return nil } + +func munlockall() error { + // We only support locking memory pages + // on linux at the moment. + return nil +} diff --git a/cmd/kes/policy.go b/cmd/kes/policy.go index 9351a724..49784a2c 100644 --- a/cmd/kes/policy.go +++ b/cmd/kes/policy.go @@ -189,9 +189,9 @@ func rmPolicyCmd(args []string) { ctx, cancelCtx := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) defer cancelCtx() - enclave := newEnclave(enclaveName, insecureSkipVerify) + client := newClient(insecureSkipVerify) for _, name := range cmd.Args() { - if err := enclave.DeletePolicy(ctx, name); err != nil { + if err := client.DeletePolicy(ctx, name); err != nil { if errors.Is(err, context.Canceled) { os.Exit(1) } @@ -247,8 +247,8 @@ func infoPolicyCmd(args []string) { defer cancelCtx() name := cmd.Arg(0) - enclave := newEnclave(enclaveName, insecureSkipVerify) - info, err := enclave.DescribePolicy(ctx, name) + client := newClient(insecureSkipVerify) + info, err := client.DescribePolicy(ctx, name) if err != nil { if errors.Is(err, context.Canceled) { os.Exit(1) @@ -322,12 +322,12 @@ func showPolicyCmd(args []string) { } name := cmd.Arg(0) - enclave := newEnclave(enclaveName, insecureSkipVerify) + client := newClient(insecureSkipVerify) ctx, cancelCtx := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) defer cancelCtx() - policy, err := enclave.GetPolicy(ctx, name) + policy, err := client.GetPolicy(ctx, name) if err != nil { if errors.Is(err, context.Canceled) { os.Exit(1) diff --git a/cmd/kes/server.go b/cmd/kes/server.go index 4618ce48..6b7aabee 100644 --- a/cmd/kes/server.go +++ b/cmd/kes/server.go @@ -5,12 +5,31 @@ package main import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/hex" "errors" "fmt" + "log/slog" "net" "os" + "os/signal" + "path/filepath" + "runtime" + "slices" + "strings" + "syscall" + "time" + tui "github.com/charmbracelet/lipgloss" + "github.com/minio/kes" + kesdk "github.com/minio/kes-go" + "github.com/minio/kes/edge" "github.com/minio/kes/internal/cli" + "github.com/minio/kes/internal/https" + "github.com/minio/kes/internal/sys" + "github.com/minio/kes/kv" flag "github.com/spf13/pflag" ) @@ -48,6 +67,14 @@ Examples: $ kes server --config config.yml --auth =off ` +type serverArgs struct { + Address string + ConfigFile string + PrivateKey string + Certificate string + TLSAuth string +} + func serverCmd(args []string) { cmd := flag.NewFlagSet(args[0], flag.ContinueOnError) cmd.Usage = func() { fmt.Fprint(os.Stderr, serverCmdUsage) } @@ -75,68 +102,490 @@ func serverCmd(args []string) { cli.Fatal("too many arguments. See 'kes server --help'") } - startGateway(gatewayConfig{ + var memLocked bool + if runtime.GOOS == "linux" { + memLocked = mlockall() == nil + defer munlockall() + } + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + config, err := readServerConfig(ctx, serverArgs{ Address: addrFlag, ConfigFile: configFlag, PrivateKey: tlsKeyFlag, Certificate: tlsCertFlag, TLSAuth: mtlsAuthFlag, }) + if err != nil { + cli.Fatal(err) + } + + srv := &kes.Server{} + srv.ErrLevel.Set(slog.LevelWarn) + + sighup := make(chan os.Signal, 10) + signal.Notify(sighup, syscall.SIGHUP) + defer signal.Stop(sighup) + + go func(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-sighup: + fmt.Fprintln(os.Stderr, "SIGHUP signal received. Reloading configuration...") + config, err := readServerConfig(ctx, serverArgs{ + Address: addrFlag, + ConfigFile: configFlag, + PrivateKey: tlsKeyFlag, + Certificate: tlsCertFlag, + TLSAuth: mtlsAuthFlag, + }) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to reload server config: %v\n", err) + continue + } + config.Keys = &kes.MemKeyStore{} + + closer, err := srv.Update(config) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to update server configuration: %v\n", err) + continue + } + + if err = closer.Close(); err != nil { + fmt.Fprintf(os.Stderr, "Failed to close previous keystore connections: %v\n", err) + } + buf, err := printServerStartup(srv, addrFlag, config, memLocked) + if err == nil { + fmt.Fprintln(buf) + fmt.Fprintln(buf, "=> Reloading configuration after SIGHUP signal completed.") + fmt.Println(buf.String()) + } + } + } + }(ctx) + + go func(ctx context.Context) { + ticker := time.NewTicker(1 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + config, err := readServerConfig(ctx, serverArgs{ + Address: addrFlag, + ConfigFile: configFlag, + PrivateKey: tlsKeyFlag, + Certificate: tlsCertFlag, + TLSAuth: mtlsAuthFlag, + }) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to reload TLS configuration: %v\n", err) + continue + } + if err = srv.UpdateTLS(config.TLS); err != nil { + fmt.Fprintf(os.Stderr, "Failed to update TLS configuration: %v\n", err) + } + } + } + }(ctx) + + buf, err := printServerStartup(srv, addrFlag, config, memLocked) + if err != nil { + cli.Fatal(err) + } + fmt.Fprintln(buf) + fmt.Fprintln(buf, "=> Server is up and running...") + fmt.Println(buf.String()) + + if err = srv.ListenAndStart(ctx, addrFlag, config); err != nil { + cli.Fatal(err) + } + fmt.Println("\n=> Stopping server... Goodbye.") } -// listeningOnV4 returns a list of the system IPv4 interface -// addresses an TCP/IP listener with the given IP is listening -// on. -// -// In particular, a TCP/IP listener listening on the pseudo -// address 0.0.0.0 listens on all network interfaces while -// a listener on a specific IP only listens on the network -// interface with that IP address. -func listeningOnV4(ip net.IP) []net.IP { - if !ip.IsUnspecified() { - return []net.IP{ip} - } - // We listen on the pseudo-address: 0.0.0.0 - // The TCP/IP listener is listening on all available - // network interfaces. - interfaces, err := net.InterfaceAddrs() +func printServerStartup(srv *kes.Server, addr string, config *kes.Config, memLocked bool) (*strings.Builder, error) { + info, err := sys.ReadBinaryInfo() + if err != nil { + return nil, err + } + + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + ip := net.IPv4zero + if host != "" { + if ip = net.ParseIP(host); ip == nil { + return nil, fmt.Errorf("'%s' is not a valid IP address", host) + } + } + ifaceIPs, err := lookupInterfaceIPs(ip) + if err != nil { + return nil, err + } + + keys := config.Keys.(adapter) + + blue := tui.NewStyle().Foreground(tui.Color("#268BD2")) + faint := tui.NewStyle().Faint(true) + + buf := &strings.Builder{} + fmt.Fprintf(buf, "%-33s %-23s %s\n", blue.Render("Version"), info.Version, faint.Render("commit="+info.CommitID)) + fmt.Fprintf(buf, "%-33s %-23s %s\n", blue.Render("Runtime"), fmt.Sprintf("%s %s/%s", info.Runtime, runtime.GOOS, runtime.GOARCH), faint.Render("compiler="+info.Compiler)) + fmt.Fprintf(buf, "%-33s %-23s %s\n", blue.Render("License"), "AGPLv3", faint.Render("https://www.gnu.org/licenses/agpl-3.0.html")) + fmt.Fprintf(buf, "%-33s %-12s 2015-%d %s\n", blue.Render("Copyright"), "MinIO, Inc.", time.Now().Year(), faint.Render("https://min.io")) + fmt.Fprintln(buf) + fmt.Fprintf(buf, "%-33s %s: %s\n", blue.Render("KMS"), keys.Type, keys.Endpoint) + fmt.Fprintf(buf, "%-33s · https://%s\n", blue.Render("API"), net.JoinHostPort(ifaceIPs[0].String(), port)) + for _, ifaceIP := range ifaceIPs[1:] { + fmt.Fprintf(buf, "%-11s · https://%s\n", " ", net.JoinHostPort(ifaceIP.String(), port)) + } + + fmt.Fprintln(buf) + fmt.Fprintf(buf, "%-33s https://min.io/docs/kes\n", blue.Render("Docs")) + + fmt.Fprintln(buf) + if _, err := hex.DecodeString(config.Admin.String()); err == nil { + fmt.Fprintf(buf, "%-33s %s\n", blue.Render("Admin"), config.Admin) + } else { + fmt.Fprintf(buf, "%-33s \n", blue.Render("Admin")) + } + fmt.Fprintf(buf, "%-33s error=stderr level=%s\n", blue.Render("Logs"), srv.ErrLevel.Level()) + if srv.AuditLevel.Level() <= slog.LevelInfo { + fmt.Fprintf(buf, "%-11s audit=stdout level=%s\n", " ", srv.AuditLevel.Level()) + } + if memLocked { + fmt.Fprintf(buf, "%-33s %s\n", blue.Render("MLock"), "enabled") + } + return buf, nil +} + +func readServerConfig(ctx context.Context, args serverArgs) (*kes.Config, error) { + file, err := os.Open(args.ConfigFile) if err != nil { - return []net.IP{} + return nil, err + } + defer file.Close() + + config, err := edge.ReadServerConfigYAML(file) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %v", err) + } + if err = file.Close(); err != nil { + return nil, err + } + + if args.Address != "" { + config.Addr = args.Address + } + if args.PrivateKey != "" { + config.TLS.PrivateKey = args.PrivateKey + } + if args.Certificate != "" { + config.TLS.Certificate = args.Certificate } - var ip4Addr []net.IP - for _, iface := range interfaces { - var ip net.IP - switch addr := iface.(type) { - case *net.IPNet: - ip = addr.IP.To4() - case *net.IPAddr: - ip = addr.IP.To4() + // Set config defaults + if config.Addr == "" { + config.Addr = "0.0.0.0:7373" + } + if config.Cache.Expiry == 0 { + config.Cache.Expiry = 5 * time.Minute + } + if config.Cache.ExpiryUnused == 0 { + config.Cache.ExpiryUnused = 30 * time.Second + } + + // Verify config + if config.Admin.IsUnknown() { + return nil, errors.New("no admin identity specified") + } + if config.TLS.PrivateKey == "" { + return nil, errors.New("no TLS private key specified") + } + if config.TLS.Certificate == "" { + return nil, errors.New("no TLS certificate specified") + } + + certificate, err := https.CertificateFromFile(config.TLS.Certificate, config.TLS.PrivateKey, config.TLS.Password) + if err != nil { + return nil, fmt.Errorf("failed to read TLS certificate: %v", err) + } + if certificate.Leaf != nil { + if len(certificate.Leaf.DNSNames) == 0 && len(certificate.Leaf.IPAddresses) == 0 { + // Support for TLS certificates with a subject CN but without any SAN + // has been removed in Go 1.15. Ref: https://go.dev/doc/go1.15#commonname + // Therefore, we require at least one SAN for the server certificate. + return nil, fmt.Errorf("invalid TLS certificate: certificate does not contain any DNS or IP address as SAN") } - if ip != nil { - ip4Addr = append(ip4Addr, ip) + } + + var rootCAs *x509.CertPool + if config.TLS.CAPath != "" { + rootCAs, err = https.CertPoolFromFile(config.TLS.CAPath) + if err != nil { + return nil, fmt.Errorf("failed to read TLS CA certificates: %v", err) } } - return ip4Addr + + var errorLog slog.Handler + var auditLog kes.AuditHandler + if config.Log.Error { + errorLog = slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}) + } + if config.Log.Audit { + auditLog = &kes.AuditLogHandler{Handler: slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})} + } + + // TODO(aead): support TLS proxies + + var apiConfig map[string]kes.RouteConfig + if config.API != nil && len(config.API.Paths) > 0 { + apiConfig = make(map[string]kes.RouteConfig, len(config.API.Paths)) + for k, v := range config.API.Paths { + k = strings.TrimSpace(k) // Ensure that the API path starts with a '/' + if !strings.HasPrefix(k, "/") { + k = "/" + k + } + + if _, ok := apiConfig[k]; ok { + return nil, fmt.Errorf("ambiguous API configuration for '%s'", k) + } + apiConfig[k] = kes.RouteConfig{ + Timeout: v.Timeout, + InsecureSkipAuth: v.InsecureSkipAuth, + } + } + } + + policies := make(map[string]kes.Policy, len(config.Policies)) + for name, policy := range config.Policies { + p := kes.Policy{ + Allow: make(map[string]kesdk.Rule, len(policy.Allow)), + Deny: make(map[string]kesdk.Rule, len(policy.Deny)), + Identities: slices.Clone(policy.Identities), + } + for _, pattern := range policy.Allow { + p.Allow[pattern] = kesdk.Rule{} + } + for _, pattern := range policy.Deny { + p.Deny[pattern] = kesdk.Rule{} + } + policies[name] = p + } + + kmsKind, kmsEndpoint, err := description(config) + if err != nil { + return nil, err + } + + store, err := config.KeyStore.Connect(ctx) + if err != nil { + return nil, err + } + + keys := adapter{ + store: store, + Type: kmsKind, + Endpoint: kmsEndpoint, + } + + return &kes.Config{ + Admin: config.Admin, + TLS: &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{certificate}, + RootCAs: rootCAs, + ClientAuth: tls.RequestClientCert, + }, + Cache: &kes.CacheConfig{ + Expiry: config.Cache.Expiry, + ExpiryUnused: config.Cache.ExpiryUnused, + ExpiryOffline: config.Cache.ExpiryOffline, + }, + Keys: keys, + Policies: policies, + Routes: apiConfig, + ErrorLog: errorLog, + AuditLog: auditLog, + }, nil } -// serverAddr takes an address string : and -// splits it into an IP address and port number. +// TODO(aead): temp adapater - remove once keystores are ported to KeyStore interface +type adapter struct { + Type string + + Endpoint string + + store kv.Store[string, []byte] +} + +func (a adapter) Status(ctx context.Context) (kes.KeyStoreState, error) { + s, err := a.store.Status(ctx) + if err != nil { + return kes.KeyStoreState{}, err + } + return kes.KeyStoreState{ + Latency: s.Latency, + }, nil +} + +func (a adapter) Create(ctx context.Context, name string, value []byte) error { + return a.store.Create(ctx, name, value) +} + +func (a adapter) Delete(ctx context.Context, name string) error { + return a.store.Delete(ctx, name) +} + +func (a adapter) Get(ctx context.Context, name string) ([]byte, error) { + return a.store.Get(ctx, name) +} + +func (a adapter) List(ctx context.Context, prefix string, n int) ([]string, string, error) { + if n == 0 { + return []string{}, prefix, nil + } + + iter, err := a.store.List(ctx) + if err != nil { + return nil, "", err + } + defer iter.Close() + + var keys []string + for key, ok := iter.Next(); ok; key, ok = iter.Next() { + keys = append(keys, key) + } + if err = iter.Close(); err != nil { + return nil, "", err + } + slices.Sort(keys) + + if prefix == "" { + if n < 0 || n >= len(keys) { + return keys, "", nil + } + return keys[:n], keys[n], nil + } + + i := slices.IndexFunc(keys, func(key string) bool { return strings.HasPrefix(key, prefix) }) + if i < 0 { + return []string{}, "", nil + } + + for j, key := range keys[i:] { + if !strings.HasPrefix(key, prefix) { + return keys[i : i+j], "", nil + } + if n > 0 && j == n { + return keys[i : i+j], key, nil + } + } + return keys[i:], "", nil +} + +func (a adapter) Close() error { return a.store.Close() } + +func description(config *edge.ServerConfig) (kind string, endpoint string, err error) { + if config.KeyStore == nil { + return "", "", errors.New("no KMS backend specified") + } + + switch kms := config.KeyStore.(type) { + case *edge.FSKeyStore: + kind = "Filesystem" + if abs, err := filepath.Abs(kms.Path); err == nil { + endpoint = abs + } else { + endpoint = kms.Path + } + case *edge.VaultKeyStore: + kind = "Hashicorp Vault" + endpoint = kms.Endpoint + case *edge.FortanixKeyStore: + kind = "Fortanix SDKMS" + endpoint = kms.Endpoint + case *edge.AWSSecretsManagerKeyStore: + kind = "AWS SecretsManager" + endpoint = kms.Endpoint + case *edge.KeySecureKeyStore: + kind = "Gemalto KeySecure" + endpoint = kms.Endpoint + case *edge.GCPSecretManagerKeyStore: + kind = "GCP SecretManager" + endpoint = "Project: " + kms.ProjectID + case *edge.AzureKeyVaultKeyStore: + kind = "Azure KeyVault" + endpoint = kms.Endpoint + case *edge.EntrustKeyControlKeyStore: + kind = "Entrust KeyControl" + endpoint = kms.Endpoint + default: + return "", "", fmt.Errorf("unknown KMS backend %T", kms) + } + return kind, endpoint, nil +} + +// lookupInterfaceIPs returns a list of IP addrs for which a listener +// listening on listenerIP is reachable. If listenerIP is not +// unspecified (0.0.0.0) it returns []net.IP{listenerIP}. // -// If addr does not contain an IP (":") then ip will be -// 0.0.0.0. -func serverAddr(addr string) (ip net.IP, port string) { - host, port, err := net.SplitHostPort(addr) +// Otherwise, lookupInterfaceIPs iterates over all available network +// interfaces excluding unicast and multicast IPs. It prefers IPv4 +// addrs and only returns IPv6 addrs if there are no IPv4 addrs. +func lookupInterfaceIPs(listenerIP net.IP) ([]net.IP, error) { + if !listenerIP.IsUnspecified() { + return []net.IP{listenerIP}, nil + } + + ifaces, err := net.Interfaces() if err != nil { - cli.Fatalf("invalid server address: %q", addr) + return nil, err } - if host == "" { - host = "0.0.0.0" + + var ipv4s, ipv6s []net.IP + for _, iface := range ifaces { + if iface.Flags&net.FlagUp == 0 { // interface is down + continue + } + addrs, err := iface.Addrs() + if err != nil { + return nil, err + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + if ip == nil || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() || ip.IsMulticast() { + continue + } + + if ipv4 := ip.To4(); ipv4 != nil && !slices.ContainsFunc(ipv4s, func(x net.IP) bool { return ipv4.Equal(x) }) { + ipv4s = append(ipv4s, ipv4) + } else if !slices.ContainsFunc(ipv6s, func(x net.IP) bool { return ip.Equal(x) }) { + ipv6s = append(ipv6s, ip) + } + } } - ip = net.ParseIP(host) - if ip == nil { - cli.Fatalf("invalid server address: %q", addr) + if len(ipv4s) > 0 { // prefer IPv4 addrs, if any + return ipv4s, nil + } + if len(ipv6s) > 0 { + return ipv6s, nil } - return ip, port + return nil, errors.New("no IPv4 or IPv6 addresses available") } diff --git a/cmd/kes/update.go b/cmd/kes/update.go index 67b8a10f..2ba846c5 100644 --- a/cmd/kes/update.go +++ b/cmd/kes/update.go @@ -160,7 +160,8 @@ func updateCmd(args []string) { version = v } - if cv, err := time.Parse(releaseTagFormat, sys.BinaryInfo().Version); err == nil { + info, _ := sys.ReadBinaryInfo() + if cv, err := time.Parse(releaseTagFormat, info.Version); err == nil { switch version.After(cv) { case true: cli.Println(fmt.Sprintf("Upgrading from '%v' to '%v'", cv, version)) diff --git a/config.go b/config.go new file mode 100644 index 00000000..62a7340d --- /dev/null +++ b/config.go @@ -0,0 +1,155 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package kes + +import ( + "crypto/tls" + "errors" + "log/slog" + "time" + + "github.com/minio/kes-go" +) + +// Config is a structure that holds configuration for a KES server. +type Config struct { + // Admin is the KES server admin identity. It must not be empty. + // To disable admin access set it to a non-hex value. For example, + // "disabled". + Admin kes.Identity + + // TLS contains the KES server's TLS configuration. + // + // A KES server requires a TLS certificate. Therefore, either + // Config.Certificates, Config.GetCertificate or + // Config.GetConfigForClient must be set. + // + // Further, the KES server has to request client certificates + // for mTLS authentication. Hence, Config.ClientAuth must be + // at least tls.RequestClientCert. + TLS *tls.Config + + // Cache specifies how long the KES server caches keys from the + // KeyStore. If nil, caching is disabled. + Cache *CacheConfig + + // Policies is a set of policies and identities. Each identity + // must be assigned to a policy only once. + Policies map[string]Policy + + // Keys is the KeyStore the KES server fetches keys from. + Keys KeyStore + + // Routes allows customization of the KES server API routes. It + // contains a set of API route paths, for example "/v1/status", + // and the corresponding route configuration. + // + // The KES server uses sane defaults for all its API routes. + Routes map[string]RouteConfig + + // ErrorLog is an optional handler for handling the server's + // error log events. If nil, defaults to a slog.TextHandler + // writing to os.Stderr. The server's error log level is + // controlled by Server.ErrLevel. + ErrorLog slog.Handler + + // AuditLog is an optional handler for handling the server's + // audit log events. If nil, defaults to a slog.TextHandler + // writing to os.Stdout. The server's audit log level is + // controlled by Server.AuditLevel. + AuditLog AuditHandler +} + +// Policy is a KES policy with associated identities. +// +// A policy contains a set of allow and deny rules. +type Policy struct { + Allow map[string]kes.Rule // Set of allow rules + + Deny map[string]kes.Rule // Set of deny rules + + Identities []kes.Identity +} + +// CacheConfig is a structure containing the KES server +// key store cache configuration. +type CacheConfig struct { + // Expiry controls how long a particular key resides + // in the cache. If zero or negative, keys remain in + // the cache as long as the KES server has sufficient + // memory. + Expiry time.Duration + + // ExpiryUnused is the interval in which a particular + // must be accessed to remain in the cache. Keys that + // haven't been accessed get evicted from the cache. + // The general cache expiry still applies. + // + // ExpiryUnused does nothing if <= 0 or greater than + // Expiry. + ExpiryUnused time.Duration + + // ExpiryOffline controls how long a particular key + // resides in the cache once the key store becomes + // unavailable. It overwrites Expiry and ExpiryUnused + // if the key store is not available. Once the key + // store is available again, Expiry and ExpiryUnused, + // if set, apply. + // + // A common use of ExpiryOffline is reducing the impact + // of a key store outage, and therefore, improving + // availability. + // + // Offline caching is disabled if ExpiryOffline <= 0. + ExpiryOffline time.Duration +} + +// RouteConfig is a structure holding API route configuration. +type RouteConfig struct { + // Timeout specifies when the API handler times out. + // + // A handler times out when it fails to send the + // *entire* response body to the client within the + // given time period. + // + // If <= 0, timeouts are disabled for the API route. + // + // Disabling timeouts may leave client/server connections + // hang or allow certain types of denial-of-service (DOS) + // attacks. + Timeout time.Duration + + // InsecureSkipAuth, if set, disables authentication for the + // API route. It allows anyone that can send HTTPS requests + // to the KES server to invoke the API. + // + // For example the KES readiness API authentication may be + // disabled when the probing clients do not support mTLS + // client authentication + // + // If setting InsecureSkipAuth for any API then clients that + // do not send a client certificate during the TLS handshake + // no longer encounter a TLS handshake error but receive a + // HTTP error instead. In particular, when the server's TLS + // client auth type has been set tls.RequireAnyClientCert + // or tls.RequireAndVerifyClientCert. + InsecureSkipAuth bool +} + +// verifyConfig reports whether the c is a valid Config +// and contains at least a TLS certificate for the server +// and a key store. +func verifyConfig(c *Config) error { + if c == nil || c.TLS == nil || (len(c.TLS.Certificates) == 0 && c.TLS.GetCertificate == nil && c.TLS.GetConfigForClient == nil) { + return errors.New("kes: tls config contains no server certificate") + } + if c.TLS.ClientAuth == tls.NoClientCert { + return errors.New("kes: tls client auth must request client certifiate") + } + if c.Keys == nil { + return errors.New("kes: config contains no key store") + } + return nil +} diff --git a/example_test.go b/example_test.go new file mode 100644 index 00000000..a90824b3 --- /dev/null +++ b/example_test.go @@ -0,0 +1,44 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package kes_test + +import ( + "context" + "log/slog" + "net/http" + "net/netip" + "os" + "time" + + "github.com/minio/kes" +) + +// This example shows how to connect an AuditHandler +// to any slog.Handler, here an TextHandler writing +// to stdout. +func ExampleAuditLogHandler() { + audit := &kes.AuditLogHandler{ + Handler: slog.NewTextHandler(os.Stdout, nil), + } + conf := &kes.Config{ + AuditLog: audit, + } + _ = conf + + // Handle will be called by the KES server internally + audit.Handle(context.Background(), kes.AuditRecord{ + Time: time.Date(2023, time.October, 19, 8, 44, 0, 0, time.UTC), + Method: http.MethodPut, + Path: "/v1/key/create/my-key", + Identity: "2ecb8804e7702a6b768e89b7bba5933044c9d071e4f4035235269b919e56e691", + RemoteIP: netip.MustParseAddr("10.1.2.3"), + StatusCode: http.StatusOK, + ResponseTime: 200 * time.Millisecond, + Level: slog.LevelInfo, + Message: "secret key 'my-key' created", + }) + // Output: + // time=2023-10-19T08:44:00.000Z level=INFO msg="secret key 'my-key' created" req.method=PUT req.path=/v1/key/create/my-key req.ip=10.1.2.3 req.identity=2ecb8804e7702a6b768e89b7bba5933044c9d071e4f4035235269b919e56e691 res.code=200 res.time=200ms +} diff --git a/go.mod b/go.mod index bb95e2ba..86072759 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/minio/kes -go 1.18 +go 1.20 require ( aead.dev/mem v0.2.0 diff --git a/internal/api/api.go b/internal/api/api.go index cee52fdc..1a20fe17 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -5,187 +5,251 @@ package api import ( + "encoding/json" "errors" - "fmt" + "log/slog" "net/http" + "net/netip" + "strconv" "strings" - "sync" "time" + "aead.dev/mem" "github.com/minio/kes-go" + "github.com/minio/kes/internal/headers" ) -// Config is a structure for configuring -// a KES server API. -type Config struct { - // Timeout is the duration after which a request - // times out. If Timeout <= 0 the API default - // is used. - Timeout time.Duration - - // InsecureSkipAuth controls whether the API verifies - // client identities. If InsecureSkipAuth is true, - // the API accepts requests from arbitrary identities. - // In this mode, the API can be used by anyone who can - // communicate to the KES server over HTTPS. - // This should only be set for testing or in certain - // cases for APIs that don't expose sensitive information, - // like metrics. - InsecureSkipAuth bool -} +// API paths exposed by KES servers. +const ( + PathVersion = "/version" + PathStatus = "/v1/status" + PathReady = "/v1/ready" + PathMetrics = "/v1/metrics" + PathListAPIs = "/v1/api" + + PathKeyCreate = "/v1/key/create/" + PathKeyImport = "/v1/key/import/" + PathKeyDescribe = "/v1/key/describe/" + PathKeyDelete = "/v1/key/delete/" + PathKeyList = "/v1/key/list/" + PathKeyGenerate = "/v1/key/generate/" + PathKeyEncrypt = "/v1/key/encrypt/" + PathKeyDecrypt = "/v1/key/decrypt/" + + PathPolicyDescribe = "/v1/policy/describe/" + PathPolicyRead = "/v1/policy/read/" + PathPolicyList = "/v1/policy/list/" + + PathIdentityDescribe = "/v1/identity/describe/" + PathIdentityList = "/v1/identity/list/" + PathIdentitySelfDescribe = "/v1/identity/self/describe" + + PathLogError = "/v1/log/error" + PathLogAudit = "/v1/log/audit" +) -// API describes a KES server API. -type API struct { - Method string // The HTTP method - Path string // The URI API path - MaxBody int64 // The max. body size the API accepts - Timeout time.Duration // The duration after which an API request times out. 0 means no timeout - Verify bool // Whether the API verifies the client identity - - // Handler implements the API. - // - // When invoked by the API's ServeHTTP method, the handler - // can rely upon: - // - the request method matching the API's HTTP method. - // - the API path being a prefix of the request URL. - // - the request body being limited to the API's MaxBody size. - // - the request timing out after the duration specified for the API. - Handler http.Handler - - _ [0]int +// Route represents an API route handling a client request. +type Route struct { + Method string // The HTTP method (GET, PUT, DELETE, ...) + Path string // The API Path + MaxBody mem.Size // The max. size of a request body + Timeout time.Duration // Timeout after which the request gets aborted + Auth Authenticator // The authentication method for this API route + Handler Handler // The API handler implementing the server-side logic } -// ServerHTTP takes an HTTP Request and ResponseWriter and executes the -// API's Handler. -func (a API) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if r.Method != a.Method { - w.Header().Set("Accept", a.Method) - Fail(w, kes.NewError(http.StatusMethodNotAllowed, http.StatusText(http.StatusMethodNotAllowed))) - return +// ServeHTTP implements the http.Handler for Route and handles an incoming +// client request as following: +// - Verify that the request method matches Route.Method. +// - Verify that the request got routed correctly, i.e. Route.Path is a +// prefix of the request path. +// - Limit the request body to Route.MaxBody. +// - Apply Route.Timeout and timeout the request if generating a response +// takes longer. +// - Authenticate the request. If Route.Auth.Authenticate returns an error +// the error is sent to the client and the route handler is not invoked. +// - Handle the request. The Route.Handler.ServeAPI is invoked with the +// authenticated request. +func (ro Route) ServeHTTP(w http.ResponseWriter, r *http.Request) { + resp := &Response{ + ResponseWriter: w, } - if !strings.HasPrefix(r.URL.Path, a.Path) { - Fail(w, fmt.Errorf("api: patch mismatch: received '%s' - expected '%s'", r.URL.Path, a.Path)) - return - } - r.Body = http.MaxBytesReader(w, r.Body, a.MaxBody) + received := time.Now() - if a.Timeout > 0 { - switch err := http.NewResponseController(w).SetWriteDeadline(time.Now().Add(a.Timeout)); { - case errors.Is(err, http.ErrNotSupported): - Fail(w, errors.New("internal error: HTTP connection does not accept a timeout")) - return - case err != nil: - Fail(w, fmt.Errorf("internal error: %v", err)) + if r.Method != ro.Method { + if !(r.Method == http.MethodPost && ro.Method == http.MethodPut) { + w.Header().Set(headers.Accept, ro.Method) + resp.Failf(http.StatusMethodNotAllowed, "received method '%s' expected '%s'", r.Method, ro.Method) return } } - a.Handler.ServeHTTP(w, r) -} -// nameFromRequest strips the API path from the request URL, verifies -// that the remaining path is a valid name, via verifyName, and returns -// the remaining path. -func nameFromRequest(r *http.Request, apiPath string) (string, error) { - name := strings.TrimPrefix(r.URL.Path, apiPath) - if len(name) == len(r.URL.Path) { - return "", fmt.Errorf("api: patch mismatch: received '%s' - expected '%s'", r.URL.Path, apiPath) + // URL path is not guaranteed to start with a leading '/' + // Hence, we add it for a canonical representation. + if len(r.URL.Path) > 0 && r.URL.Path[0] != '/' { + r.URL.Path = "/" + r.URL.Path } - if err := verifyName(name); err != nil { - return "", err + resource, ok := strings.CutPrefix(r.URL.Path, ro.Path) + if !ok { + resp.Failf(http.StatusInternalServerError, "routing error: request '%s' handled by route '%s'", r.URL.Path, ro.Path) + return } - return name, nil -} -// patternFromRequest strips the API path from the request URL, verifies -// that the remaining path is a valid pattern, via verifyPattern, and returns -// the remaining path. -func patternFromRequest(r *http.Request, apiPath string) (string, error) { - pattern := strings.TrimPrefix(r.URL.Path, apiPath) - if len(pattern) == len(r.URL.Path) { - return "", fmt.Errorf("api: patch mismatch: received '%s' - expected '%s'", r.URL.Path, apiPath) + // Limit request bodies such that handlers can read from it securely. + if ro.MaxBody >= 0 { + if r.ContentLength < 0 || r.ContentLength > int64(ro.MaxBody) { + r.ContentLength = int64(ro.MaxBody) + } + r.Body = http.MaxBytesReader(w, r.Body, r.ContentLength) } - if err := verifyPattern(pattern); err != nil { - return "", err + + // Set a timeout. + if ro.Timeout > 0 { + if err := http.NewResponseController(w).SetWriteDeadline(time.Now().Add(ro.Timeout)); err != nil { + if errors.Is(err, http.ErrNotSupported) { + Failf(resp, http.StatusInternalServerError, "route '%s' does not support timeouts", ro.Path) + return + } + resp.Failf(http.StatusInternalServerError, "failed to set timeout on route '%s'", ro.Path) + return + } } - return pattern, nil + + req, err := ro.Auth.Authenticate(r) + if err != nil { + resp.Failr(err) + return + } + + req.Resource = resource + req.Received = received + ro.Handler.ServeAPI(resp, req) } -// verifyName reports whether the name is valid. +// A Handler responds to an API request. // -// A valid name must only contain numbers (0-9), -// letters (a-z and A-Z) and '-' as well as '_' -// characters. -func verifyName(name string) error { - const MaxLength = 80 // Some arbitrary but reasonable limit - - if name == "" { - return kes.NewError(http.StatusBadRequest, "invalid argument: name is empty") - } - if len(name) > MaxLength { - return kes.NewError(http.StatusBadRequest, "invalid argument: name is too long") - } - for _, r := range name { // Valid characters are: [ 0-9 , A-Z , a-z , - , _ ] - switch { - case r >= '0' && r <= '9': - case r >= 'A' && r <= 'Z': - case r >= 'a' && r <= 'z': - case r == '-': - case r == '_': - default: - return kes.NewError(http.StatusBadRequest, "invalid argument: name contains invalid character") - } - } - return nil +// ServeAPI should either reply to the client or fail the request and +// then return.The Response type provides methods to do so. Returning +// signals that the request is finished; it is not valid to send a +// Response or read from the Request.Body after or concurrently with +// the completion of the ServeAPI call. +// +// If ServeAPI panics, the HTTP server assumes that the effect of the +// panic was isolated to the active request. It recovers the panic, +// logs a stack trace to the server error log, and either closes the +// network connection or sends an HTTP/2 RST_STREAM, depending on the +// HTTP protocol. To abort a handler so the client sees an interrupted +// response but the server doesn't log an error, panic with the value +// http.ErrAbortHandler. +type Handler interface { + ServeAPI(*Response, *Request) +} + +// The HandlerFunc type is an adapter to allow the use of +// ordinary functions as API handlers. If f is a function +// with the appropriate signature, HandlerFunc(f) is a +// Handler that calls f. +type HandlerFunc func(*Response, *Request) + +// ServeAPI calls f with the given request and response. +func (f HandlerFunc) ServeAPI(resp *Response, req *Request) { + f(resp, req) } -// verifyPattern reports whether the pattern is valid. +// An Authenticator authenticates HTTP requests. // -// A valid pattern must only contain numbers (0-9), -// letters (a-z and A-Z) and '-', '_' as well as '*' -// characters. -func verifyPattern(pattern string) error { - const MaxLength = 80 // Some arbitrary but reasonable limit - - if pattern == "" { - return kes.NewError(http.StatusBadRequest, "invalid argument: pattern is empty") - } - if len(pattern) > MaxLength { - return kes.NewError(http.StatusBadRequest, "invalid argument: pattern is too long") - } - for _, r := range pattern { // Valid characters are: [ 0-9 , A-Z , a-z , - , _ , * ] - switch { - case r >= '0' && r <= '9': - case r >= 'A' && r <= 'Z': - case r >= 'a' && r <= 'z': - case r == '-': - case r == '_': - case r == '*': - default: - return kes.NewError(http.StatusBadRequest, "invalid argument: pattern contains invalid character") - } +// Authenticate should verify an incoming HTTP request +// and return either an authenticated API request or +// an API error. +type Authenticator interface { + Authenticate(*http.Request) (*Request, Error) +} + +// InsecureSkipVerify is an Authenticator that does not verify +// incoming HTTP requests in any way. It should only be used +// for routes that do neither wish to authenticate requests nor +// care about the client identity. +var InsecureSkipVerify Authenticator = insecureSkipVerify{} + +type insecureSkipVerify struct{} + +func (insecureSkipVerify) Authenticate(r *http.Request) (*Request, Error) { + return &Request{Request: r}, nil +} + +// Request is an authenticated HTTP request. +type Request struct { + *http.Request + + Identity kes.Identity + + Resource string + + Received time.Time +} + +// LogValue returns the requests logging representation. +func (r *Request) LogValue() slog.Value { + var identity string + if r.Identity.IsUnknown() { + identity = "" + } else { + identity = r.Identity.String() } - return nil + ip, _ := netip.ParseAddrPort(r.RemoteAddr) + return slog.GroupValue( + slog.String("method", r.Method), + slog.String("path", r.URL.Path), + slog.String("ip", ip.Addr().String()), + slog.String("identity", identity), + ) } -// Sync calls f while holding the given lock and -// releases the lock once f has been finished. -// -// Sync returns the error returned by f, if any. -func Sync(locker sync.Locker, f func() error) error { - locker.Lock() - defer locker.Unlock() +// Response is an API response. +type Response struct { + http.ResponseWriter +} + +// Reply is a shorthand for api.Reply. It sends just sends an +// HTTP status code to the client. The response body is empty. +func (r *Response) Reply(code int) { Reply(r, code) } + +// Fail is a shorthand for api.Fail. It responds to the client +// with the given status code and error message. +func (r *Response) Fail(code int, msg string) error { return Fail(r, code, msg) } - return f() +// Failf is a shorthand for api.Failf. Failf responds to the +// client with the given status code and formatted error message. +func (r *Response) Failf(code int, format string, v ...any) error { + return Failf(r, code, format, v...) } -// VSync calls f while holding the given lock and -// releases the lock once f has been finished. -// -// VSync returns the result of f and its error -// if any. -func VSync[V any](locker sync.Locker, f func() (V, error)) (V, error) { - locker.Lock() - defer locker.Unlock() +// Failr is a shorthand for api.Failr. Failr responds to the +// client with err. +func (r *Response) Failr(err Error) error { return Failr(r, err) } + +// Reply sends just sends an HTTP status code to the client. +// The response body is empty. +func Reply(r *Response, code int) { + r.Header().Set(headers.ContentLength, strconv.Itoa(0)) + r.WriteHeader(code) +} - return f() +// ReplyWith sends an HTTP status code and the data as response +// body to the client. The data format is selected automatically +// based on the response content encoding. +func ReplyWith(r *Response, code int, data any) error { + r.Header().Set(headers.ContentType, headers.ContentTypeJSON) + r.WriteHeader(code) + return json.NewEncoder(r).Encode(data) +} + +// ReadBody reads the request body into v using the +// request content encoding. +// +// ReadBody assumes that the request body is limited to a +// reasonable size. It may return an error if it cannot +// determine the request content length before decoding. +func ReadBody(r *Request, v any) error { + return json.NewDecoder(r.Body).Decode(v) } diff --git a/internal/api/api_test.go b/internal/api/api_test.go deleted file mode 100644 index 57c2a241..00000000 --- a/internal/api/api_test.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package api - -import ( - "net/http" - "net/url" - "strings" - "testing" -) - -func TestVerifyName(t *testing.T) { - for i, test := range verifyNameTests { - err := verifyName(test.Name) - if err == nil && test.ShouldFail { - t.Fatalf("Test %d should have failed", i) - } - if err != nil && !test.ShouldFail { - t.Fatalf("Test %d: name '%s' is valid but got rejected: %v", i, test.Name, err) - } - } -} - -func TestPatternName(t *testing.T) { - for i, test := range verifyPatternTests { - err := verifyPattern(test.Pattern) - if err == nil && test.ShouldFail { - t.Fatalf("Test %d should have failed", i) - } - if err != nil && !test.ShouldFail { - t.Fatalf("Test %d: pattern '%s' is valid but got rejected: %v", i, test.Pattern, err) - } - } -} - -func TestNameFromRequest(t *testing.T) { - for i, test := range nameFromRequestTests { - url, err := url.Parse(test.URL) - if err != nil { - t.Fatalf("Test %d: failed to parse URL '%s': %v", i, test.URL, err) - } - - name, err := nameFromRequest(&http.Request{URL: url}, test.Path) - if err == nil && test.ShouldFail { - t.Fatalf("Test %d should have failed", i) - } - if err != nil && !test.ShouldFail { - t.Fatalf("Test %d: failed to get name from request: %v", i, err) - } - if err == nil && name != test.Name { - t.Fatalf("Test %d: got '%s' - want '%s'", i, name, test.Name) - } - } -} - -func TestPatternFromRequest(t *testing.T) { - for i, test := range patternFromRequestTests { - url, err := url.Parse(test.URL) - if err != nil { - t.Fatalf("Test %d: failed to parse URL '%s': %v", i, test.URL, err) - } - - pattern, err := patternFromRequest(&http.Request{URL: url}, test.Path) - if err == nil && test.ShouldFail { - t.Fatalf("Test %d should have failed", i) - } - if err != nil && !test.ShouldFail { - t.Fatalf("Test %d: failed to get name from request: %v", i, err) - } - if err == nil && pattern != test.Pattern { - t.Fatalf("Test %d: got '%s' - want '%s'", i, pattern, test.Pattern) - } - } -} - -var ( - verifyNameTests = []struct { - Name string - ShouldFail bool - }{ - {Name: "my-key"}, // 0 - {Name: "abc123"}, // 1 - {Name: "0"}, // 2 - {Name: "123ABC321"}, // 3 - {Name: "_-___---_"}, // 4 - {Name: "_0"}, // 5 - {Name: "0-Z"}, // 6 - {Name: "my_key-0"}, // 7 - - {Name: "", ShouldFail: true}, // 8 - {Name: "my.key", ShouldFail: true}, // 9 - {Name: "key/", ShouldFail: true}, // 10 - {Name: "", ShouldFail: true}, // 11 - {Name: "☰", ShouldFail: true}, // 12 - {Name: "hel maxSize { + size = maxSize + } + body := mem.LimitReader(resp.Body, size) + + switch resp.Header.Get(headers.ContentType) { + case headers.ContentTypeHTML, headers.ContentTypeText: + var sb strings.Builder + if _, err := io.Copy(&sb, body); err != nil { + return "", err + } + return sb.String(), nil + default: + type ErrResponse struct { + Message string `json:"error"` + } + var response ErrResponse + if err := json.NewDecoder(body).Decode(&response); err != nil { + return "", err + } + return response.Message, nil + } +} + +type codeError struct { + code int + msg string +} + +func (e *codeError) Error() string { return e.msg } + +func (e *codeError) Status() int { return e.code } diff --git a/internal/api/health.go b/internal/api/health.go deleted file mode 100644 index 875cc53c..00000000 --- a/internal/api/health.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package api - -import ( - "net/http" - "time" - - "github.com/minio/kes-go" - "github.com/minio/kes/internal/auth" - "github.com/minio/kes/kv" -) - -func edgeReady(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/ready" - MaxBody int64 - Timeout = 15 * time.Second - Verify = true - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - Verify = !c.InsecureSkipAuth - } - var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - if err := auth.VerifyRequest(r, config.Policies, config.Identities); Verify && err != nil { - Fail(w, err) - return - } - - _, err := config.Keys.Status(r.Context()) - if _, ok := kv.IsUnreachable(err); ok { - Fail(w, kes.NewError(http.StatusGatewayTimeout, err.Error())) - return - } - if err != nil { - Fail(w, kes.NewError(http.StatusBadGateway, err.Error())) - return - } - w.WriteHeader(http.StatusOK) - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Verify: Verify, - Timeout: Timeout, - Handler: handler, - } -} diff --git a/internal/api/identity.go b/internal/api/identity.go deleted file mode 100644 index 84f07a1d..00000000 --- a/internal/api/identity.go +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package api - -import ( - "encoding/json" - "net/http" - "path" - "time" - - "github.com/minio/kes-go" - "github.com/minio/kes/internal/audit" - "github.com/minio/kes/internal/auth" -) - -func edgeDescribeIdentity(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/identity/describe/" - MaxBody int64 - Timeout = 15 * time.Second - Verify = true - ContentType = "application/json" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Response struct { - IsAdmin bool `json:"admin,omitempty"` - Policy string `json:"policy"` - CreatedAt time.Time `json:"created_at,omitempty"` - CreatedBy kes.Identity `json:"created_by,omitempty"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - name, err := nameFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - - info, err := config.Identities.Get(r.Context(), kes.Identity(name)) - if err != nil { - return err - } - - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(Response{ - IsAdmin: info.IsAdmin, - Policy: info.Policy, - CreatedAt: info.CreatedAt, - CreatedBy: info.CreatedBy, - }) - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeSelfDescribeIdentity(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/identity/self/describe" - MaxBody int64 - Timeout = 15 * time.Second - Verify = false - ContentType = "application/json" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Response struct { - Identity kes.Identity `json:"identity"` - IsAdmin bool `json:"admin,omitempty"` - CreatedAt time.Time `json:"created_at,omitempty"` - CreatedBy kes.Identity `json:"created_by,omitempty"` - - Policy string `json:"policy,omitempty"` - Allow map[string]kes.Rule `json:"allow,omitempty"` - Deny map[string]kes.Rule `json:"deny,omitempty"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - identity := auth.Identify(r) - info, err := config.Identities.Get(r.Context(), identity) - if err != nil { - return err - } - policy := new(auth.Policy) - if !info.IsAdmin { - policy, err = config.Policies.Get(r.Context(), info.Policy) - if err != nil { - return err - } - } - - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(Response{ - Identity: identity, - IsAdmin: info.IsAdmin, - CreatedAt: info.CreatedAt, - CreatedBy: info.CreatedBy, - Policy: info.Policy, - Allow: policy.Allow, - Deny: policy.Deny, - }) - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeListIdentity(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/identity/list/" - MaxBody int64 - Timeout = 15 * time.Second - Verify = true - ContentType = "application/x-ndjson" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Response struct { - Identity kes.Identity `json:"identity"` - IsAdmin bool `json:"admin"` - Policy string `json:"policy"` - CreatedAt time.Time `json:"created_at,omitempty"` - CreatedBy kes.Identity `json:"created_by,omitempty"` - - Err string `json:"error,omitempty"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - pattern, err := patternFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - - iterator, err := config.Identities.List(r.Context()) - if err != nil { - return err - } - defer iterator.Close() - - var ( - encoder = json.NewEncoder(w) - hasWritten bool - ) - for iterator.Next() { - if ok, _ := path.Match(pattern, iterator.Identity().String()); !ok { - continue - } - if !hasWritten { - w.Header().Set("Content-Type", ContentType) - } - hasWritten = true - - info, err := config.Identities.Get(r.Context(), iterator.Identity()) - if err != nil { - encoder.Encode(Response{Err: err.Error()}) - return nil - } - - if err = encoder.Encode(Response{ - Identity: iterator.Identity(), - IsAdmin: info.IsAdmin, - Policy: info.Policy, - CreatedAt: info.CreatedAt, - CreatedBy: info.CreatedBy, - }); err != nil { - return nil - } - } - if err = iterator.Close(); err != nil { - if hasWritten { - encoder.Encode(Response{Err: err.Error()}) - return nil - } - return err - } - if !hasWritten { - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - } - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} diff --git a/internal/api/key.go b/internal/api/key.go deleted file mode 100644 index d49ec205..00000000 --- a/internal/api/key.go +++ /dev/null @@ -1,555 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package api - -import ( - "crypto/rand" - "encoding/json" - "net/http" - "path" - "time" - - "aead.dev/mem" - "github.com/minio/kes-go" - "github.com/minio/kes/internal/audit" - "github.com/minio/kes/internal/auth" - "github.com/minio/kes/internal/cpu" - "github.com/minio/kes/internal/fips" - "github.com/minio/kes/internal/key" -) - -func edgeCreateKey(config *EdgeRouterConfig) API { - var ( - Method = http.MethodPost - APIPath = "/v1/key/create/" - MaxBody int64 - Timeout = 15 * time.Second - Verify = true - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - name, err := nameFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - - var algorithm kes.KeyAlgorithm - if fips.Enabled || cpu.HasAESGCM() { - algorithm = kes.AES256 - } else { - algorithm = kes.ChaCha20 - } - - key, err := key.Random(algorithm, auth.Identify(r)) - if err != nil { - return err - } - if err = config.Keys.Create(r.Context(), name, key); err != nil { - return err - } - - w.WriteHeader(http.StatusOK) - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeImportKey(config *EdgeRouterConfig) API { - var ( - Method = http.MethodPost - APIPath = "/v1/key/import/" - MaxBody = 1 * mem.MiB - Timeout = 15 * time.Second - Verify = true - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Request struct { - Bytes []byte `json:"key"` - Algorithm string `json:"cipher"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - name, err := nameFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - - var req Request - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - return kes.NewError(http.StatusBadRequest, err.Error()) - } - - var algorithm kes.KeyAlgorithm - if err := algorithm.UnmarshalText([]byte(req.Algorithm)); err != nil { - return kes.NewError(http.StatusBadRequest, err.Error()) - } - if len(req.Bytes) != key.Len(algorithm) { - return kes.NewError(http.StatusBadRequest, "invalid key size") - } - key, err := key.New(algorithm, req.Bytes, auth.Identify(r)) - if err != nil { - return err - } - if err = config.Keys.Create(r.Context(), name, key); err != nil { - return err - } - - w.WriteHeader(http.StatusOK) - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: int64(MaxBody), - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeDescribeKey(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/key/describe/" - MaxBody int64 - Timeout = 15 * time.Second - Verify = true - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Response struct { - Name string `json:"name"` - ID string `json:"id,omitempty"` - Algorithm kes.KeyAlgorithm `json:"algorithm,omitempty"` - CreatedAt time.Time `json:"created_at,omitempty"` - CreatedBy kes.Identity `json:"created_by,omitempty"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - name, err := nameFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - key, err := config.Keys.Get(r.Context(), name) - if err != nil { - return err - } - - w.Header().Set("Content-Length", "application/json") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(Response{ - Name: name, - ID: key.ID(), - Algorithm: key.Algorithm(), - CreatedAt: key.CreatedAt(), - CreatedBy: key.CreatedBy(), - }) - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeDeleteKey(config *EdgeRouterConfig) API { - var ( - Method = http.MethodDelete - APIPath = "/v1/key/delete/" - MaxBody int64 - Timeout = 15 * time.Second - Verify = true - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - name, err := nameFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - if err := config.Keys.Delete(r.Context(), name); err != nil { - return err - } - - w.WriteHeader(http.StatusOK) - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeGenerateKey(config *EdgeRouterConfig) API { - var ( - Method = http.MethodPost - APIPath = "/v1/key/generate/" - MaxBody = 1 * mem.MiB - Timeout = 15 * time.Second - Verify = true - ContentType = "application/json" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Request struct { - Context []byte `json:"context"` // optional - } - type Response struct { - Plaintext []byte `json:"plaintext"` - Ciphertext []byte `json:"ciphertext"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - name, err := nameFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - - var req Request - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - return kes.NewError(http.StatusBadRequest, err.Error()) - } - key, err := config.Keys.Get(r.Context(), name) - if err != nil { - return err - } - dataKey := make([]byte, 32) - if _, err = rand.Read(dataKey); err != nil { - return err - } - ciphertext, err := key.Wrap(dataKey, req.Context) - if err != nil { - return err - } - - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(Response{ - Plaintext: dataKey, - Ciphertext: ciphertext, - }) - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: int64(MaxBody), - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeEncryptKey(config *EdgeRouterConfig) API { - var ( - Method = http.MethodPost - APIPath = "/v1/key/encrypt/" - MaxBody = int64(1 * mem.MiB) - Timeout = 15 * time.Second - Verify = true - ContentType = "application/json" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Request struct { - Plaintext []byte `json:"plaintext"` - Context []byte `json:"context"` // optional - } - type Response struct { - Ciphertext []byte `json:"ciphertext"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - name, err := nameFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - - var req Request - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - return kes.NewError(http.StatusBadRequest, err.Error()) - } - key, err := config.Keys.Get(r.Context(), name) - if err != nil { - return err - } - ciphertext, err := key.Wrap(req.Plaintext, req.Context) - if err != nil { - return err - } - - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(Response{ - Ciphertext: ciphertext, - }) - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeDecryptKey(config *EdgeRouterConfig) API { - var ( - Method = http.MethodPost - APIPath = "/v1/key/decrypt/" - MaxBody = int64(1 * mem.MiB) - Timeout = 15 * time.Second - Verify = true - ContentType = "application/json" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Request struct { - Ciphertext []byte `json:"ciphertext"` - Context []byte `json:"context"` // optional - } - type Response struct { - Plaintext []byte `json:"plaintext"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - name, err := nameFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - - var req Request - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - return err - } - key, err := config.Keys.Get(r.Context(), name) - if err != nil { - return err - } - plaintext, err := key.Unwrap(req.Ciphertext, req.Context) - if err != nil { - return err - } - - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(Response{ - Plaintext: plaintext, - }) - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeBulkDecryptKey(config *EdgeRouterConfig) API { - var ( - Method = http.MethodPost - APIPath = "/v1/key/bulk/decrypt/" - MaxBody = int64(1 * mem.MiB) - Timeout = 15 * time.Second - Verify = true - ContentType = "application/json" - MaxRequests = 1000 // For now, we limit the number of decryption requests in a single API call to 1000. - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Request struct { - Ciphertext []byte `json:"ciphertext"` - Context []byte `json:"context"` // optional - } - type Response struct { - Plaintext []byte `json:"plaintext"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - name, err := nameFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - - key, err := config.Keys.Get(r.Context(), name) - if err != nil { - return err - } - var ( - requests []Request - responses []Response - ) - if err = json.NewDecoder(r.Body).Decode(&requests); err != nil { - return kes.NewError(http.StatusBadRequest, err.Error()) - } - if len(requests) > MaxRequests { - return kes.NewError(http.StatusBadRequest, "too many ciphertexts") - } - responses = make([]Response, 0, len(requests)) - for _, req := range requests { - plaintext, err := key.Unwrap(req.Ciphertext, req.Context) - if err != nil { - return err - } - responses = append(responses, Response{ - Plaintext: plaintext, - }) - } - - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(responses) - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeListKey(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/key/list/" - MaxBody int64 - Timeout = 15 * time.Second - Verify = true - ContentType = "application/x-ndjson" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Response struct { - Name string `json:"name,omitempty"` - Err string `json:"error,omitempty"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - pattern, err := patternFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - - iterator, err := config.Keys.List(r.Context()) - if err != nil { - return err - } - defer iterator.Close() - - var ( - hasWritten bool - encoder = json.NewEncoder(w) - ) - for { - name, ok := iterator.Next() - if !ok { - break - } - if ok, _ = path.Match(pattern, name); !ok || name == "" { - continue - } - if !hasWritten { - w.Header().Set("Content-Type", ContentType) - } - hasWritten = true - - if err = encoder.Encode(Response{Name: name}); err != nil { - return nil - } - } - if err = iterator.Close(); err != nil { - if hasWritten { - encoder.Encode(Response{Err: err.Error()}) - return nil - } - return err - } - if !hasWritten { - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - } - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} diff --git a/internal/api/log.go b/internal/api/log.go deleted file mode 100644 index edcf3963..00000000 --- a/internal/api/log.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package api - -import ( - "net/http" - "time" - - "github.com/minio/kes/internal/auth" - "github.com/minio/kes/internal/https" - "github.com/minio/kes/internal/log" -) - -func edgeErrorLog(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/log/error" - MaxBody int64 - Timeout = 0 * time.Second // No timeout - Verify = true - ContentType = "application/x-ndjson" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - Fail(w, err) - return - } - r.Body = http.MaxBytesReader(w, r.Body, MaxBody) - - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - - out := log.NewErrEncoder(https.FlushOnWrite(w)) - config.ErrorLog.Add(out) - defer config.ErrorLog.Remove(out) - - <-r.Context().Done() // Wait for the client to close the connection - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(handler)), - } -} - -func edgeAuditLog(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/log/audit" - MaxBody int64 - Timeout = 0 * time.Second // No timeout - Verify = true - ContentType = "application/x-ndjson" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - Fail(w, err) - return - } - - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - - out := https.FlushOnWrite(w) - config.AuditLog.Add(out) - defer config.AuditLog.Remove(out) - - <-r.Context().Done() // Wait for the client to close the connection - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(handler)), - } -} diff --git a/internal/api/metric.go b/internal/api/metric.go deleted file mode 100644 index 71b368b6..00000000 --- a/internal/api/metric.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package api - -import ( - "net/http" - "time" - - "github.com/minio/kes/internal/auth" - "github.com/prometheus/common/expfmt" -) - -func edgeMetrics(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/metrics" - MaxBody int64 - Verify = true - Timeout = 15 * time.Second - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - Verify = !c.InsecureSkipAuth - } - var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - if err := auth.VerifyRequest(r, config.Policies, config.Identities); Verify && err != nil { - Fail(w, err) - return - } - - contentType := expfmt.Negotiate(r.Header) - w.Header().Set("Content-Type", string(contentType)) - w.WriteHeader(http.StatusOK) - - config.Metrics.EncodeTo(expfmt.NewEncoder(w, contentType)) - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: handler, - } -} diff --git a/internal/api/multicast.go b/internal/api/multicast.go new file mode 100644 index 00000000..b75276b6 --- /dev/null +++ b/internal/api/multicast.go @@ -0,0 +1,153 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package api + +import ( + "encoding/json" + "io" + "net/http" + "slices" + "sync/atomic" +) + +// Multicast is a one-to-many io.Writer. It is similar +// to io.MultiWriter but writers can be added and removed +// dynamically. A Multicast may be modified by multiple +// go routines concurrently. +// +// Its zero value is an empty group of io.Writers and +// ready for use. +type Multicast struct { + group atomic.Pointer[[]io.Writer] +} + +// Num returns how many connections are part of this Multicast. +func (m *Multicast) Num() int { + if p := m.group.Load(); p != nil { + return len(*p) + } + return 0 +} + +// Add adds w to m. Future writes to m will also reach w. +// If w is already part of m, Add does nothing. +func (m *Multicast) Add(w io.Writer) { + if m == nil { + return + } + + for { + old := m.group.Load() + if old == nil && m.group.CompareAndSwap(nil, &[]io.Writer{w}) { + return + } + if slices.Contains(*old, w) { // avoid adding an io.Writer twice + return + } + + group := make([]io.Writer, 0, len(*old)+1) + group = append(group, w) + group = append(group, *old...) + if m.group.CompareAndSwap(old, &group) { + return + } + } +} + +// Remove removes w from m. Future writes to m will no longer +// reach w. +func (m *Multicast) Remove(w io.Writer) { + if w == nil { + return + } + + for { + old := m.group.Load() + if old == nil || len(*old) == 0 || !slices.Contains(*old, w) { + return + } + + group := make([]io.Writer, 0, len(*old)-1) + for _, wr := range *old { + if wr != w { + group = append(group, wr) + } + } + if m.group.CompareAndSwap(old, &group) { + return + } + } +} + +// Write writes p to all io.Writers that are currently part of m. +// It returns the first error encountered, if any, but writes to +// all io.Writers before returning. +func (m *Multicast) Write(p []byte) (n int, err error) { + ptr := m.group.Load() + if ptr == nil { + return 0, nil + } + group := *ptr + if len(group) == 0 { + return 0, nil + } + + for _, w := range group { + nn, wErr := w.Write(p) + if wErr == nil && nn < len(p) { + wErr = io.ErrShortWrite + } + if err == nil && wErr != nil { + err = wErr + n = nn + } + } + if n == 0 && err == nil { + n = len(p) + } + return n, err +} + +// LogWriter wraps an io.Writer and encodes each +// write operation as ErrorLogEvent. +// +// It's intended to be used as adapter to send +// API error logs to an http.ResponseWriter. +type LogWriter struct { + encoder *json.Encoder + flusher http.Flusher +} + +// NewLogWriter returns a new LogWriter wrapping w. +func NewLogWriter(w io.Writer) *LogWriter { + flusher, _ := w.(http.Flusher) + return &LogWriter{ + encoder: json.NewEncoder(w), + flusher: flusher, + } +} + +// Write encodes p as ErrorLogEvent and +// writes it to the underlying io.Writer. +func (w *LogWriter) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + + n := len(p) + if p[n-1] == '\n' { // Remove trailing newline added by logger + p = p[:n-1] + } + + if err := w.encoder.Encode(ErrorLogEvent{ + Message: string(p), + }); err != nil { + return 0, err + } + if w.flusher != nil { + w.flusher.Flush() + } + return n, nil +} diff --git a/internal/api/policy.go b/internal/api/policy.go deleted file mode 100644 index 4f8ce517..00000000 --- a/internal/api/policy.go +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package api - -import ( - "encoding/json" - "net/http" - "path" - "time" - - "github.com/minio/kes-go" - "github.com/minio/kes/internal/audit" - "github.com/minio/kes/internal/auth" -) - -func edgeDescribePolicy(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/policy/describe/" - MaxBody int64 - Timeout = 15 * time.Second - Verify = true - ContentType = "application/json" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Response struct { - CreatedAt time.Time `json:"created_at,omitempty"` - CreatedBy kes.Identity `json:"created_by,omitempty"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - name, err := nameFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - - policy, err := config.Policies.Get(r.Context(), name) - if err != nil { - return err - } - - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(Response{ - CreatedAt: policy.CreatedAt, - CreatedBy: policy.CreatedBy, - }) - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeReadPolicy(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/policy/read/" - MaxBody int64 - Timeout = 15 * time.Second - Verify = true - ContentType = "application/json" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Response struct { - Allow map[string]kes.Rule `json:"allow,omitempty"` - Deny map[string]kes.Rule `json:"deny,omitempty"` - CreatedAt time.Time `json:"created_at,omitempty"` - CreatedBy kes.Identity `json:"created_by,omitempty"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - name, err := nameFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - - policy, err := config.Policies.Get(r.Context(), name) - if err != nil { - return err - } - - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(Response{ - Allow: policy.Allow, - Deny: policy.Deny, - CreatedAt: policy.CreatedAt, - CreatedBy: policy.CreatedBy, - }) - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeListPolicy(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/policy/list/" - MaxBody int64 - Timeout = 15 * time.Second - Verify = true - ContentType = "application/x-ndjson" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - } - type Response struct { - Name string `json:"name"` - CreatedAt time.Time `json:"created_at,omitempty"` - CreatedBy kes.Identity `json:"created_by,omitempty"` - - Err string `json:"error,omitempty"` - } - var handler HandlerFunc = func(w http.ResponseWriter, r *http.Request) error { - pattern, err := patternFromRequest(r, APIPath) - if err != nil { - return err - } - if err := auth.VerifyRequest(r, config.Policies, config.Identities); err != nil { - return err - } - - iterator, err := config.Policies.List(r.Context()) - if err != nil { - return err - } - defer iterator.Close() - - var hasWritten bool - encoder := json.NewEncoder(w) - w.Header().Set("Content-Type", ContentType) - for iterator.Next() { - if ok, _ := path.Match(pattern, iterator.Name()); !ok { - continue - } - if !hasWritten { - w.Header().Set("Content-Type", ContentType) - } - hasWritten = true - - policy, err := config.Policies.Get(r.Context(), iterator.Name()) - if err != nil { - encoder.Encode(Response{Err: err.Error()}) - return nil - } - if err = encoder.Encode(Response{ - Name: iterator.Name(), - CreatedAt: policy.CreatedAt, - CreatedBy: policy.CreatedBy, - }); err != nil { - return nil - } - } - if err = iterator.Close(); err != nil { - if hasWritten { - encoder.Encode(Response{Err: err.Error()}) - return nil - } - return err - } - if !hasWritten { - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - } - return nil - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} diff --git a/internal/api/proxy.go b/internal/api/proxy.go deleted file mode 100644 index 128eecf9..00000000 --- a/internal/api/proxy.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package api - -import ( - "net/http" - - "github.com/minio/kes/internal/auth" -) - -func proxy(proxy *auth.TLSProxy, f http.Handler) http.Handler { - if proxy == nil { - return f - } - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := proxy.Verify(r); err != nil { - Fail(w, err) - return - } - f.ServeHTTP(w, r) - }) -} diff --git a/internal/api/request.go b/internal/api/request.go new file mode 100644 index 00000000..5086080d --- /dev/null +++ b/internal/api/request.go @@ -0,0 +1,28 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package api + +// ImportKeyRequest is the request sent by clients when calling the ImportKey API. +type ImportKeyRequest struct { + Bytes []byte `json:"key"` + Cipher string `json:"cipher"` +} + +// EncryptKeyRequest is the request sent by clients when calling the EncryptKey API. +type EncryptKeyRequest struct { + Plaintext []byte `json:"plaintext"` + Context []byte `json:"context"` // optional +} + +// GenerateKeyRequest is the request sent by clients when calling the GenerateKey API. +type GenerateKeyRequest struct { + Context []byte `json:"context"` // optional +} + +// DecryptKeyRequest is the request sent by clients when calling the DecryptKey API. +type DecryptKeyRequest struct { + Ciphertext []byte `json:"ciphertext"` + Context []byte `json:"context"` // optional +} diff --git a/internal/api/response.go b/internal/api/response.go new file mode 100644 index 00000000..7b517edb --- /dev/null +++ b/internal/api/response.go @@ -0,0 +1,144 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package api + +import ( + "time" +) + +// VersionResponse is the response sent to clients by the Version API. +type VersionResponse struct { + Version string `json:"version"` + Commit string `json:"commit"` +} + +// StatusResponse is the response sent to clients by the Status API. +type StatusResponse struct { + Version string `json:"version"` + OS string `json:"os"` + Arch string `json:"arch"` + UpTime uint64 `json:"uptime"` // in seconds + CPUs int `json:"num_cpu"` + UsableCPUs int `json:"num_cpu_used"` + HeapAlloc uint64 `json:"mem_heap_used"` + StackAlloc uint64 `json:"mem_stack_used"` + + KeyStoreLatency int64 `json:"keystore_latency,omitempty"` // In microseconds + KeyStoreUnavailable bool `json:"keystore_unavailable,omitempty"` + KeyStoreUnreachable bool `json:"keystore_unreachable,omitempty"` +} + +// DescribeRouteResponse describes a single API route. It is part of +// a List API response. +type DescribeRouteResponse struct { + Method string `json:"method"` + Path string `json:"path"` + MaxBody int64 `json:"max_body"` + Timeout int64 `json:"timeout"` // in seconds +} + +// ListAPIsResponse is the response sent to clients by the List APIs API. +type ListAPIsResponse []DescribeRouteResponse + +// DescribeKeyResponse is the response sent to clients by the DescribeKey API. +type DescribeKeyResponse struct { + Name string `json:"name"` + Algorithm string `json:"algorithm,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` +} + +// ListKeysResponse is the response sent to clients by the ListKeys API. +type ListKeysResponse struct { + Names []string `json:"names"` + ContinueAt string `json:"continue_at,omitempty"` +} + +// EncryptKeyResponse is the response sent to clients by the EncryptKey API. +type EncryptKeyResponse struct { + Ciphertext []byte `json:"ciphertext"` +} + +// GenerateKeyResponse is the response sent to clients by the GenerateKey API. +type GenerateKeyResponse struct { + Plaintext []byte `json:"plaintext"` + Ciphertext []byte `json:"ciphertext"` +} + +// DecryptKeyResponse is the response sent to clients by the DecryptKey API. +type DecryptKeyResponse struct { + Plaintext []byte `json:"plaintext"` +} + +// ReadPolicyResponse is the response sent to clients by the ReadPolicy API. +type ReadPolicyResponse struct { + Name string `json:"name"` + Allow map[string]struct{} `json:"allow,omitempty"` + Deny map[string]struct{} `json:"deny,omitempty"` + CreatedAt time.Time `json:"created_at"` + CreatedBy string `json:"created_by"` +} + +// DescribePolicyResponse is the response sent to clients by the DescribePolicy API. +type DescribePolicyResponse struct { + Name string `json:"name"` + CreatedAt time.Time `json:"created_at"` + CreatedBy string `json:"created_by"` +} + +// ListPoliciesResponse is the response sent to clients by the ListPolicies API. +type ListPoliciesResponse struct { + Names []string `json:"names"` + ContinueAt string `json:"continue_at"` +} + +// DescribeIdentityResponse is the response sent to clients by the DescribeIdentity API. +type DescribeIdentityResponse struct { + IsAdmin bool `json:"admin,omitempty"` + Policy string `json:"policy,omitempty"` + CreatedAt time.Time `json:"created_at"` + CreatedBy string `json:"created_by,omitempty"` +} + +// ListIdentitiesResponse is the response sent to clients by the ListIdentities API. +type ListIdentitiesResponse struct { + Identities []string `json:"identities"` + ContinueAt string `json:"continue_at"` +} + +// SelfDescribeIdentityResponse is the response sent to clients by the SelfDescribeIdentity API. +type SelfDescribeIdentityResponse struct { + Identity string `json:"identity"` + IsAdmin bool `json:"admin,omitempty"` + CreatedAt time.Time `json:"created_at"` + CreatedBy string `json:"created_by,omitempty"` + + Policy *ReadPolicyResponse `json:"policy,omitempty"` +} + +// AuditLogEvent is sent to clients (as stream of events) when they subscribe to the AuditLog API. +type AuditLogEvent struct { + Time time.Time `json:"time"` + Request AuditLogRequest `json:"request"` + Response AuditLogResponse `json:"response"` +} + +// AuditLogRequest describes a client request in an AuditLogEvent. +type AuditLogRequest struct { + IP string `json:"ip,omitempty"` + APIPath string `json:"path"` + Identity string `json:"identity,omitempty"` +} + +// AuditLogResponse describes a server response in an AuditLogEvent. +type AuditLogResponse struct { + StatusCode int `json:"code"` + Time int64 `json:"time"` // In microseconds +} + +// ErrorLogEvent is sent to clients (as stream of events) when they subscribe to the ErrorLog API. +type ErrorLogEvent struct { + Message string `json:"message"` +} diff --git a/internal/api/router.go b/internal/api/router.go deleted file mode 100644 index 82d1eca6..00000000 --- a/internal/api/router.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package api - -import ( - "net/http" - "strings" - "time" - - "github.com/minio/kes-go" - "github.com/minio/kes/internal/auth" - "github.com/minio/kes/internal/keystore" - "github.com/minio/kes/internal/log" - "github.com/minio/kes/internal/metric" -) - -// EdgeRouterConfig is a structure containing the -// API configuration for a KES edge server. -type EdgeRouterConfig struct { - Keys *keystore.Cache - - Policies auth.PolicySet - - Identities auth.IdentitySet - - Metrics *metric.Metrics - - Proxy *auth.TLSProxy - - APIConfig map[string]Config - - AuditLog *log.Logger - - ErrorLog *log.Logger -} - -// NewEdgeRouter returns a new API Router for a KES edge -// server with the given configuration. -func NewEdgeRouter(config *EdgeRouterConfig) *Router { - r := &Router{ - handler: http.NewServeMux(), - } - - r.api = append(r.api, edgeVersion(config)) - r.api = append(r.api, edgeReady(config)) - r.api = append(r.api, edgeStatus(config)) - r.api = append(r.api, edgeMetrics(config)) - r.api = append(r.api, edgeListAPI(r, config)) - - r.api = append(r.api, edgeCreateKey(config)) - r.api = append(r.api, edgeImportKey(config)) - r.api = append(r.api, edgeDescribeKey(config)) - r.api = append(r.api, edgeDeleteKey(config)) - r.api = append(r.api, edgeListKey(config)) - r.api = append(r.api, edgeGenerateKey(config)) - r.api = append(r.api, edgeEncryptKey(config)) - r.api = append(r.api, edgeDecryptKey(config)) - r.api = append(r.api, edgeBulkDecryptKey(config)) - - r.api = append(r.api, edgeDescribePolicy(config)) - r.api = append(r.api, edgeReadPolicy(config)) - r.api = append(r.api, edgeListPolicy(config)) - - r.api = append(r.api, edgeDescribeIdentity(config)) - r.api = append(r.api, edgeSelfDescribeIdentity(config)) - r.api = append(r.api, edgeListIdentity(config)) - - r.api = append(r.api, edgeErrorLog(config)) - r.api = append(r.api, edgeAuditLog(config)) - - for _, a := range r.api { - r.handler.Handle(a.Path, proxy(config.Proxy, a)) - } - r.handler.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.NewResponseController(w).SetWriteDeadline(time.Now().Add(10 * time.Second)) - Fail(w, kes.NewError(http.StatusNotImplemented, "not implemented")) - })) - return r -} - -// Router is an HTTP handler that implements the KES API. -// -// It routes incoming HTTP requests and invokes the -// corresponding API handlers. -type Router struct { - handler *http.ServeMux - api []API -} - -// ServeHTTP dispatches the request to the API handler whose -// pattern most matches the request URL. -func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { - if !strings.HasPrefix(req.URL.Path, "/") { // Ensure URL paths start with a '/' - req.URL.Path = "/" + req.URL.Path - } - r.handler.ServeHTTP(w, req) -} - -// API returns a list of APIs provided by the Router. -func (r *Router) API() []API { return r.api } - -// A HandlerFunc is an adapter that allows the use of -// ordinary functions as HTTP handlers. -// -// In contrast to the http.HandlerFunc type, HandlerFunc -// returns an error. Hence, a function f, with the appropriate -// signature, can simply return an error in case of failed -// operation. If f returns a non-nil error, HandlerFunc(f) -// sends an error response to the client. -type HandlerFunc func(http.ResponseWriter, *http.Request) error - -// ServeHTTP calls f(w, r). If f returns a non-nil error -// ServeHTTP sends an error response to the client. -func (f HandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if err := f(w, r); err != nil { - Fail(w, err) - } -} diff --git a/internal/api/status.go b/internal/api/status.go deleted file mode 100644 index 82b34a87..00000000 --- a/internal/api/status.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package api - -import ( - "encoding/json" - "net/http" - "runtime" - "time" - - "github.com/minio/kes/internal/audit" - "github.com/minio/kes/internal/auth" - "github.com/minio/kes/internal/sys" - "github.com/minio/kes/kv" -) - -func edgeStatus(config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/status" - MaxBody int64 - Timeout = 15 * time.Second - Verify = true - ContentType = "application/json" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - Verify = !c.InsecureSkipAuth - } - type Response struct { - Version string `json:"version"` - OS string `json:"os"` - Arch string `json:"arch"` - UpTime time.Duration `json:"uptime"` - CPUs int `json:"num_cpu"` - UsableCPUs int `json:"num_cpu_used"` - HeapAlloc uint64 `json:"mem_heap_used"` - StackAlloc uint64 `json:"mem_stack_used"` - - KeyStoreLatency int64 `json:"keystore_latency,omitempty"` - KeyStoreUnavailable bool `json:"keystore_unavailable,omitempty"` - KeyStoreUnreachable bool `json:"keystore_unreachable,omitempty"` - } - - startTime := time.Now().UTC() - var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - if err := auth.VerifyRequest(r, config.Policies, config.Identities); Verify && err != nil { - Fail(w, err) - return - } - - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - - response := Response{ - Version: sys.BinaryInfo().Version, - OS: runtime.GOOS, - Arch: runtime.GOARCH, - UpTime: time.Since(startTime).Round(time.Second), - - CPUs: runtime.NumCPU(), - UsableCPUs: runtime.GOMAXPROCS(0), - HeapAlloc: memStats.HeapAlloc, - StackAlloc: memStats.StackSys, - } - - state, err := config.Keys.Status(r.Context()) - if err != nil { - response.KeyStoreUnavailable = true - _, response.KeyStoreUnreachable = kv.IsUnreachable(err) - } else { - latency := state.Latency.Round(time.Millisecond) - if latency == 0 { // Make sure we actually send a latency even if the key store respond time is < 1ms. - latency = 1 * time.Millisecond - } - response.KeyStoreLatency = latency.Milliseconds() - } - - w.Header().Set("Content-Type", ContentType) - json.NewEncoder(w).Encode(response) - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Verify: Verify, - Timeout: Timeout, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} - -func edgeListAPI(router *Router, config *EdgeRouterConfig) API { - var ( - Method = http.MethodGet - APIPath = "/v1/api" - MaxBody int64 - Timeout = 15 * time.Second - Verify = true - ContentType = "application/json" - ) - if c, ok := config.APIConfig[APIPath]; ok { - if c.Timeout > 0 { - Timeout = c.Timeout - } - Verify = !c.InsecureSkipAuth - } - type Response struct { - Method string `json:"method"` - Path string `json:"path"` - MaxBody int64 `json:"max_body"` - Timeout int64 `json:"timeout"` // Timeout in seconds - Verify bool `json:"verify_auth"` // Whether the API requires authentication - } - var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - if err := auth.VerifyRequest(r, config.Policies, config.Identities); Verify && err != nil { - Fail(w, err) - return - } - - apis := router.API() - responses := make([]Response, 0, len(apis)) - for _, api := range apis { - responses = append(responses, Response{ - Method: api.Method, - Path: api.Path, - MaxBody: api.MaxBody, - Timeout: int64(api.Timeout.Truncate(time.Second).Seconds()), - Verify: api.Verify, - }) - } - w.Header().Set("Content-Type", ContentType) - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(responses) - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} diff --git a/internal/api/version.go b/internal/api/version.go deleted file mode 100644 index f089f2e8..00000000 --- a/internal/api/version.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package api - -import ( - "encoding/json" - "net/http" - "time" - - "github.com/minio/kes/internal/audit" - "github.com/minio/kes/internal/sys" -) - -func edgeVersion(config *EdgeRouterConfig) API { - const ( - Method = http.MethodGet - APIPath = "/version" - MaxBody = 0 - Timeout = 15 * time.Second - Verify = false - ) - type Response struct { - Version string `json:"version"` - Commit string `json:"commit"` - } - var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { - json.NewEncoder(w).Encode(Response{ - Version: sys.BinaryInfo().Version, - Commit: sys.BinaryInfo().CommitID, - }) - } - return API{ - Method: Method, - Path: APIPath, - MaxBody: MaxBody, - Timeout: Timeout, - Verify: Verify, - Handler: config.Metrics.Count(config.Metrics.Latency(audit.Log(config.AuditLog, handler))), - } -} diff --git a/internal/audit/audit.go b/internal/audit/audit.go deleted file mode 100644 index 1c5356c3..00000000 --- a/internal/audit/audit.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package audit - -import ( - "encoding/json" - "net" - "net/http" - "net/url" - "sync/atomic" - "time" - - "github.com/minio/kes-go" - "github.com/minio/kes/internal/auth" - "github.com/minio/kes/internal/log" -) - -// Log wraps h with an http.Handler that logs an audit log -// event to the given logger. -func Log(logger *log.Logger, h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ip := auth.ForwardedIPFromContext(r.Context()) - if ip == nil { - if addr, _, err := net.SplitHostPort(r.RemoteAddr); err == nil { - ip = net.ParseIP(addr) - } - } - w = &responseWriter{ - rw: w, - - log: logger, - url: *r.URL, - ip: ip, - identity: auth.Identify(r), - timestamp: time.Now(), - } - h.ServeHTTP(w, r) - }) -} - -type responseWriter struct { - rw http.ResponseWriter - - log *log.Logger - url url.URL - ip net.IP - identity kes.Identity - timestamp time.Time - - hasSendHeaders atomic.Bool -} - -func (w *responseWriter) Header() http.Header { return w.rw.Header() } - -func (w *responseWriter) Write(p []byte) (int, error) { - w.WriteHeader(http.StatusOK) - return w.rw.Write(p) -} - -func (w *responseWriter) WriteHeader(status int) { - if !w.hasSendHeaders.CompareAndSwap(false, true) { - return - } - w.rw.WriteHeader(status) - - type RequestInfo struct { - IP net.IP `json:"ip,omitempty"` - Enclave string `json:"enclave,omitempty"` - APIPath string `json:"path"` - Identity kes.Identity `json:"identity,omitempty"` - } - type ResponseInfo struct { - StatusCode int `json:"code"` - Time time.Duration `json:"time"` - } - type Response struct { - Timestamp time.Time `json:"time"` - Request RequestInfo `json:"request"` - Response ResponseInfo `json:"response"` - } - - json.NewEncoder(w.log.Writer()).Encode(Response{ - Timestamp: w.timestamp, - Request: RequestInfo{ - IP: w.ip, - Enclave: w.url.Query().Get("enclave"), - APIPath: w.url.Path, - Identity: w.identity, - }, - Response: ResponseInfo{ - StatusCode: status, - Time: time.Now().UTC().Sub(w.timestamp.UTC()).Truncate(1 * time.Microsecond), - }, - }) -} - -func (w *responseWriter) Flush() { - if flusher, ok := w.rw.(http.Flusher); ok { - flusher.Flush() - } -} - -// Unwrap returns the underlying http.ResponseWriter. -// -// This method is implemented for http.ResponseController. -func (w *responseWriter) Unwrap() http.ResponseWriter { return w.rw } diff --git a/internal/auth/identity.go b/internal/auth/identity.go deleted file mode 100644 index 773c5c02..00000000 --- a/internal/auth/identity.go +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2022 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package auth - -import ( - "bytes" - "context" - "crypto/sha256" - "crypto/x509" - "encoding/gob" - "encoding/hex" - "errors" - "net/http" - "time" - - "github.com/minio/kes-go" -) - -// VerifyRequest verifies whether the request's identity is allowed to perform -// the request based on the given policies. -func VerifyRequest(r *http.Request, policies PolicySet, identities IdentitySet) error { - if r.TLS == nil { - return kes.NewError(http.StatusBadRequest, "insecure connection: TLS required") - } - - var peerCertificates []*x509.Certificate - switch { - case len(r.TLS.PeerCertificates) <= 1: - peerCertificates = r.TLS.PeerCertificates - case len(r.TLS.PeerCertificates) > 1: - for _, cert := range r.TLS.PeerCertificates { - if cert.IsCA { - continue - } - peerCertificates = append(peerCertificates, cert) - } - } - if len(peerCertificates) == 0 { - return kes.NewError(http.StatusBadRequest, "no client certificate is present") - } - if len(peerCertificates) > 1 { - return kes.NewError(http.StatusBadRequest, "too many client certificates are present") - } - - var ( - h = sha256.Sum256(peerCertificates[0].RawSubjectPublicKeyInfo) - identity = kes.Identity(hex.EncodeToString(h[:])) - ) - admin, err := identities.Admin(r.Context()) - if err != nil { - return err - } - if identity == admin { - return nil - } - - info, err := identities.Get(r.Context(), identity) - if errors.Is(err, kes.ErrIdentityNotFound) { - return kes.ErrNotAllowed - } - if err != nil { - return err - } - policy, err := policies.Get(r.Context(), info.Policy) - if errors.Is(err, kes.ErrPolicyNotFound) { - return kes.ErrNotAllowed - } - if err != nil { - return err - } - return policy.Verify(r) -} - -// Identify computes the identity of the given HTTP request. -// -// If the request was not sent over TLS or no client -// certificate has been provided, Identify returns -// IdentityUnknown. -func Identify(req *http.Request) kes.Identity { - if req.TLS == nil { - return kes.IdentityUnknown - } - - var cert *x509.Certificate - for _, c := range req.TLS.PeerCertificates { - if c.IsCA { - continue // Ignore CA certificates - } - - if cert != nil { - // There is more than one client certificate - // that is not a CA certificate. Hence, we - // cannot compute an non-ambiguous identity. - // Therefore, we return IdentityUnknown. - return kes.IdentityUnknown - } - cert = c - } - if cert == nil { - return kes.IdentityUnknown - } - - h := sha256.Sum256(cert.RawSubjectPublicKeyInfo) - return kes.Identity(hex.EncodeToString(h[:])) -} - -// An IdentitySet is a set of identities that are assigned to policies. -type IdentitySet interface { - // Admin returns the identity of the admin. - // - // The admin is never assigned to any policy - // and can perform any operation. - Admin(ctx context.Context) (kes.Identity, error) - - // Assign assigns the policy to the given identity. - // - // It returns an error when the identity is equal - // to the admin identity. - Assign(ctx context.Context, policy string, identity kes.Identity) error - - // Get returns the IdentityInfo of an assigned identity. - // - // It returns ErrIdentityNotFound when there is no IdentityInfo - // associated to the given identity. - Get(ctx context.Context, identity kes.Identity) (IdentityInfo, error) - - // Delete deletes the given identity from the list of - // assigned identites. - // - // It returns ErrNotAssigned when the identity is not - // assigned. - Delete(ctx context.Context, identity kes.Identity) error - - // List returns an iterator over all assigned identities. - List(ctx context.Context) (IdentityIterator, error) -} - -// An IdentityIterator iterates over a list of identites. -// -// for iterator.Next() { -// _ = iterator.Identity() // Get the next identity -// } -// if err := iterator.Close(); err != nil { -// } -// -// Once done iterating, an IdentityIterator should be closed. -// -// In general, an IdentityIterator does not provide any -// ordering guarantees. Concurrent changes to the underlying -// source may not be reflected by the iterator. -type IdentityIterator interface { - // Next moves the iterator to the subsequent identity, if any. - // This identity is available until Next is called again. - // - // It returns true if and only if there is another identity. - // Once an error occurs or once there are no more identities, - // Next returns false. - Next() bool - - // Identity returns the current identity. Identity can be - // called multiple times and returns the same value until - // Next is called again. - Identity() kes.Identity - - // Close closes the iterator and releases resources. It - // returns any error encountered while iterating, if any. - // Otherwise, it returns any error that occurred while - // closing, if any. - Close() error -} - -// IdentityInfo describes an assigned identity. -type IdentityInfo struct { - // Policy is the policy the identity is assigned to. - Policy string - - // IsAdmin indicates whether the identity has admin - // privileges. - IsAdmin bool - - // CreatedAt is the point in time when the identity - // has been assigned. - CreatedAt time.Time - - // CreatedBy is the identity that assigned this - // identity to its policy. - CreatedBy kes.Identity -} - -// MarshalBinary returns the IdentityInfo's binary representation. -func (i IdentityInfo) MarshalBinary() ([]byte, error) { - type GOB struct { - Policy string - IsAdmin bool - CreatedAt time.Time - CreatedBy kes.Identity - } - - var buffer bytes.Buffer - if err := gob.NewEncoder(&buffer).Encode(GOB(i)); err != nil { - return nil, err - } - return buffer.Bytes(), nil -} - -// UnmarshalBinary unmarshals the IdentityInfo's binary representation. -func (i *IdentityInfo) UnmarshalBinary(b []byte) error { - type GOB struct { - Policy string - IsAdmin bool - CreatedAt time.Time - CreatedBy kes.Identity - } - - var value GOB - if err := gob.NewDecoder(bytes.NewReader(b)).Decode(&value); err != nil { - return err - } - i.Policy = value.Policy - i.IsAdmin = value.IsAdmin - i.CreatedAt = value.CreatedAt - i.CreatedBy = value.CreatedBy - return nil -} diff --git a/internal/auth/policy.go b/internal/auth/policy.go deleted file mode 100644 index 532dc354..00000000 --- a/internal/auth/policy.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2019 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package auth - -import ( - "context" - "net/http" - "path" - "time" - - "github.com/minio/kes-go" -) - -// A PolicySet is a set of policies. -type PolicySet interface { - // Set creates or replaces the policy at the given name. - Set(ctx context.Context, name string, policy *Policy) error - - // Get returns the policy with the given name. - // - // It returns ErrPolicyNotFound if no policy with - // the given name exists. - Get(ctx context.Context, name string) (*Policy, error) - - // Delete deletes the policy with the given name. - // - // It returns ErrPolicyNotFound if no policy with - // the given name exists. - Delete(ctx context.Context, name string) error - - // List returns an iterator over all policies. - List(ctx context.Context) (PolicyIterator, error) -} - -// A PolicyIterator iterates over a list of policies. -// -// for iterator.Next() { -// _ = iterator.Name() // Get the next policy -// } -// if err := iterator.Close(); err != nil { -// } -// -// Once done iterating, a PolicyIterator should be closed. -// -// In general, a PolicyIterator does not provide any -// ordering guranatees. Concurrent changes to the -// underlying source may not be reflected by the iterator. -type PolicyIterator interface { - // Next moves the iterator to the subsequent policy, if any. - // This policy is available until Next is called again. - // - // It returns true if and only if there is another policy. - // Once an error occurs or once there are no more policies, - // Next returns false. - Next() bool - - // Name returns the name of the current policy. Name can be - // called multiple times and returns the same value until - // Next is called again. - Name() string - - // Close closes the iterator and releases resources. It - // returns any error encountered while iterating, if any. - // Otherwise, it returns any error that occurred while - // closing, if any. - Close() error -} - -// A Policy defines whether an HTTP request is allowed or -// should be rejected. -// -// It contains a set of allow and deny rules that are -// matched against the URL path. -type Policy struct { - // Allow is a list of glob patterns that are matched - // against the URL path of incoming requests. - Allow map[string]kes.Rule - - // Deny is a list of glob patterns that are matched - // against the URL path of incoming requests. - Deny map[string]kes.Rule - - // CreatedAt is the point in time when the policy - // has been created. - CreatedAt time.Time - - // CreatedBy is the identity that created the policy. - CreatedBy kes.Identity -} - -// Verify reports whether the given HTTP request is allowed. -// It returns no error if: -// -// (1) No deny pattern matches the URL path *AND* -// (2) At least one allow pattern matches the URL path. -// -// Otherwise, Verify returns ErrNotAllowed. -func (p *Policy) Verify(r *http.Request) error { - for pattern := range p.Deny { - if ok, err := path.Match(pattern, r.URL.Path); ok && err == nil { - return kes.ErrNotAllowed - } - } - for pattern := range p.Allow { - if ok, err := path.Match(pattern, r.URL.Path); ok && err == nil { - return nil - } - } - return kes.ErrNotAllowed -} diff --git a/internal/cache/cow.go b/internal/cache/cow.go index 46c274e2..201a1952 100644 --- a/internal/cache/cow.go +++ b/internal/cache/cow.go @@ -211,3 +211,18 @@ func (c *Cow[K, V]) Clone() *Cow[K, V] { cc.ptr.Store(&w) return cc } + +// Keys returns a slice of all keys of the Cow. +// It never returns nil. +func (c *Cow[K, _]) Keys() []K { + p := c.ptr.Load() + if len(*p) == 0 { + return []K{} + } + + keys := make([]K, 0, len(*p)) + for k := range *p { + keys = append(keys, k) + } + return keys +} diff --git a/internal/headers/header.go b/internal/headers/header.go new file mode 100644 index 00000000..f6e4ba8b --- /dev/null +++ b/internal/headers/header.go @@ -0,0 +1,64 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +// Package headers defines common HTTP headers. +package headers + +import ( + "net/http" + "slices" + "strings" +) + +// Commonly used HTTP headers. +const ( + Accept = "Accept" // RFC 2616 + Authorization = "Authorization" // RFC 2616 + ContentType = "Content-Type" // RFC 2616 + ContentLength = "Content-Length" // RFC 2616 + ETag = "ETag" // RFC 2616 + TransferEncoding = "Transfer-Encoding" // RFC 2616 +) + +// Commonly used HTTP headers for forwarding originating +// IP addresses of clients connecting through an reverse +// proxy or load balancer. +const ( + Forwarded = "Forwarded" // RFC 7239 + XForwardedFor = "X-Forwarded-For" // Non-standard + XFrameOptions = "X-Frame-Options" // Non-standard +) + +// Commonly used HTTP content type values. +const ( + ContentTypeBinary = "application/octet-stream" + ContentTypeJSON = "application/json" + ContentTypeJSONLines = "application/x-ndjson" + ContentTypeText = "text/plain" + ContentTypeHTML = "text/html" +) + +// Accepts reports whether h contains an "Accept" header +// that includes s. +func Accepts(h http.Header, s string) bool { + values := h[Accept] + if len(values) == 0 { + return false + } + + return slices.ContainsFunc(values, func(v string) bool { + if v == "*/*" { // matches any MIME type + return true + } + if v == s { + return true + } + if i := strings.IndexByte(v, '*'); i > 0 { // MIME patterns, like application/* + if v[i-1] == '/' { + return strings.HasPrefix(s, v[:i]) + } + } + return false + }) +} diff --git a/internal/headers/header_test.go b/internal/headers/header_test.go new file mode 100644 index 00000000..9ae7f447 --- /dev/null +++ b/internal/headers/header_test.go @@ -0,0 +1,40 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package headers + +import ( + "net/http" + "testing" +) + +func TestAccepts(t *testing.T) { + for i, test := range acceptsTests { + if accept := Accepts(test.Headers, test.ContentType); accept != test.Accept { + t.Errorf("Test %d: got '%v' - want '%v' for content type '%s'", i, accept, test.Accept, test.ContentType) + } + } +} + +var acceptsTests = []struct { + Headers http.Header + ContentType string + Accept bool +}{ + {http.Header{}, "", false}, // 0 + {http.Header{Accept: []string{}}, "", false}, // 1 + {http.Header{Accept: []string{ContentTypeJSON}}, ContentTypeHTML, false}, // 2 + {http.Header{Accept: []string{ContentTypeHTML, ContentTypeBinary}}, ContentTypeBinary, true}, // 3 + + {http.Header{Accept: []string{"*/*"}}, ContentTypeBinary, true}, // 4 + {http.Header{Accept: []string{"*/*"}}, ContentTypeHTML, true}, // 5 + {http.Header{Accept: []string{"*/*"}}, "", true}, // 6 + {http.Header{Accept: []string{"*"}}, ContentTypeHTML, false}, // 7 + + {http.Header{Accept: []string{"text/*"}}, ContentTypeHTML, true}, // 8 + {http.Header{Accept: []string{"text/*"}}, ContentTypeJSON, false}, // 9 + {http.Header{Accept: []string{"text*"}}, ContentTypeHTML, false}, // 10 + {http.Header{Accept: []string{"application/*"}}, ContentTypeBinary, true}, // 11 + {http.Header{Accept: []string{"application/*"}}, ContentTypeJSON, true}, // 12 +} diff --git a/internal/auth/proxy.go b/internal/https/proxy.go similarity index 90% rename from internal/auth/proxy.go rename to internal/https/proxy.go index 6cdc56d1..604b30b8 100644 --- a/internal/auth/proxy.go +++ b/internal/https/proxy.go @@ -2,11 +2,13 @@ // Use of this source code is governed by the AGPLv3 // license that can be found in the LICENSE file. -package auth +package https import ( "context" + "crypto/sha256" "crypto/x509" + "encoding/hex" "encoding/pem" "net" "net/http" @@ -135,7 +137,7 @@ func (p *TLSProxy) Verify(req *http.Request) error { } req.TLS.PeerCertificates = peerCertificates - identity := Identify(req) + identity := identify(req) if identity.IsUnknown() { return kes.ErrNotAllowed } @@ -213,6 +215,39 @@ func ForwardedIPFromContext(ctx context.Context) net.IP { return v.(net.IP) } +// Identify computes the identity of the given HTTP request. +// +// If the request was not sent over TLS or no client +// certificate has been provided, Identify returns +// IdentityUnknown. +func identify(req *http.Request) kes.Identity { + if req.TLS == nil { + return kes.IdentityUnknown + } + + var cert *x509.Certificate + for _, c := range req.TLS.PeerCertificates { + if c.IsCA { + continue // Ignore CA certificates + } + + if cert != nil { + // There is more than one client certificate + // that is not a CA certificate. Hence, we + // cannot compute an non-ambiguous identity. + // Therefore, we return IdentityUnknown. + return kes.IdentityUnknown + } + cert = c + } + if cert == nil { + return kes.IdentityUnknown + } + + h := sha256.Sum256(cert.RawSubjectPublicKeyInfo) + return kes.Identity(hex.EncodeToString(h[:])) +} + // getClientCertificate tries to extract an URL-escaped and ANS.1-encoded // X.509 certificate from the given HTTP headers. It returns an error if // no or more then one certificate are present or when the certificate diff --git a/internal/auth/proxy_test.go b/internal/https/proxy_test.go similarity index 99% rename from internal/auth/proxy_test.go rename to internal/https/proxy_test.go index 31921a22..273ada0b 100644 --- a/internal/auth/proxy_test.go +++ b/internal/https/proxy_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by the AGPLv3 // license that can be found in the LICENSE file. -package auth +package https import ( "net/http" diff --git a/internal/https/server.go b/internal/https/server.go deleted file mode 100644 index ee20a29f..00000000 --- a/internal/https/server.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package https - -import ( - "context" - "crypto/tls" - "errors" - "fmt" - "net" - "net/http" - "sync" - "time" - - "github.com/minio/kes/internal/fips" - "github.com/minio/kes/internal/log" -) - -// Config is a structure containing configuration -// fields for an HTTPS server. -type Config struct { - // Addr specifies an optional TCP address for the - // server to listen on in the form "host:port". - // If empty, ":https" (port 443) is used. - // - // The service names are defined in RFC 6335 and assigned by IANA. - // See net.Dial for details of the address format. - Addr string - - // Handler handles incoming requests. - Handler http.Handler - - // TLSConfig provides the TLS configuration. - TLSConfig *tls.Config - - Cancel context.CancelFunc -} - -// NewServer returns a new HTTPS server from -// the given config. -func NewServer(config *Config) *Server { - srv := &Server{ - addr: config.Addr, - tlsConfig: config.TLSConfig, - } - - srv.handler = &muxHandler{ - lock: srv.lock.RLocker(), - Handler: config.Handler, - } - return srv -} - -// Server is a HTTPS server. -type Server struct { - addr string - handler *muxHandler - tlsConfig *tls.Config - cancel context.CancelFunc - - lock sync.RWMutex -} - -// Update updates the Server's configuration or -// returns a non-nil error explaining why the -// server configuration couldn't be updated. -func (s *Server) Update(config *Config) error { - s.lock.Lock() - defer s.lock.Unlock() - - if config.Addr != s.addr { - return fmt.Errorf("https: failed to update server: '%s' does match existing server address", config.Addr) - } - - if s.cancel != nil { - s.cancel() - } - - s.tlsConfig = config.TLSConfig.Clone() - s.handler.Handler = config.Handler - s.cancel = config.Cancel - if s.handler.Handler == nil { - s.handler.Handler = http.NewServeMux() - } - return nil -} - -// UpdateTLS updates the Server's TLS configuration -// or returns a non-nil error explaining why the -// server configuration couldn't be updated. -func (s *Server) UpdateTLS(config *tls.Config) error { - s.lock.Lock() - defer s.lock.Unlock() - - s.tlsConfig = config.Clone() - return nil -} - -// Start starts the HTTPS server by listening on the -// Server's address. -// -// If the server address is empty, ":https" is used. -// -// Start blocks until the given ctx.Done() channel returns. -// It always returns a non-nil error. Once ctx.Done() -// returns, the Server gets closed and, if gracefully -// shutdown, Start returns http.ErrServerClosed. -func (s *Server) Start(ctx context.Context) error { - addr := s.addr - if addr == "" { - addr = ":https" - } - listener, err := tls.Listen("tcp", addr, &tls.Config{ - MinVersion: tls.VersionTLS12, - CipherSuites: fips.TLSCiphers(), - CurvePreferences: fips.TLSCurveIDs(), - - NextProtos: []string{"h2", "http/1.1"}, // Prefer HTTP/2 but also support HTTP/1.1 - GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { - s.lock.RLock() - defer s.lock.RUnlock() - return s.tlsConfig.Clone(), nil - }, - }) - if err != nil { - return err - } - - srv := &http.Server{ - Handler: s.handler, - ReadHeaderTimeout: 5 * time.Second, - WriteTimeout: 0 * time.Second, // explicitly set no write timeout - see timeout handler. - IdleTimeout: 90 * time.Second, - BaseContext: func(net.Listener) context.Context { return ctx }, - ErrorLog: log.Default().Log(), - } - srvCh := make(chan error, 1) - go func() { srvCh <- srv.Serve(listener) }() - - select { - case err := <-srvCh: - return err - case <-ctx.Done(): - graceCtx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - err := srv.Shutdown(graceCtx) - if errors.Is(err, context.DeadlineExceeded) { - err = srv.Close() - } - if err == nil { - err = http.ErrServerClosed - } - return err - } -} - -type muxHandler struct { - lock sync.Locker - http.Handler -} - -func (m *muxHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - m.lock.Lock() - handler := m.Handler - m.lock.Unlock() - - handler.ServeHTTP(w, req) -} diff --git a/internal/keystore/cache.go b/internal/keystore/cache.go deleted file mode 100644 index 4d973773..00000000 --- a/internal/keystore/cache.go +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package keystore - -import ( - "context" - "errors" - "net/http" - "sync/atomic" - "time" - - "github.com/minio/kes-go" - "github.com/minio/kes/internal/cache" - "github.com/minio/kes/internal/key" - "github.com/minio/kes/internal/log" - "github.com/minio/kes/kv" -) - -// CacheConfig is a structure containing Cache -// configuration options. -type CacheConfig struct { - // Expiry is the time period keys remain, at - // most, in the Cache. - // - // The zero value means keys never expire. - Expiry time.Duration - - // ExpiryUnused is the time period keys remain - // in the cache even though they are not used. - // - // A key that is used before one ExpiryUnused - // interval elapses is marked as used again and - // remains in the cache. - // - // The zero value means unused keys never expire. - ExpiryUnused time.Duration - - // ExpiryOffline is the time keys remain in the - // Cache, if the underlying kv.Store is offline. - // - // Offline caching is only used when the kv.Store - // is not available and ExpiryOffline > 0. - ExpiryOffline time.Duration -} - -// NewCache returns a new Cache wrapping the store. -// -// It uses the cache expiry configuration to clean -// up cache entries periodically. -// -// The Cache stops its periodic cleanup tasks once -// the ctx.Done channel returns or Stop is called; -// whatever happens first. -func NewCache(ctx context.Context, store kv.Store[string, []byte], config *CacheConfig) *Cache { - ctxGC, cancelGC := context.WithCancel(ctx) - c := &Cache{ - store: store, - cancelGC: cancelGC, - } - - go c.gc(ctxGC, config.Expiry, func() { - if offline := c.offline.Load(); !offline { - c.cache.DeleteAll() - } - }) - go c.gc(ctxGC, config.ExpiryUnused/2, func() { - if offline := c.offline.Load(); !offline { - c.cache.DeleteFunc(func(_ string, e *entry) bool { - // We remove an entry if it isn't marked as used. - // We also change all other entries to unused such - // that they get evicted on the next GC run unless - // they're used in between. - // - // Therefore, we try to switch the Used flag from - // true (used) to flase (unused). If this succeeds, - // the entry was in fact marked as used and must - // not be removed. Otherwise, the entry wasn't marked - // as used and we should evict it. - return !e.Used.CompareAndSwap(true, false) - }) - } - }) - go c.gc(ctxGC, config.ExpiryOffline, func() { - if offline := c.offline.Load(); offline { - c.cache.DeleteAll() - } - }) - go c.gc(ctxGC, 10*time.Second, func() { - _, err := c.store.Status(ctxGC) - if err != nil && !errors.Is(err, context.Canceled) { - c.offline.Store(true) - } else { - c.offline.Store(false) - } - }) - return c -} - -// A Cache caches keys in memory. -type Cache struct { - store kv.Store[string, []byte] - cache cache.Cow[string, *entry] - - // The barrier prevents reading the same key multiple - // times concurrently from the kv.Store. - // When a particular key isn't cached, we don't want - // to fetch it N times given N concurrent requests. - // Instead, we want the first request to fetch it and - // all others to wait until the first is done. - barrier cache.Barrier[string] - - // Controls whether we treat the cache as offline - // cache (with different GC config). - offline atomic.Bool - cancelGC func() // Stops the GC -} - -var _ kv.Store[string, key.Key] = (*Cache)(nil) - -// Stop stops all go routines that periodically -// remove entries from the Cache. -func (c *Cache) Stop() { c.cancelGC() } - -// Status returns the current state of the underlying -// kv.Store. -func (c *Cache) Status(ctx context.Context) (kv.State, error) { - return c.store.Status(ctx) -} - -// Create creates a new entry at the underlying kv.Store -// if and only if no entry for the given name exists. -// -// If such an entry already exists, Create returns ErrExists. -func (c *Cache) Create(ctx context.Context, name string, key key.Key) error { - b, err := key.MarshalText() - if err != nil { - log.Printf("keystore: failed to encode key '%s': %v", name, err) - return errCreateKey - } - - if err = c.store.Create(ctx, name, b); err != nil { - if errors.Is(err, kes.ErrKeyExists) { - return kes.ErrKeyExists - } - log.Printf("keystore: failed to create key '%s': %v", name, err) - return errCreateKey - } - return err -} - -// Set creates a new entry at the underlying kv.Store if and -// only if no entry for the given name exists. -// -// If such an entry already exists, Set returns ErrExists. -func (c *Cache) Set(ctx context.Context, name string, key key.Key) error { - return c.Create(ctx, name, key) -} - -// Delete deletes the key from the underlying kv.Store. -// -// It returns ErrNotExists if no such entry exists. -func (c *Cache) Delete(ctx context.Context, name string) error { - if err := c.store.Delete(ctx, name); err != nil { - if errors.Is(err, kes.ErrKeyNotFound) { - return err - } - log.Printf("keystore: failed to delete key '%s': %v", name, err) - return errDeleteKey - } - - c.cache.Delete(name) - return nil -} - -// List returns an Iter enumerating the stored keys. -func (c *Cache) List(ctx context.Context) (kv.Iter[string], error) { - iter, err := c.store.List(ctx) - if err != nil { - log.Printf("keystore: failed to list keys: %v", err) - return nil, errListKey - } - return iter, nil -} - -// Get returns the requested key. Get only fetches the key from the -// underlying kv.Store if it isn't in the Cache. -// -// It returns ErrNotExists if no such entry exists. -func (c *Cache) Get(ctx context.Context, name string) (key.Key, error) { - if entry, ok := c.cache.Get(name); ok { - entry.Used.Store(true) - return entry.Key, nil - } - - // Since the key is not in the cache, we want to fetch - but just once. - // However, we also don't want to block conccurent reads for different - // names. - // Hence, we accquire a lock per key and release it once done. - c.barrier.Lock(name) - defer c.barrier.Unlock(name) - - // Check the cache again, a previous request might have fetched the key - // while we were blocked by the barrier. - if entry, ok := c.cache.Get(name); ok { - entry.Used.Store(true) - return entry.Key, nil - } - - b, err := c.store.Get(ctx, name) - if err != nil { - if errors.Is(err, kes.ErrKeyNotFound) { - return key.Key{}, kes.ErrKeyNotFound - } - log.Printf("keystore: failed to fetch key '%s': %v", name, err) - return key.Key{}, errGetKey - } - - k, err := key.Parse(b) - if err != nil { - log.Printf("keystore: failed to fetch key '%s': %v", name, err) - return key.Key{}, errGetKey - } - - e := &entry{ - Key: k, - } - e.Used.Store(true) - c.cache.Set(name, e) - return k, nil -} - -// Close stops the Cache's GCs, if started, and closes the -// underlying keystore. -func (c *Cache) Close() error { - c.Stop() - return c.store.Close() -} - -// gc executes f periodically until the ctx.Done() channel returns. -func (c *Cache) gc(ctx context.Context, interval time.Duration, f func()) { - if interval <= 0 { - return - } - - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ticker.C: - f() - case <-ctx.Done(): - return - } - } -} - -// A cache entry with a recently used flag. -type entry struct { - Key key.Key - Used atomic.Bool -} - -// Typed errors that are returned to the client. -// The errors are generic on purpose to not leak -// any (potentially sensitive) information. -var ( - errCreateKey = kes.NewError(http.StatusBadGateway, "bad gateway: failed to create key") - errGetKey = kes.NewError(http.StatusBadGateway, "bad gateway: failed to access key") - errDeleteKey = kes.NewError(http.StatusBadGateway, "bad gateway: failed to delete key") - errListKey = kes.NewError(http.StatusBadGateway, "bad gateway: failed to list keys") -) diff --git a/internal/log/json_test.go b/internal/log/json_test.go deleted file mode 100644 index a2aedf47..00000000 --- a/internal/log/json_test.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2020 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package log - -import ( - "encoding/json" - "strings" - "testing" - - "github.com/minio/kes-go" -) - -var jsonWriterWriteTests = []struct { - Content string - Output string -}{ - { - Content: "", - Output: "{\"message\":\"\"}\n", - }, - { - Content: "\n", - Output: "{\"message\":\"\"}\n", - }, - { - Content: "Hello World", - Output: `{"message":"Hello World"}` + "\n", - }, - { - Content: "Hello \n World", - Output: `{"message":"Hello \n World"}` + "\n", - }, - { - Content: "Hello \n World" + "\n", - Output: `{"message":"Hello \n World"}` + "\n", - }, - { - Content: "Hello \t World \r" + "\n", - Output: `{"message":"Hello \t World \r"}` + "\n", - }, -} - -func TestErrEncoderWrite(t *testing.T) { - for i, test := range jsonWriterWriteTests { - var buffer strings.Builder - w := NewErrEncoder(&buffer) - w.WriteString(test.Content) - - output := buffer.String() - if output != test.Output { - t.Fatalf("Test %d: got '%s' - want '%s'", i, output, test.Output) - } - - // Apart from testing that the JSONWriter produces expected output - // we also test that the output can be un-marshaled to an ErrorEvent. - // This ensures that the JSONWriter actually implements JSON marshaling - // of the ErrorEvent type. - - newline := strings.HasSuffix(output, "\n") - if newline { - output = output[:len(output)-1] - } - var event kes.ErrorEvent - if err := json.Unmarshal([]byte(output), &event); err != nil { - t.Fatalf("Test %d: failed to unmarshal error event: %v", i, err) - } - if newline { - event.Message += "\n" - } - } -} diff --git a/internal/log/log.go b/internal/log/log.go deleted file mode 100644 index 235432b9..00000000 --- a/internal/log/log.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2020 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package log - -import ( - "encoding/json" - "io" - "log" - "os" - "strings" -) - -// These flags define how a Logger generates text for each log entry. -// Flags are or'ed together to control what gets printed. With the -// exception of the Lmsgprefix flag, there is no control over they -// appear or the format they present. -// The prefix is followed by a colon only when Llongfile or Lshortfile -// is specified. -// For example, flags Ldata | Ltime prodcue: -// -// 2023/01/01 01:23:45 message -// -// while flags Ldate | Ltime | Lmicroseconds | Llongfile produce, -// -// 2023/01/01 01:23:45.123456 /a/b/c/d.go:23: message -const ( - Ldate = log.Ldate // the date in the local time zone: 2023/01/01 - Ltime = log.Ltime // the time in the local time zone: 01:23:45 - Lmicroseconds = log.Lmicroseconds // microsecond resolution: 01:23:45.123456. assumes Ltime. - Llongfile = log.Llongfile // full file name and line number: /a/b/c/d.go:23 - Lshortfile = log.Lshortfile // final file name element and line number: d.go:23. overrides Llongfile - LUTC = log.LUTC // if Ldate or Ltime is set, use UTC rather than the local Ltime zone - Lmsgprefix = log.Lmsgprefix // move the "prefix" from the beginning of the line to before the message -) - -var std = New(os.Stderr, "Error: ", Ldate|Ltime|Lmsgprefix) - -// Default returns the package-level logger. -// -// By default, the logger writes to os.Stderr -// with an "Error" prefix and the flags: -// -// Ldate | Ltime | Lmsgprefx -func Default() *Logger { return std } - -// Print writes to the standard logger. -// Arguments are handled in the manner -// of fmt.Print. -func Print(v ...any) { std.Print(v...) } - -// Printf writes to the standard logger. -// Arguments are handled in the manner -// of fmt.Printf. -func Printf(format string, v ...any) { std.Printf(format, v...) } - -// Println writes to the standard logger. -// Arguments are handled in the manner -// of fmt.Println. -func Println(v ...any) { std.Println(v...) } - -// New creates a new Logger. The out is the destination to which -// log data will be written. The prefix appears at the beginning -// of each generated log line, or after the log header if the -// Lmsgprefix flag is provided. The flag argument defines the -// logging properties. -func New(out io.Writer, prefix string, flags int) *Logger { - mv := new(multiWriter) - mv.Add(out) - - return &Logger{ - log: log.New(mv, prefix, flags), - out: mv, - } -} - -// A Logger represents an active logging object that generates lines of -// output to one or multiple io.Writer. Each logging operation makes a -// single call to the Writer's Write method. A Logger can be used -// simultaneously from multiple goroutines; it guarantees to serialize -// access to the Writer. -type Logger struct { - log *log.Logger - out *multiWriter -} - -// Print writes to the standard logger. -// Arguments are handled in the manner -// of fmt.Print. -func (l *Logger) Print(v ...any) { l.log.Print(v...) } - -// Printf writes to the standard logger. -// Arguments are handled in the manner -// of fmt.Printf. -func (l *Logger) Printf(format string, v ...any) { l.log.Printf(format, v...) } - -// Println writes to the standard logger. -// Arguments are handled in the manner -// of fmt.Println. -func (l *Logger) Println(v ...any) { l.log.Println(v...) } - -// Add adds one or multiple io.Writer as output to -// the logger. -func (l *Logger) Add(out ...io.Writer) { l.out.Add(out...) } - -// Remove removes one or multiple io.Writer from the -// logging output. -func (l *Logger) Remove(out ...io.Writer) { l.out.Remove(out...) } - -// Log returns a new standard library log.Logger with -// logger's output, prefix and flags. -func (l *Logger) Log() *log.Logger { return log.New(l.log.Writer(), l.log.Prefix(), l.log.Flags()) } - -// Writer returns the output destination for the logger. -func (l *Logger) Writer() io.Writer { return l.out } - -// SetPrefix sets the output prefix for the logger. -func (l *Logger) SetPrefix(prefix string) { l.log.SetPrefix(prefix) } - -// ErrEncoder is an io.Writer that converts all -// log messages into a stream of kes.ErrorEvents. -// -// An ErrEncoder should be used when converting -// log messages to JSON. -type ErrEncoder struct { - encoder *json.Encoder -} - -// NewErrEncoder returns a new ErrEncoder that -// writes kes.ErrorEvents to w. -func NewErrEncoder(w io.Writer) *ErrEncoder { - return &ErrEncoder{ - encoder: json.NewEncoder(w), - } -} - -// Write converts p into an kes.ErrorEvent and -// writes its JSON representation to the underlying -// io.Writer. -func (w *ErrEncoder) Write(p []byte) (int, error) { - if len(p) == 0 { - return w.WriteString("") - } - return w.WriteString(string(p)) -} - -// WriteString converts s into an kes.ErrorEvent and -// writes its JSON representation to the underlying -// io.Writer. -func (w *ErrEncoder) WriteString(s string) (int, error) { - type Response struct { - Message string `json:"message"` - } - // A log.Logger will add a newline character to each - // log message. This newline has to be removed since - // it's not part of the actual error message. - s = strings.TrimSuffix(s, "\n") - - err := w.encoder.Encode(Response{ - Message: s, - }) - if err != nil { - return 0, err - } - return len(s), nil -} diff --git a/internal/log/writer.go b/internal/log/writer.go deleted file mode 100644 index 31eb6105..00000000 --- a/internal/log/writer.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package log - -import ( - "io" - "sync" -) - -// multiWriter is an io.Writer that writes the same data -// to multiple io.Writer sequentually. A multiWriter may -// be shared and concurrently modified by multiple go -// routines. -// -// In contrast to the io.MultiWriter, it keeps writing -// to all io.Writers even when one or multiple io.Writers -// return a non-nil. -// -// For example, when multiple HTTP clients have subscribed -// to the audit event stream, all other clients should receive -// the audit event even if one connections breaks. -type multiWriter struct { - lock sync.RWMutex - writers []io.Writer -} - -func (mw *multiWriter) Add(out ...io.Writer) { - if len(out) == 0 { - return - } - mw.lock.Lock() - defer mw.lock.Unlock() - - for _, o := range out { - if o == nil || o == io.Discard { - continue - } - if mv, ok := o.(*multiWriter); ok { - mv.lock.RLock() - mw.writers = append(mw.writers, mv.writers...) - mv.lock.RUnlock() - } else { - mw.writers = append(mw.writers, o) - } - } -} - -func (mw *multiWriter) Remove(out ...io.Writer) { - if len(out) == 0 { - return - } - mw.lock.Lock() - defer mw.lock.Unlock() - - writers := make([]io.Writer, 0, len(mw.writers)) - for _, w := range mw.writers { - var remove bool - for _, o := range out { - if w == o { - remove = true - break - } - if mv, ok := o.(*multiWriter); ok { - mv.lock.RLock() - if remove = contains(mv.writers, w); remove { - mv.lock.RUnlock() - break - } - mv.lock.RUnlock() - } - } - if !remove { - writers = append(writers, w) - } - } - mw.writers = writers -} - -func (mw *multiWriter) Write(p []byte) (n int, err error) { - if len(p) == 0 { - return 0, nil - } - mw.lock.RLock() - defer mw.lock.RUnlock() - - for _, w := range mw.writers { - nn, wErr := w.Write(p) - if err == nil && wErr != nil { - err = wErr - n = nn - } - if err == nil && nn != len(p) { - err = io.ErrShortWrite - n = nn - } - } - if err != nil { - return n, err - } - return len(p), nil -} - -func (mw *multiWriter) WriteString(s string) (n int, err error) { - if len(s) == 0 { - return 0, nil - } - mw.lock.RLock() - defer mw.lock.RUnlock() - - var p []byte // Only alloc if one writer does not implement io.StringWriter. - for _, w := range mw.writers { - var ( - nn int - wErr error - ) - if sw, ok := w.(io.StringWriter); ok { - nn, wErr = sw.WriteString(s) - } else { - if p == nil { - p = []byte(s) - } - nn, wErr = w.Write(p) - } - if err == nil && wErr != nil { - err = wErr - n = nn - } - if err == nil && nn != len(s) { - err = io.ErrShortWrite - n = nn - } - } - if err != nil { - return n, err - } - return len(s), nil -} - -func contains(writers []io.Writer, w io.Writer) bool { - for _, v := range writers { - if v == w { - return true - } - } - return false -} diff --git a/internal/secret/secret.go b/internal/secret/secret.go deleted file mode 100644 index cd482ab9..00000000 --- a/internal/secret/secret.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2022 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package secret - -import ( - "bytes" - "encoding/gob" - "time" - - "aead.dev/mem" - "github.com/minio/kes-go" -) - -// MaxSize is the maximum size of a secret. -const MaxSize = 1 * mem.MiB - -// Secret is a generic secret, like a password, -// API key, or private key. -type Secret struct { - kind kes.SecretType - createdAt time.Time - modTime time.Time - createdBy kes.Identity - - bytes []byte -} - -// NewSecret returns a new generic Secret from the given -// value. -// -// Its CreatedAt timestamp is time.Now and its CreatedBy -// identity is the owner. -func NewSecret(value []byte, owner kes.Identity) Secret { - now := time.Now().UTC() - return Secret{ - bytes: clone(value), - kind: kes.SecretGeneric, - createdAt: now, - modTime: now, - createdBy: owner, - } -} - -// Type returns the Secret's type. -func (s *Secret) Type() kes.SecretType { return s.kind } - -// CreatedAt returns the point in time when the secret has -// been created. -func (s *Secret) CreatedAt() time.Time { return s.createdAt } - -// ModTime returns the most recent point in time at which -// the secret has been modified. If the secret has never -// been modified, its ModTime is equal to its CreatedAt -// time. -func (s *Secret) ModTime() time.Time { return s.modTime } - -// CreatedBy returns the identity that created the secret. -func (s *Secret) CreatedBy() kes.Identity { return s.createdBy } - -// Bytes returns the Secret value. -func (s *Secret) Bytes() []byte { return clone(s.bytes) } - -// MarshalBinary returns the Secret's binary representation. -func (s *Secret) MarshalBinary() ([]byte, error) { - type GOB struct { - Type kes.SecretType - CreatedAt time.Time - ModTime time.Time - CreatedBy kes.Identity - Bytes []byte - } - - var buffer bytes.Buffer - err := gob.NewEncoder(&buffer).Encode(GOB{ - Type: s.kind, - Bytes: s.bytes, - CreatedAt: s.CreatedAt(), - ModTime: s.modTime, - CreatedBy: s.CreatedBy(), - }) - return buffer.Bytes(), err -} - -// UnmarshalBinary unmarshals the Secret's binary representation. -func (s *Secret) UnmarshalBinary(data []byte) error { - type GOB struct { - Type kes.SecretType - CreatedAt time.Time - ModTime time.Time - CreatedBy kes.Identity - Bytes []byte - } - - var value GOB - if err := gob.NewDecoder(bytes.NewReader(data)).Decode(&value); err != nil { - return err - } - - s.kind = value.Type - s.bytes = value.Bytes - s.createdAt = value.CreatedAt - s.modTime = value.ModTime - s.createdBy = value.CreatedBy - return nil -} - -// Iter is an iterator over secrets. -type Iter interface { - // Next fetches the next secret entry. It returns - // false when there are no more entries or once it - // encounters an error. - // - // Once Next returns false, it returns false on any - // subsequent Next call. - Next() bool - - // Name returns the name of the latest fetched entry. - // It returns the same name until Next is called again. - // - // As long as Next hasn't been called once or once Next - // returns false, Name returns the empty string. - Name() string - - // Close closes the Iter. Once closed, any subsequent - // Next call returns false. - // - // Close returns the first error encountered while iterating - // over the entires, if any. Otherwise, it returns the error - // encountered while cleaning up any resources, if any. - // Subsequent calls to Close return the same error. - Close() error -} - -func clone(data []byte) []byte { return append(make([]byte, 0, len(data)), data...) } diff --git a/internal/sys/build.go b/internal/sys/build.go index c4484a90..a9e6a502 100644 --- a/internal/sys/build.go +++ b/internal/sys/build.go @@ -5,62 +5,56 @@ package sys import ( + "errors" + "runtime" "runtime/debug" "strings" "sync" ) -// BuildInfo contains build information -// about a Go binary. -type BuildInfo struct { - Version string - CommitID string - Data string +// BinaryInfo contains build information about a Go binary. +type BinaryInfo struct { + Version string // The version of this binary + CommitID string // The git commit hash + Runtime string // The Go runtime version, e.g. go1.21.0 + Compiler string // The Go compiler used to build this binary } -// BinaryInfo returns the BuildInfo of the -// binary itself. -// -// It returns some default information -// when no build information has been -// compiled into the binary. -func BinaryInfo() BuildInfo { - readBinaryOnce.Do(func() { binaryInfo = readBinaryInfo() }) - return binaryInfo -} +// ReadBinaryInfo returns the ReadBinaryInfo about this program. +func ReadBinaryInfo() (BinaryInfo, error) { return readBinaryInfo() } -func readBinaryInfo() BuildInfo { +var readBinaryInfo = sync.OnceValues[BinaryInfo, error](func() (BinaryInfo, error) { const ( DefaultVersion = "" DefaultCommitID = "" + DefaultCompiler = "" ) - binaryInfo := BuildInfo{ + binaryInfo := BinaryInfo{ Version: DefaultVersion, CommitID: DefaultCommitID, + Runtime: runtime.Version(), + Compiler: DefaultCompiler, } info, ok := debug.ReadBuildInfo() if !ok { - return binaryInfo + return binaryInfo, errors.New("sys: binary does not contain build info") } const ( GitTimeKey = "vcs.time" GitRevisionKey = "vcs.revision" + CompilerKey = "-compiler" ) for _, setting := range info.Settings { - if setting.Key == GitTimeKey { + switch setting.Key { + case GitTimeKey: binaryInfo.Version = strings.ReplaceAll(setting.Value, ":", "-") - } - if setting.Key == GitRevisionKey { + case GitRevisionKey: binaryInfo.CommitID = setting.Value + case CompilerKey: + binaryInfo.Compiler = setting.Value } } - binaryInfo.Data = info.String() - return binaryInfo -} - -var ( - readBinaryOnce sync.Once - binaryInfo BuildInfo // protected by the sync.Once above -) + return binaryInfo, nil +}) diff --git a/kestest/example_test.go b/kestest/example_test.go deleted file mode 100644 index 71258adf..00000000 --- a/kestest/example_test.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2021 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package kestest_test - -import ( - "context" - "crypto/tls" - "fmt" - "log" - - "github.com/minio/kes-go" - "github.com/minio/kes/internal/keystore/mem" - "github.com/minio/kes/kestest" -) - -func ExampleGateway() { - server := kestest.NewGateway(&mem.Store{}) - defer server.Close() - - version, err := server.Client().Version(context.Background()) - if err != nil { - log.Fatal(err) - } - fmt.Println(version) - - // Output: - // -} - -func ExampleGateway_IssueClientCertificate() { - server := kestest.NewGateway(&mem.Store{}) - defer server.Close() - - server.Policy().Allow("test-policy", - "/v1/key/create/*", - "/v1/key/generate/*", - "/v1/key/decrypt/*", - ) - - var ( - clientCert = server.IssueClientCertificate("test-client") - client = kes.NewClientWithConfig(server.URL, &tls.Config{ - Certificates: []tls.Certificate{clientCert}, - RootCAs: server.CAs(), - }) - ) - server.Policy().Assign("test-policy", kestest.Identify(&clientCert)) - - if err := client.CreateKey(context.Background(), "test-key"); err != nil { - log.Fatal(err) - } - if err := client.DeleteKey(context.Background(), "test-key"); err != kes.ErrNotAllowed { - log.Fatalf("Deleting a key did not fail with %v", kes.ErrNotAllowed) - } - // Output: - // -} diff --git a/kestest/gateway.go b/kestest/gateway.go deleted file mode 100644 index 034d972d..00000000 --- a/kestest/gateway.go +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2021 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -// Package kestest provides utilities for end-to-end -// KES testing. -package kestest - -import ( - "context" - "crypto" - "crypto/ed25519" - "crypto/rand" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "fmt" - "io" - "math/big" - "net" - "net/http/httptest" - "time" - - "github.com/minio/kes-go" - "github.com/minio/kes/internal/api" - "github.com/minio/kes/internal/auth" - "github.com/minio/kes/internal/keystore" - "github.com/minio/kes/internal/log" - "github.com/minio/kes/internal/metric" - "github.com/minio/kes/kv" -) - -// NewGateway starts and returns a new Gateway. -// The caller should call Close when finished, -// to shut it down. -func NewGateway(store kv.Store[string, []byte]) *Gateway { - g := &Gateway{} - g.start(store) - return g -} - -// A Gateway is a KES gateway listening on a system-chosen -// port on the local loopback interface, for use in -// end-to-end tests. -type Gateway struct { - URL string - - policies *PolicySet - client *kes.Client - - caPrivateKey crypto.PrivateKey - caCertificate *x509.Certificate - - server *httptest.Server -} - -// Client returns a KES client configured for making requests -// to the Gateway as admin identity. -// -// It is configured to trust the Gateway's TLS test certificate. -func (g *Gateway) Client() *kes.Client { return g.client } - -// Policy returns the PolicySet that contains all KES policies -// and identity-policy associations. -func (g *Gateway) Policy() *PolicySet { return g.policies } - -// Close shuts down the Gateway and blocks until all outstanding -// requests on this server have completed. -func (g *Gateway) Close() { g.server.Close() } - -// IssueClientCertificate returns a new TLS certificate for -// client authentication with the given common name. -// -// The returned certificate is issued by a testing CA that is -// trusted by the Gateway. -func (g *Gateway) IssueClientCertificate(name string) tls.Certificate { - if g.caCertificate == nil || g.caPrivateKey == nil { - g.caPrivateKey, g.caCertificate = newCA() - } - return issueCertificate(name, g.caCertificate, g.caPrivateKey, x509.ExtKeyUsageClientAuth) -} - -// CAs returns the Gateway's root CAs. -func (g *Gateway) CAs() *x509.CertPool { - if g.caCertificate == nil || g.caPrivateKey == nil { - g.caPrivateKey, g.caCertificate = newCA() - } - - certpool := x509.NewCertPool() - certpool.AddCert(g.caCertificate) - return certpool -} - -func (g *Gateway) start(kmsStore kv.Store[string, []byte]) { - var ( - rootCAs = g.CAs() - auditLog = log.New(io.Discard, "", 0) - errorLog = log.New(io.Discard, "Error", log.Ldate|log.Ltime) - metrics = metric.New() - adminCert = g.IssueClientCertificate("kestest: admin") - ) - g.policies = &PolicySet{ - admin: Identify(&adminCert), - policies: make(map[string]*auth.Policy), - identities: make(map[kes.Identity]auth.IdentityInfo), - } - - auditLog.Add(metrics.AuditEventCounter()) - errorLog.Add(metrics.ErrorEventCounter()) - store := keystore.NewCache(context.Background(), kmsStore, &keystore.CacheConfig{ - Expiry: 30 * time.Second, - ExpiryUnused: 5 * time.Second, - }) - - serverCert := issueCertificate("kestest: gateway", g.caCertificate, g.caPrivateKey, x509.ExtKeyUsageServerAuth) - g.server = httptest.NewUnstartedServer(api.NewEdgeRouter(&api.EdgeRouterConfig{ - Keys: store, - Policies: g.policies.policySet(), - Identities: g.policies.identitySet(), - Proxy: nil, - AuditLog: auditLog, - ErrorLog: errorLog, - Metrics: metrics, - })) - g.server.TLS = &tls.Config{ - RootCAs: rootCAs, - ClientCAs: rootCAs, - Certificates: []tls.Certificate{serverCert}, - ClientAuth: tls.RequireAndVerifyClientCert, - } - g.server.StartTLS() - g.URL = g.server.URL - - g.client = kes.NewClientWithConfig(g.URL, &tls.Config{ - Certificates: []tls.Certificate{adminCert}, - RootCAs: rootCAs, - }) -} - -func newCA() (crypto.PrivateKey, *x509.Certificate) { - publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - panic(fmt.Sprintf("kestest: failed to generate CA private key: %v", err)) - } - - serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) - if err != nil { - panic(fmt.Sprintf("kestest: failed to generate CA certificate serial number: %v", err)) - } - - template := x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - CommonName: "kestest Root CA", - }, - NotBefore: time.Now(), - NotAfter: time.Now().Add(24 * time.Hour), - KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, - BasicConstraintsValid: true, - IsCA: true, - } - certBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey, privateKey) - if err != nil { - panic(fmt.Sprintf("kestest: failed to generate CA certificate: %v", err)) - } - certificate, err := x509.ParseCertificate(certBytes) - if err != nil { - panic(fmt.Sprintf("kestest: failed to generate CA certificate: %v", err)) - } - return privateKey, certificate -} - -func issueCertificate(name string, caCert *x509.Certificate, caKey crypto.PrivateKey, extKeyUsage ...x509.ExtKeyUsage) tls.Certificate { - publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - panic(fmt.Sprintf("kestest: failed to generate private/public key pair: %v", err)) - } - - serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) - if err != nil { - panic(fmt.Sprintf("kestest: failed to generate certificate serial number: %v", err)) - } - - template := x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - CommonName: name, - }, - NotBefore: time.Now(), - NotAfter: time.Now().Add(24 * time.Hour), - KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: extKeyUsage, - IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, - DNSNames: []string{"localhost"}, - BasicConstraintsValid: true, - } - - rawCertificate, err := x509.CreateCertificate(rand.Reader, &template, caCert, publicKey, caKey) - if err != nil { - panic(fmt.Sprintf("kestest: failed to create certificate: %v", err)) - } - rawPrivateKey, err := x509.MarshalPKCS8PrivateKey(privateKey) - if err != nil { - panic(fmt.Sprintf("kestest: failed to create certificate: %v", err)) - } - certificate, err := tls.X509KeyPair(pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Bytes: rawCertificate, - }), pem.EncodeToMemory(&pem.Block{ - Type: "PRIVATE KEY", - Bytes: rawPrivateKey, - })) - if err != nil { - panic(fmt.Sprintf("kestest: failed to create certificate: %v", err)) - } - return certificate -} diff --git a/kestest/gateway_aws_test.go b/kestest/gateway_aws_test.go deleted file mode 100644 index 1906dcdb..00000000 --- a/kestest/gateway_aws_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package kestest_test - -import ( - "context" - "flag" - "os" - "testing" - - "github.com/minio/kes/edge" -) - -var awsConfigFile = flag.String("aws.config", "", "Path to a KES config file with AWS SecretsManager config") - -func TestGatewayAWS(t *testing.T) { - if *awsConfigFile == "" { - t.Skip("AWS tests disabled. Use -aws.config= to enable them") - } - - file, err := os.Open(*awsConfigFile) - if err != nil { - t.Fatal(err) - } - defer file.Close() - srvrConfig, err := edge.ReadServerConfigYAML(file) - if err != nil { - t.Fatal(err) - } - - ctx, cancel := testingContext(t) - defer cancel() - - store, err := srvrConfig.KeyStore.Connect(context.Background()) - if err != nil { - t.Fatal(err) - } - - t.Run("Metrics", func(t *testing.T) { testMetrics(ctx, store, t) }) - t.Run("APIs", func(t *testing.T) { testAPIs(ctx, store, t) }) - t.Run("CreateKey", func(t *testing.T) { testCreateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("ImportKey", func(t *testing.T) { testImportKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("GenerateKey", func(t *testing.T) { testGenerateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("EncryptKey", func(t *testing.T) { testEncryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DecryptKey", func(t *testing.T) { testDecryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DescribePolicy", func(t *testing.T) { testDescribePolicy(ctx, store, t) }) - t.Run("GetPolicy", func(t *testing.T) { testGetPolicy(ctx, store, t) }) - t.Run("SelfDescribe", func(t *testing.T) { testSelfDescribe(ctx, store, t) }) -} diff --git a/kestest/gateway_azure_test.go b/kestest/gateway_azure_test.go deleted file mode 100644 index f1805043..00000000 --- a/kestest/gateway_azure_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package kestest_test - -import ( - "context" - "flag" - "os" - "testing" - - "github.com/minio/kes/edge" -) - -var azureConfigFile = flag.String("azure.config", "", "Path to a KES config file with Azure SecretsManager config") - -func TestGatewayAzure(t *testing.T) { - if *azureConfigFile == "" { - t.Skip("Azure tests disabled. Use -azure.config= to enable them") - } - file, err := os.Open(*azureConfigFile) - if err != nil { - t.Fatal(err) - } - defer file.Close() - srvrConfig, err := edge.ReadServerConfigYAML(file) - if err != nil { - t.Fatal(err) - } - - ctx, cancel := testingContext(t) - defer cancel() - - store, err := srvrConfig.KeyStore.Connect(context.Background()) - if err != nil { - t.Fatal(err) - } - - t.Run("Metrics", func(t *testing.T) { testMetrics(ctx, store, t) }) - t.Run("APIs", func(t *testing.T) { testAPIs(ctx, store, t) }) - t.Run("CreateKey", func(t *testing.T) { testCreateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("ImportKey", func(t *testing.T) { testImportKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("GenerateKey", func(t *testing.T) { testGenerateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("EncryptKey", func(t *testing.T) { testEncryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DecryptKey", func(t *testing.T) { testDecryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DescribePolicy", func(t *testing.T) { testDescribePolicy(ctx, store, t) }) - t.Run("GetPolicy", func(t *testing.T) { testGetPolicy(ctx, store, t) }) - t.Run("SelfDescribe", func(t *testing.T) { testSelfDescribe(ctx, store, t) }) -} diff --git a/kestest/gateway_entrust_test.go b/kestest/gateway_entrust_test.go deleted file mode 100644 index a2c43391..00000000 --- a/kestest/gateway_entrust_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package kestest_test - -import ( - "context" - "flag" - "os" - "testing" - - "github.com/minio/kes/edge" -) - -var entrustConfigFile = flag.String("entrust.config", "", "Path to a KES config file with Entrust KeyControl config") - -func TestGatewayEntrust(t *testing.T) { - if *entrustConfigFile == "" { - t.Skip("KeyControl tests disabled. Use -entrust.config= to enable them") - } - file, err := os.Open(*entrustConfigFile) - if err != nil { - t.Fatal(err) - } - defer file.Close() - srvrConfig, err := edge.ReadServerConfigYAML(file) - if err != nil { - t.Fatal(err) - } - - ctx, cancel := testingContext(t) - defer cancel() - - store, err := srvrConfig.KeyStore.Connect(context.Background()) - if err != nil { - t.Fatal(err) - } - - t.Run("Metrics", func(t *testing.T) { testMetrics(ctx, store, t) }) - t.Run("APIs", func(t *testing.T) { testAPIs(ctx, store, t) }) - t.Run("CreateKey", func(t *testing.T) { testCreateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("ImportKey", func(t *testing.T) { testImportKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("GenerateKey", func(t *testing.T) { testGenerateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("EncryptKey", func(t *testing.T) { testEncryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DecryptKey", func(t *testing.T) { testDecryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DescribePolicy", func(t *testing.T) { testDescribePolicy(ctx, store, t) }) - t.Run("GetPolicy", func(t *testing.T) { testGetPolicy(ctx, store, t) }) - t.Run("SelfDescribe", func(t *testing.T) { testSelfDescribe(ctx, store, t) }) -} diff --git a/kestest/gateway_fortanix_test.go b/kestest/gateway_fortanix_test.go deleted file mode 100644 index e4197af0..00000000 --- a/kestest/gateway_fortanix_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package kestest_test - -import ( - "context" - "flag" - "os" - "testing" - - "github.com/minio/kes/edge" -) - -var fortanixConfigFile = flag.String("fortanix.config", "", "Path to a KES config file with Fortanix SecretsManager config") - -func TestGatewayFortanix(t *testing.T) { - if *fortanixConfigFile == "" { - t.Skip("Fortanix tests disabled. Use -fortanix.config= to enable them") - } - file, err := os.Open(*fortanixConfigFile) - if err != nil { - t.Fatal(err) - } - defer file.Close() - srvrConfig, err := edge.ReadServerConfigYAML(file) - if err != nil { - t.Fatal(err) - } - - ctx, cancel := testingContext(t) - defer cancel() - - store, err := srvrConfig.KeyStore.Connect(context.Background()) - if err != nil { - t.Fatal(err) - } - - t.Run("Metrics", func(t *testing.T) { testMetrics(ctx, store, t) }) - t.Run("APIs", func(t *testing.T) { testAPIs(ctx, store, t) }) - t.Run("CreateKey", func(t *testing.T) { testCreateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("ImportKey", func(t *testing.T) { testImportKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("GenerateKey", func(t *testing.T) { testGenerateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("EncryptKey", func(t *testing.T) { testEncryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DecryptKey", func(t *testing.T) { testDecryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DescribePolicy", func(t *testing.T) { testDescribePolicy(ctx, store, t) }) - t.Run("GetPolicy", func(t *testing.T) { testGetPolicy(ctx, store, t) }) - t.Run("SelfDescribe", func(t *testing.T) { testSelfDescribe(ctx, store, t) }) -} diff --git a/kestest/gateway_fs_test.go b/kestest/gateway_fs_test.go deleted file mode 100644 index 68b9c6a0..00000000 --- a/kestest/gateway_fs_test.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package kestest_test - -import ( - "flag" - "testing" - - "github.com/minio/kes/internal/keystore/fs" -) - -var fsPath = flag.String("fs.path", "", "FS Path") - -func TestGatewayFS(t *testing.T) { - if *fsPath == "" { - t.Skip("FS tests disabled. Use -fs.path= to enable them.") - } - var err error - store, err := fs.NewStore(*fsPath) - if err != nil { - t.Fatal(err) - } - - ctx, cancel := testingContext(t) - defer cancel() - - t.Run("Metrics", func(t *testing.T) { testMetrics(ctx, store, t) }) - t.Run("APIs", func(t *testing.T) { testAPIs(ctx, store, t) }) - t.Run("CreateKey", func(t *testing.T) { testCreateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("ImportKey", func(t *testing.T) { testImportKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("GenerateKey", func(t *testing.T) { testGenerateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("EncryptKey", func(t *testing.T) { testEncryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DecryptKey", func(t *testing.T) { testDecryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DescribePolicy", func(t *testing.T) { testDescribePolicy(ctx, store, t) }) - t.Run("GetPolicy", func(t *testing.T) { testGetPolicy(ctx, store, t) }) - t.Run("SelfDescribe", func(t *testing.T) { testSelfDescribe(ctx, store, t) }) -} diff --git a/kestest/gateway_gcp_test.go b/kestest/gateway_gcp_test.go deleted file mode 100644 index 1a1201a1..00000000 --- a/kestest/gateway_gcp_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package kestest_test - -import ( - "context" - "flag" - "os" - "testing" - - "github.com/minio/kes/edge" -) - -var gcpConfigFile = flag.String("gcp.config", "", "Path to a KES config file with GCP SecretsManager config") - -func TestGatewayGCP(t *testing.T) { - if *gcpConfigFile == "" { - t.Skip("GCP tests disabled. Use -gcp.config= to enable them") - } - - file, err := os.Open(*gcpConfigFile) - if err != nil { - t.Fatal(err) - } - defer file.Close() - srvrConfig, err := edge.ReadServerConfigYAML(file) - if err != nil { - t.Fatal(err) - } - - ctx, cancel := testingContext(t) - defer cancel() - - store, err := srvrConfig.KeyStore.Connect(context.Background()) - if err != nil { - t.Fatal(err) - } - - t.Run("Metrics", func(t *testing.T) { testMetrics(ctx, store, t) }) - t.Run("APIs", func(t *testing.T) { testAPIs(ctx, store, t) }) - t.Run("CreateKey", func(t *testing.T) { testCreateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("ImportKey", func(t *testing.T) { testImportKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("GenerateKey", func(t *testing.T) { testGenerateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("EncryptKey", func(t *testing.T) { testEncryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DecryptKey", func(t *testing.T) { testDecryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DescribePolicy", func(t *testing.T) { testDescribePolicy(ctx, store, t) }) - t.Run("GetPolicy", func(t *testing.T) { testGetPolicy(ctx, store, t) }) - t.Run("SelfDescribe", func(t *testing.T) { testSelfDescribe(ctx, store, t) }) -} diff --git a/kestest/gateway_gemalto_test.go b/kestest/gateway_gemalto_test.go deleted file mode 100644 index e45152a1..00000000 --- a/kestest/gateway_gemalto_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package kestest_test - -import ( - "context" - "flag" - "os" - "testing" - - "github.com/minio/kes/edge" -) - -var gemaltoConfigFile = flag.String("gemalto.config", "", "Path to a KES config file with Gemalto SecretsManager config") - -func TestGatewayGemalto(t *testing.T) { - if *gemaltoConfigFile == "" { - t.Skip("Gemalto tests disabled. Use -gemalto.config= to enable them") - } - file, err := os.Open(*gemaltoConfigFile) - if err != nil { - t.Fatal(err) - } - defer file.Close() - srvrConfig, err := edge.ReadServerConfigYAML(file) - if err != nil { - t.Fatal(err) - } - - ctx, cancel := testingContext(t) - defer cancel() - - store, err := srvrConfig.KeyStore.Connect(context.Background()) - if err != nil { - t.Fatal(err) - } - - t.Run("Metrics", func(t *testing.T) { testMetrics(ctx, store, t) }) - t.Run("APIs", func(t *testing.T) { testAPIs(ctx, store, t) }) - t.Run("CreateKey", func(t *testing.T) { testCreateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("ImportKey", func(t *testing.T) { testImportKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("GenerateKey", func(t *testing.T) { testGenerateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("EncryptKey", func(t *testing.T) { testEncryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DecryptKey", func(t *testing.T) { testDecryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DescribePolicy", func(t *testing.T) { testDescribePolicy(ctx, store, t) }) - t.Run("GetPolicy", func(t *testing.T) { testGetPolicy(ctx, store, t) }) - t.Run("SelfDescribe", func(t *testing.T) { testSelfDescribe(ctx, store, t) }) -} diff --git a/kestest/gateway_mem_test.go b/kestest/gateway_mem_test.go deleted file mode 100644 index 1ef7c7ca..00000000 --- a/kestest/gateway_mem_test.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package kestest_test - -import ( - "testing" - - "github.com/minio/kes/internal/keystore/mem" - "github.com/minio/kes/kv" -) - -func TestGatewayMem(t *testing.T) { - ctx, cancel := testingContext(t) - defer cancel() - - store := kv.Store[string, []byte](&mem.Store{}) - - t.Run("Metrics", func(t *testing.T) { testMetrics(ctx, store, t) }) - t.Run("APIs", func(t *testing.T) { testAPIs(ctx, store, t) }) - t.Run("CreateKey", func(t *testing.T) { testCreateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("ImportKey", func(t *testing.T) { testImportKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("GenerateKey", func(t *testing.T) { testGenerateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("EncryptKey", func(t *testing.T) { testEncryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DecryptKey", func(t *testing.T) { testDecryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DescribePolicy", func(t *testing.T) { testDescribePolicy(ctx, store, t) }) - t.Run("GetPolicy", func(t *testing.T) { testGetPolicy(ctx, store, t) }) - t.Run("SelfDescribe", func(t *testing.T) { testSelfDescribe(ctx, store, t) }) -} diff --git a/kestest/gateway_test.go b/kestest/gateway_test.go deleted file mode 100644 index 11f48814..00000000 --- a/kestest/gateway_test.go +++ /dev/null @@ -1,546 +0,0 @@ -// Copyright 2022 - MinIO, Inc. All rights reserved. - -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package kestest_test - -import ( - "bytes" - "context" - "crypto/tls" - "encoding/base64" - "errors" - "fmt" - "io" - "maps" - "math/rand" - "net/http" - "strconv" - "testing" - "time" - - "github.com/minio/kes-go" - "github.com/minio/kes/kestest" - "github.com/minio/kes/kv" -) - -const ranStringLength = 8 - -var gatewayAPIs = map[string]struct { - Method string - MaxBody int64 - Timeout time.Duration -}{ - "/version": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - "/v1/ready": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - "/v1/status": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - "/v1/metrics": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - "/v1/api": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - - "/v1/key/create/": {Method: http.MethodPost, MaxBody: 0, Timeout: 15 * time.Second}, - "/v1/key/import/": {Method: http.MethodPost, MaxBody: 1 << 20, Timeout: 15 * time.Second}, - "/v1/key/describe/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - "/v1/key/list/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - "/v1/key/delete/": {Method: http.MethodDelete, MaxBody: 0, Timeout: 15 * time.Second}, - "/v1/key/generate/": {Method: http.MethodPost, MaxBody: 1 << 20, Timeout: 15 * time.Second}, - "/v1/key/encrypt/": {Method: http.MethodPost, MaxBody: 1 << 20, Timeout: 15 * time.Second}, - "/v1/key/decrypt/": {Method: http.MethodPost, MaxBody: 1 << 20, Timeout: 15 * time.Second}, - "/v1/key/bulk/decrypt/": {Method: http.MethodPost, MaxBody: 1 << 20, Timeout: 15 * time.Second}, - - "/v1/policy/describe/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - "/v1/policy/read/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - "/v1/policy/list/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - - "/v1/identity/describe/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - "/v1/identity/self/describe": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - "/v1/identity/list/": {Method: http.MethodGet, MaxBody: 0, Timeout: 15 * time.Second}, - - "/v1/log/error": {Method: http.MethodGet, MaxBody: 0, Timeout: 0}, - "/v1/log/audit": {Method: http.MethodGet, MaxBody: 0, Timeout: 0}, -} - -func testMetrics(ctx context.Context, store kv.Store[string, []byte], t *testing.T) { - server := kestest.NewGateway(store) - defer server.Close() - client := server.Client() - - metric, err := client.Metrics(ctx) - if err != nil { - t.Fatalf("Failed fetch server metrics: %v", err) - } - if n := metric.RequestOK + metric.RequestErr + metric.RequestFail; n != metric.RequestN() { - t.Fatalf("metrics request count differs: got %d - want %d", n, metric.RequestN()) - } - if metric.CPUs == 0 { - t.Fatalf("metrics contains no number of CPUs") - } - if metric.HeapAlloc == 0 { - t.Fatalf("metrics contains no heap allocations") - } - if metric.HeapObjects == 0 { - t.Fatalf("metrics contains no heap objects") - } - if metric.StackAlloc == 0 { - t.Fatalf("metrics contains no stack allocations") - } -} - -func testAPIs(ctx context.Context, store kv.Store[string, []byte], t *testing.T) { - server := kestest.NewGateway(store) - defer server.Close() - client := server.Client() - - apis, err := client.APIs(ctx) - if err != nil { - t.Fatalf("Failed fetch server APIs: %v", err) - } - if len(apis) != len(gatewayAPIs) { - t.Fatalf("API mismatch: got len '%d' - want len '%d'", len(apis), len(gatewayAPIs)) - } - for i := range apis { - api, ok := gatewayAPIs[apis[i].Path] - if !ok { - t.Fatalf("API '%s': API not found", apis[i].Path) - } - if apis[i].Method != api.Method { - t.Fatalf("API '%s': method mismatch: got '%s' - want '%s'", apis[i].Path, apis[i].Method, api.Method) - } - if apis[i].MaxBody != api.MaxBody { - t.Fatalf("API '%s': max body mismatch: got '%d' - want '%d'", apis[i].Path, apis[i].MaxBody, api.MaxBody) - } - if apis[i].Timeout != api.Timeout { - t.Fatalf("API '%s': timeout mismatch: got '%v' - want '%v'", apis[i].Path, apis[i].Timeout, api.Timeout) - } - } -} - -var createKeyTests = []struct { - Name string - ShouldFail bool - Err error -}{ - { // 0 - Name: "kestest", - }, - { // 1 - Name: "kestest", - ShouldFail: true, - Err: kes.ErrKeyExists, - }, -} - -func testCreateKey(ctx context.Context, store kv.Store[string, []byte], t *testing.T, seed string) { - server := kestest.NewGateway(store) - defer server.Close() - client := server.Client() - - defer clean(ctx, client, t) - - for i, test := range createKeyTests { - err := client.CreateKey(ctx, fmt.Sprintf("%s-%s", test.Name, seed)) - if err == nil && test.ShouldFail { - t.Fatalf("Test %d: should fail but succeeded", i) - } - if err != nil && !test.ShouldFail { - t.Fatalf("Test %d: failed to create key: %v", i, err) - } - if test.ShouldFail && test.Err != nil && err != test.Err { - t.Fatalf("Test %d: expected to fail with: '%v' - got: '%v'", i, test.Err, err) - } - } -} - -var importKeyTests = []struct { - Name string - Key []byte - ShouldFail bool - Err error -}{ - { // 0 - Name: "kestest", - Key: make([]byte, 32), - }, - { // 1 - Name: "kestest", - Key: make([]byte, 32), - ShouldFail: true, - Err: kes.ErrKeyExists, - }, - - { // 2 - Name: "fail-key", - Key: make([]byte, 0), - ShouldFail: true, - }, - { // 3 - Name: "fail-key2", - Key: make([]byte, 1<<20), - ShouldFail: true, - }, -} - -func testImportKey(ctx context.Context, store kv.Store[string, []byte], t *testing.T, seed string) { - server := kestest.NewGateway(store) - defer server.Close() - client := server.Client() - - defer clean(ctx, client, t) - - for i, test := range importKeyTests { - err := client.ImportKey(ctx, fmt.Sprintf("%s-%s", test.Name, seed), &kes.ImportKeyRequest{Key: test.Key}) - if err == nil && test.ShouldFail { - t.Fatalf("Test %d: should fail but succeeded", i) - } - if err != nil && !test.ShouldFail { - t.Fatalf("Test %d: failed to import key: %v", i, err) - } - if test.ShouldFail && test.Err != nil && err != test.Err { - t.Fatalf("Test %d: expected to fail with: '%v' - got: '%v'", i, test.Err, err) - } - } -} - -var generateKeyTests = []struct { - Context []byte - ShouldFail bool - Err error -}{ - {Context: make([]byte, 0)}, - {Context: []byte("Hello World")}, - {Context: make([]byte, 1<<20), ShouldFail: true}, -} - -func testGenerateKey(ctx context.Context, store kv.Store[string, []byte], t *testing.T, seed string) { - keyName := fmt.Sprintf("kestest-%s", seed) - - server := kestest.NewGateway(store) - defer server.Close() - client := server.Client() - - defer clean(ctx, client, t) - - if err := client.CreateKey(ctx, keyName); err != nil { - t.Fatalf("Failed to create %q: %v", keyName, err) - } - for i, test := range generateKeyTests { - dek, err := client.GenerateKey(ctx, keyName, test.Context) - if err == nil && test.ShouldFail { - t.Fatalf("Test %d: should fail but succeeded", i) - } - if err != nil && !test.ShouldFail { - t.Fatalf("Test %d: failed to generate DEK: %v", i, err) - } - if test.ShouldFail && test.Err != nil && err != test.Err { - t.Fatalf("Test %d: expected to fail with: '%v' - got: '%v'", i, test.Err, err) - } - - if !test.ShouldFail { - plaintext, err := client.Decrypt(ctx, keyName, dek.Ciphertext, test.Context) - if err != nil { - t.Fatalf("Test %d: failed to decrypt ciphertext: %v", i, err) - } - if !bytes.Equal(dek.Plaintext, plaintext) { - t.Fatalf("Test %d: decryption failed: got %x - want %x", i, plaintext, dek.Plaintext) - } - } - } -} - -var encryptKeyTests = []struct { - Plaintext []byte - Context []byte - ShouldFail bool - Err error -}{ - {Plaintext: []byte("Hello World"), Context: make([]byte, 0)}, - {Plaintext: []byte("Hello World"), Context: make([]byte, 32)}, - - {Plaintext: make([]byte, 1<<20), Context: make([]byte, 0), ShouldFail: true}, - {Plaintext: make([]byte, 0), Context: make([]byte, 1<<20), ShouldFail: true}, - {Plaintext: make([]byte, 512*1024), Context: make([]byte, 512*1024), ShouldFail: true}, -} - -func testEncryptKey(ctx context.Context, store kv.Store[string, []byte], t *testing.T, seed string) { - keyName := fmt.Sprintf("kestest-%s", seed) - server := kestest.NewGateway(store) - defer server.Close() - client := server.Client() - - defer clean(ctx, client, t) - - if err := client.CreateKey(ctx, keyName); err != nil { - t.Fatalf("Failed to create %q: %v", keyName, err) - } - for i, test := range encryptKeyTests { - ciphertext, err := client.Encrypt(ctx, keyName, test.Plaintext, test.Context) - if err == nil && test.ShouldFail { - t.Fatalf("Test %d: should fail but succeeded", i) - } - if err != nil && !test.ShouldFail { - t.Fatalf("Test %d: failed to encrypt plaintext: %v", i, err) - } - if test.ShouldFail && test.Err != nil && err != test.Err { - t.Fatalf("Test %d: expected to fail with: '%v' - got: '%v'", i, test.Err, err) - } - - if !test.ShouldFail { - plaintext, err := client.Decrypt(ctx, keyName, ciphertext, test.Context) - if err != nil { - t.Fatalf("Test %d: failed to decrypt ciphertext: %v", i, err) - } - if !bytes.Equal(test.Plaintext, plaintext) { - t.Fatalf("Test %d: decryption failed: got %x - want %x", i, plaintext, test.Plaintext) - } - } - } -} - -var decryptKeyTests = []struct { - Ciphertext []byte - Plaintext []byte - Context []byte - ShouldFail bool -}{ - { - Ciphertext: mustDecodeB64("eyJhZWFkIjoiQUVTLTI1Ni1HQ00tSE1BQy1TSEEtMjU2IiwiaWQiOiI2MmNmMjEzMDY2OTI3MmYzOWY3ZGU2MDU4Y2YzNzEyMyIsIml2IjoiQkpDU2FRZ1MrMUovZ3ZhcWZNaXJYUT09Iiwibm9uY2UiOiJHZkllRHdSdjByRDBIYncrIiwiYnl0ZXMiOiIvNndhelRQbnREMHhra0w5RWFGWjduK0s5SEJhem5YaDlKYjcifQ=="), - Plaintext: []byte("Hello World"), - Context: nil, - }, - { - Ciphertext: mustDecodeB64("eyJhZWFkIjoiQUVTLTI1Ni1HQ00tSE1BQy1TSEEtMjU2IiwiaWQiOiI2MmNmMjEzMDY2OTI3MmYzOWY3ZGU2MDU4Y2YzNzEyMyIsIml2IjoiYVN0OExGZWE2UUlFNVhhaEpTQ0w0Zz09Iiwibm9uY2UiOiJISjYyYndDcW1vMWVncHoxIiwiYnl0ZXMiOiJ1c291ZjhTb0Z5R1dybStOV0ZUQXFnPT0ifQ=="), - Plaintext: nil, - Context: make([]byte, 32), - }, - { - Ciphertext: mustDecodeB64("eyJhZWFkIjoiQUVTLTI1Ni1HQ00tSE1BQy1TSEEtMjU2IiwiaWQiOiI2MmNmMjEzMDY2OTI3MmYzOWY3ZGU2MDU4Y2YzNzEyMyIsIml2Ijoia3dLalZpcTBHSXpnMWJVcDM3QVNwZz09Iiwibm9uY2UiOiJ3Q0lJZEorcys3NTdGNjZFIiwiYnl0ZXMiOiIwYkZDSEY3NFUwZ29CT2w2d1lTaGE2K3FFV2FtNVZYYllxTW4ifQ=="), - Plaintext: []byte("Hello World"), - Context: make([]byte, 32), - }, -} - -func testDecryptKey(ctx context.Context, store kv.Store[string, []byte], t *testing.T, seed string) { - keyName := fmt.Sprintf("kestest-%s", seed) - server := kestest.NewGateway(store) - defer server.Close() - client := server.Client() - - defer clean(ctx, client, t) - - const KeyValue = "pQLPe6/f87AMSItvZzEbrxYdRUzmM81ziXF95HOFE4Y=" - if err := client.ImportKey(ctx, keyName, &kes.ImportKeyRequest{Key: mustDecodeB64(KeyValue)}); err != nil { - t.Fatalf("Failed to create %q: %v", keyName, err) - } - for i, test := range decryptKeyTests { - plaintext, err := client.Decrypt(ctx, keyName, test.Ciphertext, test.Context) - if test.ShouldFail { - if err == nil { - t.Fatalf("Test %d: should fail but succeeded", i) - } - continue - } - if err != nil { - t.Fatalf("Test %d: failed to decrypt ciphertext: %v", i, err) - } - if !bytes.Equal(plaintext, test.Plaintext) { - t.Fatalf("Test %d: failed to decrypt ciphertext: got '%x' - want '%x'", i, plaintext, test.Plaintext) - } - } -} - -var getPolicyTests = []struct { - Name string - Policy *kes.Policy -}{ - {Name: "my-policy", Policy: &kes.Policy{}}, - { - Name: "my-policy2", - Policy: &kes.Policy{ - Allow: map[string]kes.Rule{"/v1/key/create/*": {}, "/v1/key/generate/*": {}}, - }, - }, - { - Name: "my-policy2", - Policy: &kes.Policy{ - Allow: map[string]kes.Rule{"/v1/key/create/*": {}, "/v1/key/generate/*": {}}, - Deny: map[string]kes.Rule{"/v1/key/create/my-key2": {}}, - }, - }, -} - -func testDescribePolicy(ctx context.Context, store kv.Store[string, []byte], t *testing.T) { - for i, test := range getPolicyTests { - t.Run(fmt.Sprintf("Test %d", i), func(t *testing.T) { - server := kestest.NewGateway(store) - defer server.Close() - - server.Policy().Add(test.Name, test.Policy) - client := server.Client() - - info, err := client.DescribePolicy(ctx, test.Name) - if err != nil { - t.Fatalf("Test %d: failed to describe policy: %v", i, err) - } - if info.Name != test.Name { - t.Fatalf("Test %d: policy name mismatch: got '%s' - want '%s'", i, info.Name, test.Name) - } - if info.CreatedAt.IsZero() { - t.Fatalf("Test %d: created_at timestamp not set", i) - } - if info.CreatedBy.IsUnknown() { - t.Fatalf("Test %d: created_by identity not set", i) - } - }) - } -} - -func testGetPolicy(ctx context.Context, store kv.Store[string, []byte], t *testing.T) { - for i, test := range getPolicyTests { - t.Run(fmt.Sprintf("Test %d", i), func(t *testing.T) { - server := kestest.NewGateway(store) - defer server.Close() - - server.Policy().Add(test.Name, test.Policy) - client := server.Client() - - policy, err := client.GetPolicy(ctx, test.Name) - if err != nil { - t.Fatalf("Test %d: failed to describe policy: %v", i, err) - } - if policy.CreatedAt.IsZero() { - t.Fatalf("Test %d: created_at timestamp not set", i) - } - if policy.CreatedBy.IsUnknown() { - t.Fatalf("Test %d: created_by identity not set", i) - } - - if !maps.Equal(policy.Allow, test.Policy.Allow) { - t.Fatalf("Test %d: allow policy mismatch: got '%v' - want '%v'", i, policy.Allow, test.Policy.Allow) - } - if !maps.Equal(policy.Deny, test.Policy.Deny) { - t.Fatalf("Test %d: deny policy mismatch: got '%v' - want '%v'", i, policy.Deny, test.Policy.Deny) - } - }) - } -} - -var selfDescribeTests = []struct { - Policy kes.Policy -}{ - { // 0 - Policy: kes.Policy{}, - }, - { // 1 - Policy: kes.Policy{Allow: map[string]kes.Rule{}, Deny: map[string]kes.Rule{}}, - }, - { // 2 - Policy: kes.Policy{ - Allow: map[string]kes.Rule{ - "/v1/key/create/my-key-*": {}, - "/v1/key/generate/my-key-*": {}, - "/v1/key/decrypt/my-key-*": {}, - "/v1/key/delete/my-key-*": {}, - }, - Deny: map[string]kes.Rule{ - "/v1/key/delete/my-key-prod-*": {}, - }, - }, - }, -} - -func testSelfDescribe(ctx context.Context, store kv.Store[string, []byte], t *testing.T) { - server := kestest.NewGateway(store) - defer server.Close() - - client := server.Client() - info, policy, err := client.DescribeSelf(ctx) - if err != nil { - t.Fatalf("Failed to self-describe client: %v", err) - } - if !info.IsAdmin { - t.Fatalf("Identity hasn't admin privileges: got '%s' - want '%s'", info.Identity, server.Policy().Admin()) - } - if admin := server.Policy().Admin(); info.Identity != admin { - t.Fatalf("Identity hasn't admin privileges: got '%s' - want '%s'", info.Identity, server.Policy().Admin()) - } - if len(policy.Allow) != 0 || len(policy.Deny) != 0 { - t.Fatalf("Admin identity has a policy: %v", policy) - } - - for i, test := range selfDescribeTests { - cert := server.IssueClientCertificate("self-describe test") - client = kes.NewClientWithConfig(server.URL, &tls.Config{ - RootCAs: server.CAs(), - Certificates: []tls.Certificate{cert}, - }) - policyName := "Test-" + strconv.Itoa(i) - server.Policy().Add(policyName, &test.Policy) - server.Policy().Assign(policyName, kestest.Identify(&cert)) - - info, policy, err = client.DescribeSelf(ctx) - if err != nil { - t.Fatalf("Test %d: failed to self-describe client: %v", i, err) - } - if info.IsAdmin { - t.Fatalf("Test %d: identity has admin privileges", i) - } - if info.Policy != policyName { - t.Fatalf("Test %d: policy name mismatch: got '%s' - want '%s'", i, info.Policy, policyName) - } - if id := kestest.Identify(&cert); info.Identity != id { - t.Fatalf("Test %d: identity mismatch: got '%v' - want '%v'", i, info.Identity, id) - } - if !maps.Equal(policy.Allow, test.Policy.Allow) { - t.Fatalf("Test %d: allow policy mismatch: got '%v' - want '%v'", i, policy.Allow, test.Policy.Allow) - } - if !maps.Equal(policy.Deny, test.Policy.Deny) { - t.Fatalf("Test %d: deny policy mismatch: got '%v' - want '%v'", i, policy.Deny, test.Policy.Deny) - } - } -} - -func testingContext(t *testing.T) (context.Context, context.CancelFunc) { - deadline, ok := t.Deadline() - if ok { - return context.WithDeadline(context.Background(), deadline) - } - return context.WithCancel(context.Background()) -} - -func mustDecodeB64(s string) []byte { - b, err := base64.StdEncoding.DecodeString(s) - if err != nil { - panic(err) - } - return b -} - -func clean(ctx context.Context, client *kes.Client, t *testing.T) { - iter := &kes.ListIter[string]{ - NextFunc: client.ListKeys, - } - - var names []string - for name, err := iter.SeekTo(ctx, "*"); err != io.EOF; name, err = iter.Next(ctx) { - if err != nil { - t.Fatal(err) - } - names = append(names, name) - } - - for _, name := range names { - if err := client.DeleteKey(ctx, name); err != nil && !errors.Is(err, kes.ErrKeyNotFound) { - t.Errorf("Cleanup: failed to delete '%s': %v", name, err) - } - } -} - -const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - -func RandString(n int) string { - b := make([]byte, n) - for i := range b { - b[i] = letters[rand.Intn(len(letters))] - } - return string(b) -} diff --git a/kestest/gateway_vault_test.go b/kestest/gateway_vault_test.go deleted file mode 100644 index efcecd41..00000000 --- a/kestest/gateway_vault_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2023 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package kestest_test - -import ( - "context" - "flag" - "os" - "testing" - - "github.com/minio/kes/edge" -) - -var vaultConfigFile = flag.String("vault.config", "", "Path to a KES config file with Vault SecretsManager config") - -func TestGatewayVault(t *testing.T) { - if *vaultConfigFile == "" { - t.Skip("Vault tests disabled. Use -vault.config= to enable them") - } - file, err := os.Open(*vaultConfigFile) - if err != nil { - t.Fatal(err) - } - defer file.Close() - srvrConfig, err := edge.ReadServerConfigYAML(file) - if err != nil { - t.Fatal(err) - } - - ctx, cancel := testingContext(t) - defer cancel() - - store, err := srvrConfig.KeyStore.Connect(context.Background()) - if err != nil { - t.Fatal(err) - } - - t.Run("Metrics", func(t *testing.T) { testMetrics(ctx, store, t) }) - t.Run("APIs", func(t *testing.T) { testAPIs(ctx, store, t) }) - t.Run("CreateKey", func(t *testing.T) { testCreateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("ImportKey", func(t *testing.T) { testImportKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("GenerateKey", func(t *testing.T) { testGenerateKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("EncryptKey", func(t *testing.T) { testEncryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DecryptKey", func(t *testing.T) { testDecryptKey(ctx, store, t, RandString(ranStringLength)) }) - t.Run("DescribePolicy", func(t *testing.T) { testDescribePolicy(ctx, store, t) }) - t.Run("GetPolicy", func(t *testing.T) { testGetPolicy(ctx, store, t) }) - t.Run("SelfDescribe", func(t *testing.T) { testSelfDescribe(ctx, store, t) }) -} diff --git a/kestest/policy.go b/kestest/policy.go deleted file mode 100644 index 03c9dddc..00000000 --- a/kestest/policy.go +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright 2021 - MinIO, Inc. All rights reserved. -// Use of this source code is governed by the AGPLv3 -// license that can be found in the LICENSE file. - -package kestest - -import ( - "context" - "crypto/sha256" - "crypto/tls" - "crypto/x509" - "encoding/hex" - "fmt" - "net/http" - "sync" - "time" - - "github.com/minio/kes-go" - "github.com/minio/kes/internal/auth" -) - -// PolicySet holds a set of KES policies and -// the identity-policy associations. -type PolicySet struct { - admin kes.Identity - policies map[string]*auth.Policy - identities map[kes.Identity]auth.IdentityInfo -} - -// Admin returns the admin Identity that can -// perform any KES API operation. -func (p *PolicySet) Admin() kes.Identity { return p.admin } - -// Add adds the given KES policy to the PolicySet. -// Any existing policy with the same name is replaced. -func (p *PolicySet) Add(name string, policy *kes.Policy) { - p.policies[name] = &auth.Policy{ - Allow: policy.Allow, - Deny: policy.Deny, - CreatedAt: time.Now().UTC(), - CreatedBy: p.admin, - } -} - -// Allow adds a new KES policy that allows the given API -// patterns to the PolicySet. -// -// Allow is a shorthand for first creating a KES Policy -// and then adding it to the PolicySet. -func (p *PolicySet) Allow(name string, patterns ...string) { - allow := make(map[string]kes.Rule, len(patterns)) - for _, pattern := range patterns { - allow[pattern] = kes.Rule{} - } - p.Add(name, &kes.Policy{Allow: allow}) -} - -// Assign assigns the KES policy with the given name to -// all given identities. -// -// It returns the first error encountered when assigning -// identities, if any. -func (p *PolicySet) Assign(name string, ids ...kes.Identity) error { - for _, id := range ids { - if id.IsUnknown() { - return fmt.Errorf("kestest: failed to assign policy %q to %q: identity is empty", name, id) - } - if id == p.Admin() { - return fmt.Errorf("kestest: failed to assign policy %q to %q: equal to admin identity", name, id) - } - p.identities[id] = auth.IdentityInfo{ - Policy: name, - CreatedAt: time.Now().UTC(), - CreatedBy: p.admin, - } - } - return nil -} - -func (p *PolicySet) policySet() auth.PolicySet { - return &policySet{ - policies: p.policies, - } -} - -func (p *PolicySet) identitySet() auth.IdentitySet { - return &identitySet{ - admin: p.admin, - createdAt: time.Now().UTC(), - roles: p.identities, - } -} - -// Identify returns the Identity of the TLS certificate. -// -// It computes the Identity as fingerprint of the -// X.509 leaf certificate. -func Identify(cert *tls.Certificate) kes.Identity { - if cert.Leaf == nil { - var err error - cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) - if err != nil { - panic(fmt.Sprintf("kestest: failed to parse X.509 certificate: %v", err)) - } - } - - id := sha256.Sum256(cert.Leaf.RawSubjectPublicKeyInfo) - return kes.Identity(hex.EncodeToString(id[:])) -} - -type policySet struct { - lock sync.RWMutex - policies map[string]*auth.Policy -} - -func (p *policySet) Set(_ context.Context, name string, policy *auth.Policy) error { - p.lock.Lock() - defer p.lock.Unlock() - - p.policies[name] = policy - return nil -} - -func (p *policySet) Get(_ context.Context, name string) (*auth.Policy, error) { - p.lock.RLock() - defer p.lock.RUnlock() - - policy, ok := p.policies[name] - if !ok { - return nil, kes.ErrPolicyNotFound - } - return policy, nil -} - -func (p *policySet) Delete(_ context.Context, name string) error { - p.lock.Lock() - defer p.lock.Unlock() - - delete(p.policies, name) - return nil -} - -func (p *policySet) List(_ context.Context) (auth.PolicyIterator, error) { - p.lock.RLock() - defer p.lock.RUnlock() - - names := make([]string, 0, len(p.policies)) - for name := range p.policies { - names = append(names, name) - } - return &policyIterator{ - values: names, - }, nil -} - -type policyIterator struct { - values []string - current string -} - -func (i *policyIterator) Next() bool { - next := len(i.values) > 0 - if next { - i.current = i.values[0] - i.values = i.values[1:] - } - return next -} - -func (i *policyIterator) Name() string { return i.current } - -func (i *policyIterator) Close() error { return nil } - -type identitySet struct { - admin kes.Identity - createdAt time.Time - - lock sync.RWMutex - roles map[kes.Identity]auth.IdentityInfo -} - -func (i *identitySet) Admin(context.Context) (kes.Identity, error) { return i.admin, nil } - -func (i *identitySet) SetAdmin(context.Context, kes.Identity) error { - return kes.NewError(http.StatusNotImplemented, "cannot set admin identity") -} - -func (i *identitySet) Assign(_ context.Context, policy string, identity kes.Identity) error { - if i.admin == identity { - return kes.NewError(http.StatusBadRequest, "identity is root") - } - i.lock.Lock() - defer i.lock.Unlock() - - i.roles[identity] = auth.IdentityInfo{ - Policy: policy, - CreatedAt: time.Now().UTC(), - } - return nil -} - -func (i *identitySet) Get(_ context.Context, identity kes.Identity) (auth.IdentityInfo, error) { - if identity == i.admin { - return auth.IdentityInfo{ - IsAdmin: true, - CreatedAt: i.createdAt, - }, nil - } - i.lock.RLock() - defer i.lock.RUnlock() - - policy, ok := i.roles[identity] - if !ok { - return auth.IdentityInfo{}, kes.ErrIdentityNotFound - } - return policy, nil -} - -func (i *identitySet) Delete(_ context.Context, identity kes.Identity) error { - i.lock.Lock() - defer i.lock.Unlock() - - delete(i.roles, identity) - return nil -} - -func (i *identitySet) List(_ context.Context) (auth.IdentityIterator, error) { - i.lock.RLock() - defer i.lock.RUnlock() - - values := make([]kes.Identity, 0, len(i.roles)) - for identity := range i.roles { - values = append(values, identity) - } - return &identityIterator{ - values: values, - }, nil -} - -type identityIterator struct { - values []kes.Identity - current kes.Identity -} - -func (i *identityIterator) Next() bool { - next := len(i.values) > 0 - if next { - i.current = i.values[0] - i.values = i.values[1:] - } - return next -} - -func (i *identityIterator) Identity() kes.Identity { return i.current } - -func (i *identityIterator) Close() error { return nil } diff --git a/keystore.go b/keystore.go new file mode 100644 index 00000000..15ad592c --- /dev/null +++ b/keystore.go @@ -0,0 +1,360 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package kes + +import ( + "context" + "errors" + "io" + "slices" + "strings" + "sync/atomic" + "time" + + "github.com/minio/kes-go" + "github.com/minio/kes/internal/cache" + "github.com/minio/kes/internal/key" +) + +// A KeyStore stores key-value pairs. It provides durable storage for a +// KES server to persist and access keys. A KeyStore may be modified +// concurrently by different go routines. +type KeyStore interface { + // Closes the key store and releases associated resources, + // like background go routines, if any. + io.Closer + + // Status returns the current state of the KeyStore. + Status(context.Context) (KeyStoreState, error) + + // Create creates a new entry with the given name if and only + // if no such entry exists. + // Otherwise, Create returns kes.ErrKeyExists. + Create(ctx context.Context, name string, value []byte) error + + // Delete removes the entry. It may return either no error or + // kes.ErrKeyNotFound if no such entry exists. + Delete(ctx context.Context, name string) error + + // Get returns the value for the given name. It returns + // kes.ErrKeyNotFound if no such entry exits. + Get(ctx context.Context, name string) ([]byte, error) + + // List returns the first n key names, that start with the given + // prefix, and the next prefix from which the listing should + // continue. + // + // It returns all keys with the prefix if n < 0 and less then n + // names if n is greater than the number of keys with the prefix. + // + // An empty prefix matches any key name. At the end of the listing + // or when there are no (more) keys starting with the prefix, the + // returned prefix is empty. + List(ctx context.Context, prefix string, n int) ([]string, string, error) +} + +// KeyStoreState is a structure containing information about +// the current state of a KeyStore. +type KeyStoreState struct { + Latency time.Duration +} + +// MemKeyStore is a volatile KeyStore that stores key-value pairs in +// memory. Its zero value is ready and safe to be used concurrently +// from different go routines. It is optimized for reads but not +// well-suited for many writes/deletes. +type MemKeyStore struct { + keys cache.Cow[string, []byte] +} + +var _ KeyStore = (*MemKeyStore)(nil) // compiler check + +// Status returns the current state of the MemKeyStore. +// It never returns an error. +func (ks *MemKeyStore) Status(context.Context) (KeyStoreState, error) { + return KeyStoreState{ + Latency: 1 * time.Millisecond, + }, nil +} + +// Create creates a new entry with the given name if and only +// if no such entry exists. +// Otherwise, Create returns kes.ErrKeyExists. +func (ks *MemKeyStore) Create(_ context.Context, name string, value []byte) error { + if !ks.keys.Add(name, slices.Clone(value)) { + return kes.ErrKeyExists + } + return nil +} + +// Delete removes the entry. It may return either no error or +// kes.ErrKeyNotFound if no such entry exists. +func (ks *MemKeyStore) Delete(_ context.Context, name string) error { + if !ks.keys.Delete(name) { + return kes.ErrKeyNotFound + } + return nil +} + +// Get returns the value for the given name. It returns +// kes.ErrKeyNotFound if no such entry exits. +func (ks *MemKeyStore) Get(_ context.Context, name string) ([]byte, error) { + if val, ok := ks.keys.Get(name); ok { + return slices.Clone(val), nil + } + return nil, kes.ErrKeyNotFound +} + +// List returns the first n key names that start with the given +// prefix and the next prefix from which to continue the listing. +// +// It returns all keys with the prefix if n < 0 and less then n +// names if n is grater than the number of keys with the prefix. +// +// An empty prefix matches any key name. At the end of the listing +// or when there are no (more) keys starting with the prefix, the +// returned prefix is empty. +// +// List never returns an error. +func (ks *MemKeyStore) List(_ context.Context, prefix string, n int) ([]string, string, error) { + if n == 0 { + return []string{}, prefix, nil + } + + keys := ks.keys.Keys() + slices.Sort(keys) + + if prefix == "" { + if n < 0 || n >= len(keys) { + return keys, "", nil + } + return keys[:n], keys[n], nil + } + + i := slices.IndexFunc(keys, func(key string) bool { return strings.HasPrefix(key, prefix) }) + if i < 0 { + return []string{}, "", nil + } + + for j, key := range keys[i:] { + if !strings.HasPrefix(key, prefix) { + return keys[i : i+j], "", nil + } + if n > 0 && j == n { + return keys[i : i+j], key, nil + } + } + return keys[i:], "", nil +} + +// Close does nothing and returns no error. +// +// It is implemented to satisfy the KeyStore +// interface. +func (ks *MemKeyStore) Close() error { return nil } + +// newCache returns a new keyCache wrapping the KeyStore. +// It caches keys in memory and evicts cache entries based +// on the CacheConfig. +// +// Close the keyCache to release to the stop background +// garbage collector evicting cache entries and release +// associated resources. +func newCache(store KeyStore, conf *CacheConfig) *keyCache { + ctx, stop := context.WithCancel(context.Background()) + c := &keyCache{ + store: store, + stop: stop, + } + + go c.gc(ctx, conf.Expiry, func() { + if offline := c.offline.Load(); !offline { + c.cache.DeleteAll() + } + }) + go c.gc(ctx, conf.ExpiryUnused/2, func() { + if offline := c.offline.Load(); !offline { + c.cache.DeleteFunc(func(_ string, e *cacheEntry) bool { + // We remove an entry if it isn't marked as used. + // We also change all other entries to unused such + // that they get evicted on the next GC run unless + // they're used in between. + // + // Therefore, we try to switch the Used flag from + // true (used) to flase (unused). If this succeeds, + // the entry was in fact marked as used and must + // not be removed. Otherwise, the entry wasn't marked + // as used and we should evict it. + return !e.Used.CompareAndSwap(true, false) + }) + } + }) + go c.gc(ctx, conf.ExpiryOffline, func() { + if offline := c.offline.Load(); offline { + c.cache.DeleteAll() + } + }) + go c.gc(ctx, 10*time.Second, func() { + _, err := c.store.Status(ctx) + if err != nil && !errors.Is(err, context.Canceled) { + c.offline.Store(true) + } else { + c.offline.Store(false) + } + }) + return c +} + +// keyCache is an in-memory cache for keys fetched from a Keystore. +// A keyCache runs a background garbage collector that periodically +// evicts cache entries based on a CacheConfig. +// +// It uses lock-free concurrency primitives to optimize for fast +// concurrent reads. +type keyCache struct { + store KeyStore + cache cache.Cow[string, *cacheEntry] + + // The barrier prevents reading the same key multiple + // times concurrently from the kv.Store. + // When a particular key isn't cached, we don't want + // to fetch it N times given N concurrent requests. + // Instead, we want the first request to fetch it and + // all others to wait until the first is done. + barrier cache.Barrier[string] + + // Controls whether we treat the cache as offline + // cache (with different GC config). + offline atomic.Bool + stop func() // Stops the GC +} + +// A cache entry with a recently used flag. +type cacheEntry struct { + Key key.Key + Used atomic.Bool +} + +// Status returns the current state of the underlying KeyStore. +func (c *keyCache) Status(ctx context.Context) (KeyStoreState, error) { + return c.store.Status(ctx) +} + +// Create creates a new key with the given name if and only if +// no such entry exists. Otherwise, kes.ErrKeyExists is returned. +func (c *keyCache) Create(ctx context.Context, name string, key key.Key) error { + b, err := key.MarshalText() + if err != nil { + return err + } + + if err = c.store.Create(ctx, name, b); err != nil { + if errors.Is(err, kes.ErrKeyExists) { + return kes.ErrKeyExists + } + } + return err +} + +// Delete deletes the key from the key store and removes it from the +// cache. It may return either no error or kes.ErrKeyNotFound if no +// such entry exists. +func (c *keyCache) Delete(ctx context.Context, name string) error { + if err := c.store.Delete(ctx, name); err != nil { + if errors.Is(err, kes.ErrKeyNotFound) { + return err + } + return err + } + c.cache.Delete(name) + return nil +} + +// Get returns the key from the cache. If it key is not in the cache, +// Get tries to fetch it from the key store and put it into the cache. +// If the key is also not found at the key store, it returns +// kes.ErrKeyNotFound. +// +// Get tries to make as few calls to the underlying key store. Multiple +// concurrent Get calls for the same key, that is not in the cache, are +// serialized. +func (c *keyCache) Get(ctx context.Context, name string) (key.Key, error) { + if entry, ok := c.cache.Get(name); ok { + entry.Used.Store(true) + return entry.Key, nil + } + + // Since the key is not in the cache, we want to fetch it, once. + // However, we also don't want to block conccurent reads for different + // key names. + // Hence, we accquire a lock per key and release it once done. + c.barrier.Lock(name) + defer c.barrier.Unlock(name) + + // Check the cache again, a previous request might have fetched the key + // while we were blocked by the barrier. + if entry, ok := c.cache.Get(name); ok { + entry.Used.Store(true) + return entry.Key, nil + } + + b, err := c.store.Get(ctx, name) + if err != nil { + if errors.Is(err, kes.ErrKeyNotFound) { + return key.Key{}, kes.ErrKeyNotFound + } + return key.Key{}, err + } + + k, err := key.Parse(b) + if err != nil { + return key.Key{}, err + } + + entry := &cacheEntry{ + Key: k, + } + entry.Used.Store(true) + c.cache.Set(name, entry) + return k, nil +} + +// List returns the first n key names, that start with the given prefix, +// and the next prefix from which the listing should continue. +// +// It returns all keys with the prefix if n < 0 and less then n +// names if n is greater than the number of keys with the prefix. +// +// An empty prefix matches any key name. At the end of the listing +// or when there are no (more) keys starting with the prefix, the +// returned prefix is empty. +func (c *keyCache) List(ctx context.Context, prefix string, n int) ([]string, string, error) { + return c.store.List(ctx, prefix, n) +} + +// Close stops the cache's background garbage collector and +// releases associated resources. +func (c *keyCache) Close() error { + c.stop() + return nil +} + +// gc executes f periodically until the ctx.Done() channel returns. +func (c *keyCache) gc(ctx context.Context, interval time.Duration, f func()) { + if interval <= 0 { + return + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + f() + case <-ctx.Done(): + return + } + } +} diff --git a/log.go b/log.go new file mode 100644 index 00000000..0dcfc4cb --- /dev/null +++ b/log.go @@ -0,0 +1,91 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package kes + +import ( + "context" + "log/slog" + + "github.com/minio/kes/internal/api" +) + +// logHandler is an slog.Handler that handles Server log records. +// +// It wraps a custom slog.Handlers provided by Config.ErrorLog. If +// Config.ErrorLog is nil, a slog.TextHandler to os.Stderr is used +// as default. +// +// Log records may be handled twice. First, they are passed to the +// custom/default handler. For example to write to standard error. +// Second, they are sent to clients, that have subscribed to the +// ErrorLog API, if any. +type logHandler struct { + h slog.Handler + level slog.Leveler + + text slog.Handler + out *api.Multicast // clients subscribed to the ErrorLog API +} + +// newLogHandler returns a new logHandler that passing records to h. +// +// A record is only sent to clients subscribed to the ErrorLog API if +// its log level is >= level. +func newLogHandler(h slog.Handler, level slog.Leveler) *logHandler { + handler := &logHandler{ + h: h, + level: level, + out: &api.Multicast{}, + } + handler.text = slog.NewTextHandler(handler.out, &slog.HandlerOptions{ + Level: level, + }) + return handler +} + +// Enabled reports whether h handles records at the given level. +func (h *logHandler) Enabled(ctx context.Context, level slog.Level) bool { + return level >= h.level.Level() && h.h.Enabled(ctx, level) || + (h.text.Enabled(ctx, level) && h.out.Num() > 0) +} + +// Handle handles r by passing it first to the custom/default handler and +// then sending it to all clients subscribed to the ErrorLog API. +func (h *logHandler) Handle(ctx context.Context, r slog.Record) error { + var err error + if r.Level >= h.level.Level() { + err = h.h.Handle(ctx, r) + } + if h.out.Num() > 0 && h.text.Enabled(ctx, r.Level) { + if tErr := h.text.Handle(ctx, r); err == nil { + err = tErr + } + } + return err +} + +// WithAttrs returns a new Handler whose attributes consist of +// both the receiver's attributes and the arguments. +// The Handler owns the slice: it may retain, modify or discard it. +func (h *logHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return &logHandler{ + h: h.h.WithAttrs(attrs), + text: h.text.WithAttrs(attrs), + out: h.out, // Share all connections to clients + } +} + +// WithGroup returns a new Handler with the given group appended to +// the receiver's existing groups. +func (h *logHandler) WithGroup(name string) slog.Handler { + return &logHandler{ + h: h.h.WithGroup(name), + text: h.text.WithGroup(name), + out: h.out, // Share all connections to clients + } +} + +// Handler returns the underlying custom/default slog.Handler. +func (h *logHandler) Handler() slog.Handler { return h.h } diff --git a/server-config.yaml b/server-config.yaml index 0ac5f813..7b231fbf 100644 --- a/server-config.yaml +++ b/server-config.yaml @@ -228,27 +228,7 @@ keystore: # and development. It should not be used for production. fs: path: "" # Path to directory. Keys will be stored as files. - - # Configuration for storing keys on a KES server. - kes: - endpoint: - - "" # The endpoint (or list of endpoints) to the KES server(s) - enclave: "" # An optional enclave name. If empty, the default enclave will be used - tls: # The KES mTLS authentication credentials - i.e. client certificate. - cert: "" # Path to the TLS client certificate for mTLS authentication - key: "" # Path to the TLS client private key for mTLS authentication - ca: "" # Path to one or multiple PEM root CA certificates - # Configuration for storing keys via a KES KeyStore plugin or at - # a KeyStore that exposes an API compatible to the KES KeyStore - # plugin specification: https://github.com/minio/kes/blob/master/internal/generic/spec-v1.md - generic: - endpoint: "" # The plugin endpoint - e.g. https://127.0.0.1:7001 - tls: # The KES client TLS configuration for mTLS authentication and certificate verification. - key: "" # Path to the TLS client private key for mTLS authentication - cert: "" # Path to the TLS client certificate for mTLS authentication - ca: "" # Path to one or multiple PEM root CA certificates - # Hashicorp Vault configuration. The KES server will store/fetch # secret keys at/from Vault's key-value backend. # diff --git a/server.go b/server.go new file mode 100644 index 00000000..8d960eb2 --- /dev/null +++ b/server.go @@ -0,0 +1,1075 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package kes + +import ( + "context" + "crypto/rand" + "crypto/tls" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "runtime" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/minio/kes-go" + "github.com/minio/kes/internal/api" + "github.com/minio/kes/internal/cpu" + "github.com/minio/kes/internal/fips" + "github.com/minio/kes/internal/headers" + "github.com/minio/kes/internal/key" + "github.com/minio/kes/internal/metric" + "github.com/minio/kes/internal/sys" + "github.com/minio/kes/kv" + "github.com/prometheus/common/expfmt" +) + +// ServerShutdownTimeout is the default time period the server +// waits while trying to shutdown gracefully before forcefully +// closing connections. +const ServerShutdownTimeout = 1 * time.Second + +// Server is a KES server. +type Server struct { + // ShutdownTimeout controls how long Server.Close + // tries to shutdown the server gracefully without + // interrupting any active connections. + // + // If 0, defaults to ServerShutdownTimeout. If + // negative, Server.Close waits indefinitely for + // connections to return to idle and then shut down. + ShutdownTimeout time.Duration + + // ErrLevel controls which errors are logged by the server. + // It may be adjusted after the server has been started to + // change its logging behavior. + // + // Log records are passed to the Config.ErrorLog handler + // if and only if their log level is equal or greater than + // ErrLevel. A custom Config.ErrorLog may handle records + // independently from this ErrLevel. + // + // Defaults to slog.LevelInfo which includes TLS and HTTP + // errors when handling requests. + ErrLevel slog.LevelVar + + // AuditLevel controls which audit events are logged by + // the server. It may be adjusted after the server has + // been started to change its logging behavior. + // + // Log records are passed to the Config.AuditLog handler + // if and only if their log level is equal or greater than + // AuditLevel. A custom Config.AuditLog may handle records + // independently from this AuditLevel. + // + // Defaults to slog.LevelInfo. + AuditLevel slog.LevelVar + + tls atomic.Pointer[tls.Config] + state atomic.Pointer[serverState] + handler atomic.Pointer[http.ServeMux] + + mu sync.Mutex + srv *http.Server + started, closed bool + cErr error +} + +// Addr returns the server's listener address, or the +// empty string if the server hasn't been started. +func (s *Server) Addr() string { + s.mu.Lock() + defer s.mu.Unlock() + + state := s.state.Load() + if state == nil { + return "" + } + return state.Addr.String() +} + +// UpdateAdmin updates the server's admin identity. +// All other server configuration options remain +// unchanged. It returns an error if the server +// has not been started or has been closed. +func (s *Server) UpdateAdmin(admin kes.Identity) error { + if admin.IsUnknown() { + return errors.New("kes: admin identity is empty") + } + + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return errors.New("kes: server is closed") + } + if !s.started { + return errors.New("kes: server not started") + } + + old := s.state.Load() + s.state.Store(&serverState{ + Addr: old.Addr, + StartTime: old.StartTime, + Admin: admin, + Keys: old.Keys, + Policies: old.Policies, + Identities: old.Identities, + Metrics: old.Metrics, + Routes: old.Routes, + LogHandler: old.LogHandler, + Log: old.Log, + Audit: old.Audit, + }) + return nil +} + +// UpdateTLS updates the server's TLS configuration. +// All other server configuration options remain +// unchanged. It returns an error if the server +// has not been started or has been closed. +func (s *Server) UpdateTLS(conf *tls.Config) error { + if conf == nil || (len(conf.Certificates) == 0 && conf.GetCertificate == nil && conf.GetConfigForClient == nil) { + return errors.New("kes: tls config contains no server certificate") + } + if conf.ClientAuth == tls.NoClientCert { + return errors.New("kes: tls client auth must request client certifiate") + } + + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return errors.New("kes: server is closed") + } + if !s.started { + return errors.New("kes: server not started") + } + + s.tls.Store(conf) + return nil +} + +// UpdatePolicies updates the server policies. +// All other server configuration options remain +// unchanged. It returns an error if the server +// has not been started or has been closed. +func (s *Server) UpdatePolicies(policies map[string]Policy) error { + policySet, identitySet, err := initPolicies(policies) + if err != nil { + return err + } + + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return errors.New("kes: server is closed") + } + if !s.started { + return errors.New("kes: server not started") + } + + old := s.state.Load() + s.state.Store(&serverState{ + Addr: old.Addr, + StartTime: old.StartTime, + Admin: old.Admin, + Keys: old.Keys, + Policies: policySet, + Identities: identitySet, + Metrics: old.Metrics, + Routes: old.Routes, + LogHandler: old.LogHandler, + Log: old.Log, + Audit: old.Audit, + }) + return nil +} + +// Update changes the server's configuration. Callers should +// close the returned io.Closer once they want to releases any +// resources allocated by the previous configuration, like open +// file handles or background go routines. +// +// For only changing the server's admin identity, TLS configuration +// or policies use [Server.UpdateAdmin], [Server.UpdateTLS] or +// [Server.UpdatePolicies]. These more specific methods are usually +// simpler to use and more efficient. +func (s *Server) Update(conf *Config) (io.Closer, error) { + if err := verifyConfig(conf); err != nil { + return nil, err + } + policySet, identitySet, err := initPolicies(conf.Policies) + if err != nil { + return nil, err + } + + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return nil, errors.New("kes: server is closed") + } + if !s.started { + return nil, errors.New("kes: server not started") + } + + old := s.state.Load() + state := &serverState{ + Addr: old.Addr, + StartTime: old.StartTime, + Admin: conf.Admin, + Keys: newCache(conf.Keys, conf.Cache), + Policies: policySet, + Identities: identitySet, + Metrics: old.Metrics, + + LogHandler: old.LogHandler, + Log: old.Log, + Audit: old.Audit, + } + + if conf.ErrorLog != nil && conf.ErrorLog != state.LogHandler.Handler() { + state.LogHandler = &logHandler{ + h: conf.ErrorLog, + text: state.LogHandler.text, + out: state.LogHandler.out, + } + state.Log = slog.New(state.LogHandler) + } + if conf.AuditLog != nil { + state.Audit.h = conf.AuditLog + } + + mux, routes := initRoutes(s, conf.Routes) + state.Routes = routes + + s.tls.Store(conf.TLS.Clone()) + s.state.Store(state) + s.handler.Store(mux) + + return old.Keys, nil +} + +// ListenAndStart listens on the TCP network address addr and +// then calls Start to start the server using the given config. +// Accepted connections are configured to enable TCP keep-alives. +// +// HTTP/2 support is only enabled if conf.TLS is configured +// with "h2" in the TLS Config.NextProtos. +// +// ListenAndStart returns once the server is closed or ctx.Done +// returns, whatever happens first. It returns the first error +// encountered while shutting down the HTTPS server and closing +// the listener, if any. It attempts to shutdown the server +// gracefully by waiting for requests to finish before closing +// the server forcefully. +func (s *Server) ListenAndStart(ctx context.Context, addr string, conf *Config) error { + if err := verifyConfig(conf); err != nil { + return err + } + + if addr == "" { + addr = ":https" + } + + var lnConf net.ListenConfig + listener, err := lnConf.Listen(ctx, "tcp", addr) + if err != nil { + return err + } + defer listener.Close() + + return s.serve(ctx, listener, conf) +} + +// Start starts the server using the given config and accepts +// incoming HTTPS connections on the listener ln, creating a +// new service goroutine for each. +// +// HTTP/2 support is only enabled if conf.TLS is configured +// with "h2" in the TLS Config.NextProtos. +// +// Start returns once the server is closed or ctx.Done returns, +// whatever happens first. It returns the first error encountered +// while shutting down the HTTPS server and closing the listener, +// if any. It attempts to shutdown the server gracefully by waiting +// for requests to finish before closing the server forcefully. +func (s *Server) Start(ctx context.Context, ln net.Listener, conf *Config) error { + if err := verifyConfig(conf); err != nil { + return err + } + return s.serve(ctx, ln, conf) +} + +// Close closes the server and underlying listener. +// It first tries to shutdown the server gracefully +// by waiting for requests to finish before closing +// the server forcefully. +func (s *Server) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return s.cErr + } + s.closed = true + + if s.srv == nil { + if state := s.state.Load(); state != nil && state.Keys != nil { + s.cErr = state.Keys.Close() + } + return s.cErr + } + + shutdownTimeout := s.ShutdownTimeout + if shutdownTimeout == 0 { + shutdownTimeout = ServerShutdownTimeout + } + + ctx := context.Background() + if shutdownTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, shutdownTimeout) + defer cancel() + } + + s.cErr = s.srv.Shutdown(ctx) + if errors.Is(s.cErr, context.Canceled) || errors.Is(s.cErr, context.DeadlineExceeded) { + s.cErr = s.srv.Close() + } + if err := s.state.Load().Keys.Close(); s.cErr == nil { + s.cErr = err + } + return s.cErr +} + +func (s *Server) serve(ctx context.Context, ln net.Listener, conf *Config) error { + listener, err := s.listen(ctx, ln, conf) + if err != nil { + return err + } + defer listener.Close() + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + <-ctx.Done() + s.Close() + }() + + err = s.srv.Serve(listener) + if errors.Is(err, http.ErrServerClosed) { + return s.Close() + } + return err +} + +func (s *Server) listen(ctx context.Context, ln net.Listener, conf *Config) (net.Listener, error) { + policySet, identitySet, err := initPolicies(conf.Policies) + if err != nil { + return nil, err + } + + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return nil, errors.New("kes: server is closed") + } + if s.started { + return nil, errors.New("kes: server already started") + } + + state := &serverState{ + Addr: ln.Addr(), + StartTime: time.Now(), + Admin: conf.Admin, + Keys: newCache(conf.Keys, conf.Cache), + Policies: policySet, + Identities: identitySet, + Metrics: metric.New(), + } + + if conf.ErrorLog == nil { + state.LogHandler = newLogHandler( + slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: &s.ErrLevel, + }), + &s.ErrLevel, + ) + } else { + state.LogHandler = newLogHandler(conf.ErrorLog, &s.ErrLevel) + } + state.Log = slog.New(state.LogHandler) + + if conf.AuditLog == nil { + handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: &s.AuditLevel}) + state.Audit = newAuditLogger(&AuditLogHandler{Handler: handler}, &s.AuditLevel) + } else { + state.Audit = newAuditLogger(conf.AuditLog, &s.AuditLevel) + } + + mux, routes := initRoutes(s, conf.Routes) + state.Routes = routes + + s.tls.Store(conf.TLS.Clone()) + s.state.Store(state) + s.handler.Store(mux) + + s.srv = &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.handler.Load().ServeHTTP(w, r) + }), + + ReadHeaderTimeout: 5 * time.Second, + WriteTimeout: 0 * time.Second, // explicitly set no write timeout - api.Route uses http.ResponseController + IdleTimeout: 90 * time.Second, + BaseContext: func(net.Listener) context.Context { return ctx }, + ErrorLog: slog.NewLogLogger(s.state.Load().LogHandler, slog.LevelInfo), // TODO: wrap + } + s.started = true + + return tls.NewListener(ln, &tls.Config{ + GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + return s.tls.Load(), nil + }, + }), nil +} + +func (s *Server) version(resp *api.Response, req *api.Request) { + info, err := sys.ReadBinaryInfo() + if err != nil { + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusInternalServerError, "failed to read server version") + return + } + api.ReplyWith(resp, http.StatusOK, api.VersionResponse{ + Version: info.Version, + Commit: info.CommitID, + }) +} + +func (s *Server) ready(resp *api.Response, req *api.Request) { + _, err := s.state.Load().Keys.Status(req.Context()) + if _, ok := kv.IsUnreachable(err); ok { + s.state.Load().Log.WarnContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusGatewayTimeout, "key store is not reachable") + return + } + if err != nil { + s.state.Load().Log.WarnContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadGateway, "key store is unavailable") + return + } + resp.Reply(http.StatusOK) +} + +func (s *Server) status(resp *api.Response, req *api.Request) { + var ( + unavailable, unreachable bool + latency time.Duration + ) + state, err := s.state.Load().Keys.Status(req.Context()) + if err != nil { + unavailable = true + _, unreachable = kv.IsUnreachable(err) + } else { + latency = state.Latency.Round(time.Millisecond) + if latency == 0 { // Make sure we actually send a latency even if the key store respond time is < 1ms. + latency = 1 * time.Millisecond + } + } + + info, err := sys.ReadBinaryInfo() + if err != nil { + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusInternalServerError, "failed to read server version") + return + } + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + api.ReplyWith(resp, http.StatusOK, api.StatusResponse{ + Version: info.Version, + OS: runtime.GOOS, + Arch: runtime.GOARCH, + UpTime: uint64(time.Since(s.state.Load().StartTime).Round(time.Second).Seconds()), + + CPUs: runtime.NumCPU(), + UsableCPUs: runtime.GOMAXPROCS(0), + HeapAlloc: memStats.HeapAlloc, + StackAlloc: memStats.StackSys, + + KeyStoreLatency: latency.Milliseconds(), + KeyStoreUnavailable: unavailable, + KeyStoreUnreachable: unreachable, + }) +} + +func (s *Server) metrics(resp *api.Response, req *api.Request) { + contentType := expfmt.Negotiate(req.Header) + resp.Header().Set(headers.ContentType, string(contentType)) + resp.WriteHeader(http.StatusOK) + s.state.Load().Metrics.EncodeTo(expfmt.NewEncoder(resp, contentType)) +} + +// ListAPIs is a HandlerFunc that sends the list of server API +// routes to the client. +func (s *Server) listAPIs(resp *api.Response, _ *api.Request) { + routes := s.state.Load().Routes + responses := make(api.ListAPIsResponse, 0, len(routes)) + for _, ro := range routes { + responses = append(responses, api.DescribeRouteResponse{ + Method: ro.Method, + Path: ro.Path, + MaxBody: int64(ro.MaxBody), + Timeout: int64(ro.Timeout.Truncate(time.Second).Seconds()), + }) + } + api.ReplyWith(resp, http.StatusOK, responses) +} + +func (s *Server) createKey(resp *api.Response, req *api.Request) { + if !validName(req.Resource) { + resp.Failf(http.StatusBadRequest, "key name '%s' is empty, too long or contains invalid characters", req.Resource) + return + } + + var algorithm kes.KeyAlgorithm + if fips.Enabled || cpu.HasAESGCM() { + algorithm = kes.AES256 + } else { + algorithm = kes.ChaCha20 + } + + key, err := key.Random(algorithm, req.Identity) + if err != nil { + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusInternalServerError, "failed to generate encryption key") + return + } + + if err = s.state.Load().Keys.Create(req.Context(), req.Resource, key); err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadGateway, "failed to create key") + return + } + + const StatusOK = http.StatusOK + s.state.Load().Audit.Log( + fmt.Sprintf("secret key '%s' created", req.Resource), + StatusOK, + req, + ) + resp.Reply(StatusOK) +} + +func (s *Server) importKey(resp *api.Response, req *api.Request) { + if !validName(req.Resource) { + resp.Failf(http.StatusBadRequest, "key name '%s' is empty, too long or contains invalid characters", req.Resource) + return + } + + var imp api.ImportKeyRequest + if err := api.ReadBody(req, &imp); err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadRequest, "invalid import key request body") + return + } + + var algorithm kes.KeyAlgorithm + switch imp.Cipher { + case "AES256", "AES256-GCM_SHA256": + algorithm = kes.AES256 + case "ChaCha20", "XCHACHA20-POLY1305": + if fips.Enabled { + resp.Failf(http.StatusNotAcceptable, "algorithm '%s' not supported by FIPS 140-2", imp.Cipher) + return + } + algorithm = kes.ChaCha20 + default: + resp.Failf(http.StatusNotAcceptable, "algorithm '%s' is not supported", imp.Cipher) + return + } + + if len(imp.Bytes) != key.Len(algorithm) { + resp.Failf(http.StatusNotAcceptable, "invalid key size for '%s'", imp.Cipher) + return + } + + key, err := key.New(algorithm, imp.Bytes, req.Identity) + if err != nil { + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusInternalServerError, "failed to create key") + return + } + if err = s.state.Load().Keys.Create(req.Context(), req.Resource, key); err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadGateway, "failed to create key") + return + } + + const StatusOK = http.StatusOK + s.state.Load().Audit.Log( + fmt.Sprintf("secret key '%s' created", req.Resource), + StatusOK, + req, + ) + resp.Reply(StatusOK) +} + +func (s *Server) describeKey(resp *api.Response, req *api.Request) { + if !validName(req.Resource) { + resp.Failf(http.StatusBadRequest, "key name '%s' is empty, too long or contains invalid characters", req.Resource) + return + } + + key, err := s.state.Load().Keys.Get(req.Context(), req.Resource) + if err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadGateway, "failed to read key") + return + } + + api.ReplyWith(resp, http.StatusOK, api.DescribeKeyResponse{ + Name: req.Resource, + Algorithm: key.Algorithm().String(), + CreatedAt: key.CreatedAt(), + CreatedBy: key.CreatedBy().String(), + }) +} + +func (s *Server) listKeys(resp *api.Response, req *api.Request) { + if !validPattern(req.Resource) { + resp.Failf(http.StatusBadRequest, "listing pattern '%s' is empty, too long or is invalid", req.Resource) + return + } + + prefix := req.Resource + if prefix == "*" { + prefix = "" + } + + names, prefix, err := s.state.Load().Keys.List(req.Context(), prefix, -1) + if err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadGateway, "failed to list keys") + } + + api.ReplyWith(resp, http.StatusOK, api.ListKeysResponse{ + Names: names, + ContinueAt: prefix, + }) +} + +func (s *Server) deleteKey(resp *api.Response, req *api.Request) { + if !validName(req.Resource) { + resp.Failf(http.StatusBadRequest, "key name '%s' is empty, too long or contains invalid characters", req.Resource) + return + } + + if err := s.state.Load().Keys.Delete(req.Context(), req.Resource); err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadGateway, "failed to delete key") + return + } + + const StatusOK = http.StatusOK + s.state.Load().Audit.Log( + fmt.Sprintf("secret key '%s' deleted", req.Resource), + StatusOK, + req, + ) + resp.Reply(http.StatusOK) +} + +func (s *Server) encryptKey(resp *api.Response, req *api.Request) { + if !validName(req.Resource) { + resp.Failf(http.StatusBadRequest, "key name '%s' is empty, too long or contains invalid characters", req.Resource) + return + } + + var enc api.EncryptKeyRequest + if err := api.ReadBody(req, &enc); err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadRequest, "invalid request body") + return + } + + key, err := s.state.Load().Keys.Get(req.Context(), req.Resource) + if err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadGateway, "failed to read key") + return + } + ciphertext, err := key.Wrap(enc.Plaintext, enc.Context) + if err != nil { + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusInternalServerError, "failed to encrypt plaintext") + return + } + + api.ReplyWith(resp, http.StatusOK, api.EncryptKeyResponse{ + Ciphertext: ciphertext, + }) +} + +func (s *Server) generateKey(resp *api.Response, req *api.Request) { + if !validName(req.Resource) { + resp.Failf(http.StatusBadRequest, "key name '%s' is empty, too long or contains invalid characters", req.Resource) + return + } + + var gen api.GenerateKeyRequest + if req.ContentLength > 0 { + if err := api.ReadBody(req, &gen); err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadRequest, "invalid request body") + return + } + } + + key, err := s.state.Load().Keys.Get(req.Context(), req.Resource) + if err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadGateway, "failed to read key") + return + } + + dataKey := make([]byte, 32) + if _, err = rand.Read(dataKey); err != nil { + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusInternalServerError, "failed to generate encryption key") + return + } + ciphertext, err := key.Wrap(dataKey, gen.Context) + if err != nil { + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusInternalServerError, "failed to generate encryption key") + return + } + + api.ReplyWith(resp, http.StatusOK, api.GenerateKeyResponse{ + Plaintext: dataKey, + Ciphertext: ciphertext, + }) +} + +func (s *Server) decryptKey(resp *api.Response, req *api.Request) { + if !validName(req.Resource) { + resp.Failf(http.StatusBadRequest, "key name '%s' is empty, too long or contains invalid characters", req.Resource) + return + } + + var enc api.DecryptKeyRequest + if err := api.ReadBody(req, &enc); err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadRequest, "invalid request body") + return + } + + key, err := s.state.Load().Keys.Get(req.Context(), req.Resource) + if err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusBadGateway, "failed to read key") + return + } + plaintext, err := key.Unwrap(enc.Ciphertext, enc.Context) + if err != nil { + if err, ok := api.IsError(err); ok { + resp.Failr(err) + return + } + + s.state.Load().Log.ErrorContext(req.Context(), err.Error(), "req", req) + resp.Fail(http.StatusInternalServerError, "failed to decrypt ciphertext") + return + } + + api.ReplyWith(resp, http.StatusOK, api.DecryptKeyResponse{ + Plaintext: plaintext, + }) +} + +func (s *Server) describePolicy(resp *api.Response, req *api.Request) { + if !validName(req.Resource) { + resp.Failf(http.StatusBadRequest, "policy name '%s' is empty, too long or contains invalid characters", req.Resource) + return + } + + state := s.state.Load() + if _, ok := state.Policies[req.Resource]; !ok { + resp.Failr(kes.ErrPolicyNotFound) + return + } + api.ReplyWith(resp, http.StatusOK, api.DescribeKeyResponse{ + Name: req.Resource, + CreatedAt: state.StartTime, + CreatedBy: state.Admin.String(), + }) +} + +func (s *Server) readPolicy(resp *api.Response, req *api.Request) { + if !validName(req.Resource) { + resp.Failf(http.StatusBadRequest, "policy name '%s' is empty, too long or contains invalid characters", req.Resource) + return + } + + state := s.state.Load() + policy, ok := state.Policies[req.Resource] + if !ok { + resp.Failr(kes.ErrPolicyNotFound) + return + } + + allow := make(map[string]struct{}, len(policy.Allow)) + for p := range policy.Allow { + allow[p] = struct{}{} + } + deny := make(map[string]struct{}, len(policy.Deny)) + for p := range policy.Deny { + deny[p] = struct{}{} + } + + api.ReplyWith(resp, http.StatusOK, api.ReadPolicyResponse{ + Name: req.Resource, + Allow: allow, + Deny: deny, + CreatedAt: state.StartTime, + CreatedBy: state.Admin.String(), + }) +} + +func (s *Server) listPolicies(resp *api.Response, req *api.Request) { + if !validPattern(req.Resource) { + resp.Failf(http.StatusBadRequest, "listing pattern '%s' is empty, too long or is invalid", req.Resource) + return + } + + policies := s.state.Load().Policies + var names []string + if req.Resource == "" || req.Resource == "*" { // fast path + names = make([]string, 0, len(policies)) + for name := range policies { + names = append(names, name) + } + } else { + prefix := req.Resource + if prefix[len(prefix)-1] == '*' { + prefix = prefix[:len(prefix)-1] + } + + names = make([]string, 0, 1+len(policies)/10) // pre-alloc space for ~10% + for name := range policies { + if strings.HasPrefix(name, prefix) { + names = append(names, name) + } + } + } + slices.Sort(names) + + api.ReplyWith(resp, http.StatusOK, api.ListPoliciesResponse{ + Names: names, + }) +} + +func (s *Server) describeIdentity(resp *api.Response, req *api.Request) { + if !validName(req.Resource) { + resp.Failf(http.StatusBadRequest, "identity '%s' is empty, too long or contains invalid characters", req.Resource) + return + } + + state := s.state.Load() + identity := kes.Identity(req.Resource) + if identity == state.Admin { + api.ReplyWith(resp, http.StatusOK, api.DescribeIdentityResponse{ + IsAdmin: true, + CreatedAt: state.StartTime, + }) + return + } + + info, ok := state.Identities[kes.Identity(req.Resource)] + if !ok { + resp.Failr(kes.ErrIdentityNotFound) + return + } + api.ReplyWith(resp, http.StatusOK, api.DescribeIdentityResponse{ + Policy: info.Name, + CreatedAt: state.StartTime, + CreatedBy: state.Admin.String(), + }) +} + +func (s *Server) listIdentities(resp *api.Response, req *api.Request) { + if !validPattern(req.Resource) { + resp.Failf(http.StatusBadRequest, "listing pattern '%s' is empty, too long or is invalid", req.Resource) + return + } + + state := s.state.Load() + var ids []string + if req.Resource == "" || req.Resource == "*" { // fast path + ids = make([]string, 0, 1+len(state.Identities)) + ids = append(ids, state.Admin.String()) + for id := range state.Identities { + ids = append(ids, id.String()) + } + } else { + prefix := req.Resource + if prefix[len(prefix)-1] == '*' { + prefix = prefix[:len(prefix)-1] + } + + ids = make([]string, 0, 1+len(state.Identities)/10) // pre-alloc space for ~10% + if strings.HasPrefix(state.Admin.String(), prefix) { + ids = append(ids, state.Admin.String()) + } + for id := range state.Identities { + if strings.HasPrefix(id.String(), prefix) { + ids = append(ids, id.String()) + } + } + } + + slices.Sort(ids) + + api.ReplyWith(resp, http.StatusOK, api.ListIdentitiesResponse{ + Identities: ids, + }) +} + +func (s *Server) selfDescribeIdentity(resp *api.Response, req *api.Request) { + state := s.state.Load() + if req.Identity == state.Admin { + api.ReplyWith(resp, http.StatusOK, api.SelfDescribeIdentityResponse{ + Identity: req.Identity.String(), + IsAdmin: true, + CreatedAt: state.StartTime, + }) + return + } + + info, ok := state.Identities[kes.Identity(req.Resource)] + if !ok { + resp.Failr(kes.ErrIdentityNotFound) + return + } + + allow := make(map[string]struct{}, len(info.Allow)) + for p := range info.Allow { + allow[p] = struct{}{} + } + deny := make(map[string]struct{}, len(info.Deny)) + for p := range info.Deny { + deny[p] = struct{}{} + } + + api.ReplyWith(resp, http.StatusOK, api.SelfDescribeIdentityResponse{ + Identity: req.Identity.String(), + CreatedAt: state.StartTime, + CreatedBy: state.Admin.String(), + Policy: &api.ReadPolicyResponse{ + Name: info.Name, + Allow: allow, + Deny: deny, + CreatedAt: state.StartTime, + CreatedBy: state.Admin.String(), + }, + }) +} + +func (s *Server) logError(resp *api.Response, req *api.Request) { + resp.Header().Set(headers.ContentType, headers.ContentTypeJSONLines) + resp.WriteHeader(http.StatusOK) + + w := api.NewLogWriter(resp.ResponseWriter) + errLog := s.state.Load().LogHandler + errLog.out.Add(w) + defer errLog.out.Remove(w) + + <-req.Context().Done() +} + +func (s *Server) logAudit(resp *api.Response, req *api.Request) { + resp.Header().Set(headers.ContentType, headers.ContentTypeJSONLines) + resp.WriteHeader(http.StatusOK) + + auditLog := s.state.Load().Audit + auditLog.out.Add(resp.ResponseWriter) + defer auditLog.out.Remove(resp.ResponseWriter) + + <-req.Context().Done() +} diff --git a/server_test.go b/server_test.go new file mode 100644 index 00000000..bb16c682 --- /dev/null +++ b/server_test.go @@ -0,0 +1,169 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package kes + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "log/slog" + "net" + "sync" + "testing" + "time" + + "github.com/minio/kes-go" +) + +// Self-signed, valid from Oct. 10 2023 until Oct 10 2050 +const ( + srvCertificate = `-----BEGIN CERTIFICATE----- +MIIBlTCCATugAwIBAgIQVBb0Y6QgG4y/Uhsqr15ixDAKBggqhkjOPQQDAjAUMRIw +EAYDVQQDEwlsb2NhbGhvc3QwIBcNMjMxMDEwMDAwMDAwWhgPMjA1MDEwMTAwMDAw +MDBaMBQxEjAQBgNVBAMTCWxvY2FsaG9zdDBZMBMGByqGSM49AgEGCCqGSM49AwEH +A0IABGSF1/2rUFcQSfd1SY3jBF82BY0MH77fDn7+aR7V8L1M5joDHBqR+TAoqS04 +GVIFrMC9vKSYuNVx5Pn0hfQ+Z92jbTBrMA4GA1UdDwEB/wQEAwIChDAdBgNVHSUE +FjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAsBgNVHREEJTAj +gglsb2NhbGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwCgYIKoZIzj0EAwID +SAAwRQIhAPXQ9LRiCQZJruplDQnrRUt3OJxd9vhZQmmhbWC8zKMPAiB7sy46Fgrg +DB5wr8jkeZpC5Inb1yjbyoHOD6sfQUdm9g== +-----END CERTIFICATE-----` + + srvPrivateKey = `-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgj0xKJXLMx/S9dc5w +dJ9Dm4+lX7qYfHRNGoJiF+DAbtKhRANCAARkhdf9q1BXEEn3dUmN4wRfNgWNDB++ +3w5+/mke1fC9TOY6AxwakfkwKKktOBlSBazAvbykmLjVceT59IX0Pmfd +-----END PRIVATE KEY-----` +) + +const ( + defaultAPIKey = "kes:v1:AD9E7FSYWrMD+VjhI6q545cYT9YOyFxZb7UnjEepYDRc" + defaultIdentity = "3ecfcdf38fcbe141ae26a1030f81e96b753365a46760ae6b578698a97c59fd22" +) + +func startServer(ctx context.Context, conf *Config) (*Server, string) { + ln := newLocalListener() + + if conf == nil { + conf = &Config{} + } + if conf.Admin == "" { + conf.Admin = defaultIdentity + } + if conf.TLS == nil { + conf.TLS = &tls.Config{ + MinVersion: tls.VersionTLS12, + ClientAuth: tls.RequestClientCert, + Certificates: []tls.Certificate{defaultServerCertificate()}, + NextProtos: []string{"h2", "http/1.1"}, + } + } + if conf.Cache == nil { + conf.Cache = &CacheConfig{ + Expiry: 5 * time.Minute, + ExpiryUnused: 30 * time.Second, + ExpiryOffline: 0, + } + } + if conf.Keys == nil { + conf.Keys = &MemKeyStore{} + } + if conf.ErrorLog == nil { + conf.ErrorLog = discardLog{} + } + if conf.AuditLog == nil { + conf.AuditLog = discardAudit{} + } + + srv := &Server{ + ShutdownTimeout: -1, // wait for all requests to finish + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + wg.Done() + if err := srv.Start(ctx, ln, conf); err != nil { + panic(fmt.Sprintf("serve failed: %v", err)) + } + }() + wg.Wait() + + for srv.Addr() == "" { + time.Sleep(5 * time.Microsecond) + } + return srv, "https://" + ln.Addr().String() +} + +func testContext(t *testing.T) context.Context { + if deadline, ok := t.Deadline(); ok { + ctx, cancel := context.WithDeadline(context.Background(), deadline) + t.Cleanup(cancel) + return ctx + + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + return ctx +} + +func defaultClient(endpoint string) *kes.Client { + adminKey, err := kes.ParseAPIKey(defaultAPIKey) + if err != nil { + panic(fmt.Sprintf("kes: failed to parse API key '%s': %v", defaultAPIKey, err)) + } + clientCert, err := kes.GenerateCertificate(adminKey) + if err != nil { + panic(fmt.Sprintf("kes: failed to generate client certificate: %v", err)) + } + + rootCAs := x509.NewCertPool() + rootCAs.AddCert(defaultServerCertificate().Leaf) + + return kes.NewClientWithConfig(endpoint, &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: rootCAs, + GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return &clientCert, nil + }, + }) +} + +func newLocalListener() net.Listener { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + if l, err = net.Listen("tcp6", "[::1]:0"); err != nil { + panic(fmt.Sprintf("kes: failed to listen on a port: %v", err)) + } + } + return l +} + +func defaultServerCertificate() tls.Certificate { + cert, err := tls.X509KeyPair([]byte(srvCertificate), []byte(srvPrivateKey)) + if err != nil { + panic(fmt.Sprintf("kes: failed to parse server certificate: %v", err)) + } + cert.Leaf, _ = x509.ParseCertificate(cert.Certificate[0]) + return cert +} + +type discardLog struct{} + +func (discardLog) Enabled(context.Context, slog.Level) bool { return false } + +func (discardLog) Handle(context.Context, slog.Record) error { return nil } + +func (h discardLog) WithAttrs([]slog.Attr) slog.Handler { return h } + +func (h discardLog) WithGroup(string) slog.Handler { return h } + +type discardAudit struct{} + +func (discardAudit) Enabled(context.Context, slog.Level) bool { return false } + +func (discardAudit) Handle(context.Context, AuditRecord) error { return nil } diff --git a/state.go b/state.go new file mode 100644 index 00000000..5b5b43ab --- /dev/null +++ b/state.go @@ -0,0 +1,267 @@ +// Copyright 2023 - MinIO, Inc. All rights reserved. +// Use of this source code is governed by the AGPLv3 +// license that can be found in the LICENSE file. + +package kes + +import ( + "fmt" + "log/slog" + "maps" + "net" + "net/http" + "time" + + "aead.dev/mem" + "github.com/minio/kes-go" + "github.com/minio/kes/internal/api" + "github.com/minio/kes/internal/metric" +) + +type serverState struct { + Addr net.Addr + StartTime time.Time + + Admin kes.Identity + Keys *keyCache + Policies map[string]*kes.Policy + Identities map[kes.Identity]identityEntry + + Metrics *metric.Metrics + Routes map[string]api.Route + + LogHandler *logHandler + Log *slog.Logger + Audit *auditLogger +} + +type identityEntry struct { + Name string + *kes.Policy +} + +func initRoutes(s *Server, routeConfig map[string]RouteConfig) (*http.ServeMux, map[string]api.Route) { + routes := map[string]api.Route{ + api.PathVersion: { + Method: http.MethodGet, + Path: api.PathVersion, + MaxBody: 0, + Timeout: 10 * time.Second, + Auth: api.InsecureSkipVerify, + Handler: api.HandlerFunc(s.version), + }, + api.PathReady: { + Method: http.MethodGet, + Path: api.PathReady, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.ready), + }, + api.PathStatus: { + Method: http.MethodGet, + Path: api.PathStatus, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.status), + }, + api.PathMetrics: { + Method: http.MethodGet, + Path: api.PathMetrics, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.metrics), + }, + api.PathListAPIs: { + Method: http.MethodGet, + Path: api.PathListAPIs, + MaxBody: 0, + Timeout: 10 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.listAPIs), + }, + + api.PathKeyCreate: { + Method: http.MethodPut, + Path: api.PathKeyCreate, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.createKey), + }, + api.PathKeyImport: { + Method: http.MethodPut, + Path: api.PathKeyImport, + MaxBody: 1 * mem.MB, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.importKey), + }, + api.PathKeyDescribe: { + Method: http.MethodGet, + Path: api.PathKeyDescribe, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.describeKey), + }, + api.PathKeyList: { + Method: http.MethodGet, + Path: api.PathKeyList, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.listKeys), + }, + api.PathKeyDelete: { + Method: http.MethodDelete, + Path: api.PathKeyDelete, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.deleteKey), + }, + api.PathKeyEncrypt: { + Method: http.MethodPut, + Path: api.PathKeyEncrypt, + MaxBody: 1 * mem.MB, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.encryptKey), + }, + api.PathKeyGenerate: { + Method: http.MethodPut, + Path: api.PathKeyGenerate, + MaxBody: 1 * mem.MB, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.generateKey), + }, + api.PathKeyDecrypt: { + Method: http.MethodPut, + Path: api.PathKeyDecrypt, + MaxBody: 1 * mem.MB, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.decryptKey), + }, + + api.PathPolicyDescribe: { + Method: http.MethodGet, + Path: api.PathPolicyDescribe, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.describePolicy), + }, + api.PathPolicyRead: { + Method: http.MethodGet, + Path: api.PathPolicyRead, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.readPolicy), + }, + api.PathPolicyList: { + Method: http.MethodGet, + Path: api.PathPolicyList, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.listPolicies), + }, + + api.PathIdentityDescribe: { + Method: http.MethodGet, + Path: api.PathIdentityDescribe, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.describeIdentity), + }, + api.PathIdentityList: { + Method: http.MethodGet, + Path: api.PathIdentityList, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.listIdentities), + }, + api.PathIdentitySelfDescribe: { + Method: http.MethodGet, + Path: api.PathIdentitySelfDescribe, + MaxBody: 0, + Timeout: 15 * time.Second, + Auth: insecureIdentifyOnly{}, // Anyone can use the self-describe API as long as a client cert is provided + Handler: api.HandlerFunc(s.selfDescribeIdentity), + }, + + api.PathLogError: { + Method: http.MethodGet, + Path: api.PathLogError, + MaxBody: 0, + Timeout: 0, // No timeout + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.logError), + }, + api.PathLogAudit: { + Method: http.MethodGet, + Path: api.PathLogAudit, + MaxBody: 0, + Timeout: 0, // No timeout + Auth: (*verifyIdentity)(&s.state), + Handler: api.HandlerFunc(s.logAudit), + }, + } + + for path, conf := range routeConfig { // apply API customization + route, ok := routes[path] + if !ok { + continue + } + if conf.InsecureSkipAuth { + route.Auth = insecureIdentifyOnly{} + } + if conf.Timeout > 0 { + route.Timeout = conf.Timeout + } + routes[path] = route + } + + mux := http.NewServeMux() + for path, route := range routes { + mux.Handle(path, route) + } + return mux, routes +} + +func initPolicies(policies map[string]Policy) (map[string]*kes.Policy, map[kes.Identity]identityEntry, error) { + policySet := make(map[string]*kes.Policy, len(policies)) + identitySet := make(map[kes.Identity]identityEntry, len(policies)) + for name, policy := range policies { + if !validName(name) { + return nil, nil, fmt.Errorf("kes: policy name '%s' is empty, too long or contains invalid characters", name) + } + p := &kes.Policy{ + Allow: maps.Clone(policy.Allow), + Deny: maps.Clone(policy.Deny), + } + + policySet[name] = p + for _, id := range policy.Identities { + if !validName(id.String()) { + return nil, nil, fmt.Errorf("kes: identity '%s' is empty, too long or contains invalid characters", id) + } + if _, ok := identitySet[id]; ok { + return nil, nil, fmt.Errorf("kes: cannot assign policy '%s' to '%v': identity already has a policy", name, id) + } + identitySet[id] = identityEntry{ + Name: name, + Policy: p, + } + } + } + return policySet, identitySet, nil +}