From 18bf2400e0d6b037a3259f2b7b7cb048410de3e9 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 30 Oct 2023 10:56:24 -0400 Subject: [PATCH] [Fleet] Support output secrets (#3061) --- .../fragments/1698420960-output-secrets.yaml | 33 +++++++ internal/pkg/policy/parsed_policy.go | 7 ++ internal/pkg/policy/secret.go | 85 +++++++++++++++++++ internal/pkg/policy/secret_test.go | 65 ++++++++++++++ 4 files changed, 190 insertions(+) create mode 100644 changelog/fragments/1698420960-output-secrets.yaml diff --git a/changelog/fragments/1698420960-output-secrets.yaml b/changelog/fragments/1698420960-output-secrets.yaml new file mode 100644 index 000000000..cf133425c --- /dev/null +++ b/changelog/fragments/1698420960-output-secrets.yaml @@ -0,0 +1,33 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: feature + +# Change summary; a 80ish characters long description of the change. +summary: Support output secrets + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +description: | + Add support for outputs secrets, secrets are being stored in a separate index in elasticsearch + and replaced when processing the policy in fleet server. + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: "fleet-server" +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: 3061 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +issue: 2966 diff --git a/internal/pkg/policy/parsed_policy.go b/internal/pkg/policy/parsed_policy.go index 53bf53e85..9beb52698 100644 --- a/internal/pkg/policy/parsed_policy.go +++ b/internal/pkg/policy/parsed_policy.go @@ -19,6 +19,7 @@ import ( const ( FieldOutputs = "outputs" FieldOutputType = "type" + FieldOutputSecrets = "secrets" FieldOutputFleetServer = "fleet_server" FieldOutputServiceToken = "service_token" FieldOutputPermissions = "output_permissions" @@ -63,6 +64,12 @@ func NewParsedPolicy(ctx context.Context, bulker bulk.Bulk, p model.Policy) (*Pa if err != nil { return nil, err } + for _, policyOutput := range p.Data.Outputs { + err := processOutputSecret(ctx, policyOutput, bulker) + if err != nil { + return nil, err + } + } defaultName, err := findDefaultOutputName(p.Data.Outputs) if err != nil { return nil, err diff --git a/internal/pkg/policy/secret.go b/internal/pkg/policy/secret.go index d4c499309..dd79b7d1e 100644 --- a/internal/pkg/policy/secret.go +++ b/internal/pkg/policy/secret.go @@ -11,6 +11,7 @@ import ( "github.com/elastic/fleet-server/v7/internal/pkg/bulk" "github.com/elastic/fleet-server/v7/internal/pkg/model" + "github.com/elastic/fleet-server/v7/internal/pkg/smap" ) var ( @@ -77,6 +78,90 @@ func getPolicyInputsWithSecrets(ctx context.Context, data *model.PolicyData, bul return result, nil } +type OutputSecret struct { + Path []string + ID string +} + +func getSecretIDAndPath(secret smap.Map) ([]OutputSecret, error) { + outputSecrets := make([]OutputSecret, 0) + + secretID := secret.GetString("id") + if secretID != "" { + outputSecrets = append(outputSecrets, OutputSecret{ + Path: make([]string, 0), + ID: secretID, + }) + + return outputSecrets, nil + } + + for secretKey := range secret { + newOutputSecrets, err := getSecretIDAndPath(secret.GetMap(secretKey)) + if err != nil { + return nil, err + } + + for _, secret := range newOutputSecrets { + path := append([]string{secretKey}, secret.Path...) + outputSecrets = append(outputSecrets, OutputSecret{ + Path: path, + ID: secret.ID, + }) + } + } + + return outputSecrets, nil +} + +func setSecretPath(output smap.Map, secretValue string, secretPaths []string) error { + // Break the recursion + if len(secretPaths) == 1 { + output[secretPaths[0]] = secretValue + + return nil + } + path, secretPaths := secretPaths[0], secretPaths[1:] + + if output.GetMap(path) == nil { + output[path] = make(map[string]interface{}) + } + + return setSecretPath(output.GetMap(path), secretValue, secretPaths) +} + +// Read secret from output and mutate output with secret value +func processOutputSecret(ctx context.Context, output smap.Map, bulker bulk.Bulk) error { + secrets := output.GetMap(FieldOutputSecrets) + + delete(output, FieldOutputSecrets) + secretReferences := make([]model.SecretReferencesItems, 0) + outputSecrets, err := getSecretIDAndPath(secrets) + if err != nil { + return err + } + + for _, secret := range outputSecrets { + secretReferences = append(secretReferences, model.SecretReferencesItems{ + ID: secret.ID, + }) + } + if len(secretReferences) == 0 { + return nil + } + secretValues, err := getSecretValues(ctx, secretReferences, bulker) + if err != nil { + return err + } + for _, secret := range outputSecrets { + err = setSecretPath(output, secretValues[secret.ID], secret.Path) + if err != nil { + return err + } + } + return nil +} + func processStreams(streams []any, secretValues map[string]string) []any { newStreams := make([]any, 0) for _, stream := range streams { diff --git a/internal/pkg/policy/secret_test.go b/internal/pkg/policy/secret_test.go index bd4b58fa0..8163206c7 100644 --- a/internal/pkg/policy/secret_test.go +++ b/internal/pkg/policy/secret_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/elastic/fleet-server/v7/internal/pkg/model" + "github.com/elastic/fleet-server/v7/internal/pkg/smap" ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing" "github.com/stretchr/testify/assert" @@ -131,3 +132,67 @@ func TestGetPolicyInputsNoopWhenNoSecrets(t *testing.T) { assert.Equal(t, expectedResult, result) } + +func TestProcessOutputSecret(t *testing.T) { + tests := []struct { + name string + outputJSON string + expectOutputJSON string + }{ + { + name: "Output without secrets", + outputJSON: `{"password": "test"}`, + expectOutputJSON: `{"password": "test"}`, + }, + { + name: "Output with secrets", + outputJSON: `{ + "secrets": { + "password": {"id": "passwordid"} + } + }`, + expectOutputJSON: `{ + "password": "passwordid_value" + }`, + }, + { + name: "Output with nested secrets", + outputJSON: `{ + "secrets": { + "ssl": { "key" : { "id": "sslkey" } } + } + }`, + expectOutputJSON: `{ + "ssl": {"key": "sslkey_value"} + }`, + }, + { + name: "Output with multiple secrets", + outputJSON: `{ + "secrets": { + "ssl": { "key" : { "id": "sslkey" }, "other": {"id": "sslother"} } + } + }`, + expectOutputJSON: `{ + "ssl": {"key": "sslkey_value", "other": "sslother_value"} + }`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + bulker := ftesting.NewMockBulk() + output, err := smap.Parse([]byte(tc.outputJSON)) + assert.NoError(t, err) + + expectOutput, err := smap.Parse([]byte(tc.expectOutputJSON)) + assert.NoError(t, err) + + err = processOutputSecret(context.Background(), output, bulker) + assert.NoError(t, err) + + assert.Equal(t, expectOutput, output) + + }) + } +}