[Fleet] Support output secrets (#3061)

changelog/fragments/1698420960-output-secrets.yaml

@@ -0,0 +1,33 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Support output secrets
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+  Add support for outputs secrets, secrets are being stored in a separate index in elasticsearch
+  and replaced when processing the policy in fleet server.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: "fleet-server"
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: 3061
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: 2966

internal/pkg/policy/parsed_policy.go

@@ -19,6 +19,7 @@ import (
 const (
 	FieldOutputs            = "outputs"
 	FieldOutputType         = "type"
+	FieldOutputSecrets      = "secrets"
 	FieldOutputFleetServer  = "fleet_server"
 	FieldOutputServiceToken = "service_token"
 	FieldOutputPermissions  = "output_permissions"
@@ -63,6 +64,12 @@ func NewParsedPolicy(ctx context.Context, bulker bulk.Bulk, p model.Policy) (*Pa
 	if err != nil {
 		return nil, err
 	}
+	for _, policyOutput := range p.Data.Outputs {
+		err := processOutputSecret(ctx, policyOutput, bulker)
+		if err != nil {
+			return nil, err
+		}
+	}
 	defaultName, err := findDefaultOutputName(p.Data.Outputs)
 	if err != nil {
 		return nil, err

internal/pkg/policy/secret.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
 	"github.com/elastic/fleet-server/v7/internal/pkg/model"
+	"github.com/elastic/fleet-server/v7/internal/pkg/smap"
 )
 
 var (
@@ -77,6 +78,90 @@ func getPolicyInputsWithSecrets(ctx context.Context, data *model.PolicyData, bul
 	return result, nil
 }
 
+type OutputSecret struct {
+	Path []string
+	ID   string
+}
+
+func getSecretIDAndPath(secret smap.Map) ([]OutputSecret, error) {
+	outputSecrets := make([]OutputSecret, 0)
+
+	secretID := secret.GetString("id")
+	if secretID != "" {
+		outputSecrets = append(outputSecrets, OutputSecret{
+			Path: make([]string, 0),
+			ID:   secretID,
+		})
+
+		return outputSecrets, nil
+	}
+
+	for secretKey := range secret {
+		newOutputSecrets, err := getSecretIDAndPath(secret.GetMap(secretKey))
+		if err != nil {
+			return nil, err
+		}
+
+		for _, secret := range newOutputSecrets {
+			path := append([]string{secretKey}, secret.Path...)
+			outputSecrets = append(outputSecrets, OutputSecret{
+				Path: path,
+				ID:   secret.ID,
+			})
+		}
+	}
+
+	return outputSecrets, nil
+}
+
+func setSecretPath(output smap.Map, secretValue string, secretPaths []string) error {
+	// Break the recursion
+	if len(secretPaths) == 1 {
+		output[secretPaths[0]] = secretValue
+
+		return nil
+	}
+	path, secretPaths := secretPaths[0], secretPaths[1:]
+
+	if output.GetMap(path) == nil {
+		output[path] = make(map[string]interface{})
+	}
+
+	return setSecretPath(output.GetMap(path), secretValue, secretPaths)
+}
+
+// Read secret from output and mutate output with secret value
+func processOutputSecret(ctx context.Context, output smap.Map, bulker bulk.Bulk) error {
+	secrets := output.GetMap(FieldOutputSecrets)
+
+	delete(output, FieldOutputSecrets)
+	secretReferences := make([]model.SecretReferencesItems, 0)
+	outputSecrets, err := getSecretIDAndPath(secrets)
+	if err != nil {
+		return err
+	}
+
+	for _, secret := range outputSecrets {
+		secretReferences = append(secretReferences, model.SecretReferencesItems{
+			ID: secret.ID,
+		})
+	}
+	if len(secretReferences) == 0 {
+		return nil
+	}
+	secretValues, err := getSecretValues(ctx, secretReferences, bulker)
+	if err != nil {
+		return err
+	}
+	for _, secret := range outputSecrets {
+		err = setSecretPath(output, secretValues[secret.ID], secret.Path)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func processStreams(streams []any, secretValues map[string]string) []any {
 	newStreams := make([]any, 0)
 	for _, stream := range streams {

internal/pkg/policy/secret_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 
 	"github.com/elastic/fleet-server/v7/internal/pkg/model"
+	"github.com/elastic/fleet-server/v7/internal/pkg/smap"
 	ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing"
 
 	"github.com/stretchr/testify/assert"
@@ -131,3 +132,67 @@ func TestGetPolicyInputsNoopWhenNoSecrets(t *testing.T) {
 
 	assert.Equal(t, expectedResult, result)
 }
+
+func TestProcessOutputSecret(t *testing.T) {
+	tests := []struct {
+		name             string
+		outputJSON       string
+		expectOutputJSON string
+	}{
+		{
+			name:             "Output without secrets",
+			outputJSON:       `{"password": "test"}`,
+			expectOutputJSON: `{"password": "test"}`,
+		},
+		{
+			name: "Output with secrets",
+			outputJSON: `{
+				"secrets": {
+					"password": {"id": "passwordid"}	
+				}
+			}`,
+			expectOutputJSON: `{
+				"password": "passwordid_value"
+			}`,
+		},
+		{
+			name: "Output with nested secrets",
+			outputJSON: `{
+				"secrets": {
+					"ssl": { "key" : { "id": "sslkey" }	}
+				}
+			}`,
+			expectOutputJSON: `{
+				"ssl": {"key": "sslkey_value"}
+			}`,
+		},
+		{
+			name: "Output with multiple secrets",
+			outputJSON: `{
+				"secrets": {
+					"ssl": { "key" : { "id": "sslkey" }, "other": {"id": "sslother"}	}
+				}
+			}`,
+			expectOutputJSON: `{
+				"ssl": {"key": "sslkey_value", "other": "sslother_value"}
+			}`,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			bulker := ftesting.NewMockBulk()
+			output, err := smap.Parse([]byte(tc.outputJSON))
+			assert.NoError(t, err)
+
+			expectOutput, err := smap.Parse([]byte(tc.expectOutputJSON))
+			assert.NoError(t, err)
+
+			err = processOutputSecret(context.Background(), output, bulker)
+			assert.NoError(t, err)
+
+			assert.Equal(t, expectOutput, output)
+
+		})
+	}
+}