[Fleet] making service_token an output secret (elastic#171875)

## Summary Related to elastic#104986 Making remote ES output's service_token a secret. fleet-server change here: elastic/fleet-server#3051 (comment) Steps to verify: - Enable remote ES output and output secrets in `kibana.dev.yml` locally: ``` xpack.fleet.enableExperimental: ['remoteESOutput', 'outputSecretsStorage'] ``` - Start es, kibana, fleet-server locally and start a second es locally - see detailed steps here: elastic/fleet-server#3051 - Create a remote ES output, verify that the service_token is stored as a secret reference ``` GET .kibana_ingest/_search?q=type:ingest-outputs ``` - Verify that the enrolled agent sends data to the remote ES successfully <img width="561" alt="image" src="https://github.com/elastic/kibana/assets/90178898/122d9800-a2ec-47f8-97a7-acf64b87172a"> <img width="549" alt="image" src="https://github.com/elastic/kibana/assets/90178898/e1751bdd-5aaf-4f68-9f92-7076b306cdfe"> ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
jpdjere · Nov 28, 2023 · 517c815 · 517c815
1 parent 5031231
commit 517c815
Show file tree

Hide file tree

Showing 21 changed files with 347 additions and 46 deletions.
diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json
@@ -1844,6 +1844,14 @@
                 }
               }
             }
+          },
+          "service_token": {
+            "dynamic": false,
+            "properties": {
+              "id": {
+                "type": "keyword"
+              }
+            }
           }
         }
       }

diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@@ -106,7 +106,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4",
         "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437",
         "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d",
-        "ingest-outputs": "8546f1123ec30dcbd6f238f72729c5f1656a4d9b",
+        "ingest-outputs": "4dd3cb38a91c848df95336a24a5abde2c8560fd1",
         "ingest-package-policies": "f4c2767e852b700a8b82678925b86bac08958b43",
         "ingest_manager_settings": "64955ef1b7a9ffa894d4bb9cf863b5602bfa6885",
         "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83",

diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json
@@ -8257,6 +8257,50 @@
           "type"
         ]
       },
+      "output_create_request_remote_elasticsearch": {
+        "title": "remote_elasticsearch",
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string"
+          },
+          "is_default": {
+            "type": "boolean"
+          },
+          "is_default_monitoring": {
+            "type": "boolean"
+          },
+          "name": {
+            "type": "string"
+          },
+          "type": {
+            "type": "string",
+            "enum": [
+              "remote_elasticsearch"
+            ]
+          },
+          "hosts": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "service_token": {
+            "type": "string"
+          },
+          "secrets": {
+            "type": "object",
+            "properties": {
+              "service_token": {
+                "type": "string"
+              }
+            }
+          }
+        },
+        "required": [
+          "name"
+        ]
+      },
       "output_create_request": {
         "title": "Output",
         "oneOf": [
@@ -8268,14 +8312,18 @@
           },
           {
             "$ref": "#/components/schemas/output_create_request_logstash"
+          },
+          {
+            "$ref": "#/components/schemas/output_create_request_remote_elasticsearch"
           }
         ],
         "discriminator": {
           "propertyName": "type",
           "mapping": {
             "elasticsearch": "#/components/schemas/output_create_request_elasticsearch",
             "kafka": "#/components/schemas/output_create_request_kafka",
-            "logstash": "#/components/schemas/output_create_request_logstash"
+            "logstash": "#/components/schemas/output_create_request_logstash",
+            "remote_elasticsearch": "#/components/schemas/output_create_request_remote_elasticsearch"
           }
         }
       },

diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml
@@ -5331,18 +5331,49 @@ components:
         - name
         - hosts
         - type
+    output_create_request_remote_elasticsearch:
+      title: remote_elasticsearch
+      type: object
+      properties:
+        id:
+          type: string
+        is_default:
+          type: boolean
+        is_default_monitoring:
+          type: boolean
+        name:
+          type: string
+        type:
+          type: string
+          enum:
+            - remote_elasticsearch
+        hosts:
+          type: array
+          items:
+            type: string
+        service_token:
+          type: string
+        secrets:
+          type: object
+          properties:
+            service_token:
+              type: string
+      required:
+        - name
     output_create_request:
       title: Output
       oneOf:
         - $ref: '#/components/schemas/output_create_request_elasticsearch'
         - $ref: '#/components/schemas/output_create_request_kafka'
         - $ref: '#/components/schemas/output_create_request_logstash'
+        - $ref: '#/components/schemas/output_create_request_remote_elasticsearch'
       discriminator:
         propertyName: type
         mapping:
           elasticsearch: '#/components/schemas/output_create_request_elasticsearch'
           kafka: '#/components/schemas/output_create_request_kafka'
           logstash: '#/components/schemas/output_create_request_logstash'
+          remote_elasticsearch: '#/components/schemas/output_create_request_remote_elasticsearch'
     output_update_request_elasticsearch:
       title: elasticsearch
       type: object

diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output_create_request.yaml
@@ -3,9 +3,11 @@ oneOf:
   - $ref: './output_create_request_elasticsearch.yaml'
   - $ref: './output_create_request_kafka.yaml'
   - $ref: './output_create_request_logstash.yaml'
+  - $ref: './output_create_request_remote_elasticsearch.yaml'
 discriminator:
   propertyName: type
   mapping:
     elasticsearch: './output_create_request_elasticsearch.yaml'
     kafka: './output_create_request_kafka.yaml'
     logstash: './output_create_request_logstash.yaml'
+    remote_elasticsearch: './output_create_request_remote_elasticsearch.yaml'
diff --git a/...s/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml b/...s/fleet/common/openapi/components/schemas/output_create_request_remote_elasticsearch.yaml
@@ -0,0 +1,27 @@
+title: remote_elasticsearch
+type: object
+properties:
+  id:
+    type: string
+  is_default:
+    type: boolean
+  is_default_monitoring:
+    type: boolean
+  name:
+    type: string
+  type:
+    type: string
+    enum: ['remote_elasticsearch']
+  hosts:
+    type: array
+    items:
+      type: string
+  service_token:
+    type: string
+  secrets:
+      type: object
+      properties:
+        service_token:
+          type: string
+required:
+  - name
diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts
@@ -43,15 +43,7 @@ interface NewBaseOutput {
   proxy_id?: string | null;
   shipper?: ShipperOutput | null;
   allow_edit?: string[];
-  secrets?: {
-    ssl?: {
-      key?:
-        | string
-        | {
-            id: string;
-          };
-    };
-  };
+  secrets?: {};
 }
 
 export interface NewElasticsearchOutput extends NewBaseOutput {
@@ -61,10 +53,26 @@ export interface NewElasticsearchOutput extends NewBaseOutput {
 export interface NewRemoteElasticsearchOutput extends NewBaseOutput {
   type: OutputType['RemoteElasticsearch'];
   service_token?: string;
+  secrets?: {
+    service_token?:
+      | string
+      | {
+          id: string;
+        };
+  };
 }
 
 export interface NewLogstashOutput extends NewBaseOutput {
   type: OutputType['Logstash'];
+  secrets?: {
+    ssl?: {
+      key?:
+        | string
+        | {
+            id: string;
+          };
+    };
+  };
 }
 
 export type NewOutput =

diff --git a/.../public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx b/.../public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx
@@ -191,7 +191,9 @@ describe('EditOutputFlyout', () => {
   });
 
   it('should render the flyout if the output provided is a remote ES output', async () => {
-    jest.spyOn(ExperimentalFeaturesService, 'get').mockReturnValue({ remoteESOutput: true });
+    jest
+      .spyOn(ExperimentalFeaturesService, 'get')
+      .mockReturnValue({ remoteESOutput: true, outputSecretsStorage: true });
     const { utils } = renderFlyout({
       type: 'remote_elasticsearch',
       name: 'remote es output',
@@ -208,6 +210,8 @@ describe('EditOutputFlyout', () => {
     expect(utils.queryByTestId('settingsOutputsFlyout.typeInput')?.textContent).toContain(
       'Remote Elasticsearch'
     );
+
+    expect(utils.queryByTestId('serviceTokenSecretInput')).not.toBeNull();
   });
 
   it('should not display remote ES output in type lists if serverless', async () => {

diff --git a/...fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx b/...fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx
@@ -272,7 +272,13 @@ export const EditOutputFlyout: React.FunctionComponent<EditOutputFlyoutProps> =
 
   const renderRemoteElasticsearchSection = () => {
     if (isRemoteESOutputEnabled) {
-      return <OutputFormRemoteEsSection inputs={inputs} />;
+      return (
+        <OutputFormRemoteEsSection
+          inputs={inputs}
+          useSecretsStorage={useSecretsStorage}
+          onUsePlainText={onUsePlainText}
+        />
+      );
     }
     return null;
   };

diff --git a/...lications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx b/...lications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx
@@ -13,13 +13,16 @@ import { i18n } from '@kbn/i18n';
 import { MultiRowInput } from '../multi_row_input';
 
 import type { OutputFormInputsType } from './use_output_form';
+import { SecretFormRow } from './output_form_secret_form_row';
 
 interface Props {
   inputs: OutputFormInputsType;
+  useSecretsStorage: boolean;
+  onUsePlainText: () => void;
 }
 
 export const OutputFormRemoteEsSection: React.FunctionComponent<Props> = (props) => {
-  const { inputs } = props;
+  const { inputs, useSecretsStorage, onUsePlainText } = props;
 
   return (
     <>
@@ -38,27 +41,50 @@ export const OutputFormRemoteEsSection: React.FunctionComponent<Props> = (props)
         isUrl
       />
       <EuiSpacer size="m" />
-      <EuiFormRow
-        fullWidth
-        label={
-          <FormattedMessage
-            id="xpack.fleet.settings.editOutputFlyout.serviceTokenLabel"
-            defaultMessage="Service Token"
+      {inputs.serviceTokenInput.value || !useSecretsStorage ? (
+        <EuiFormRow
+          fullWidth
+          label={
+            <FormattedMessage
+              id="xpack.fleet.settings.editOutputFlyout.serviceTokenLabel"
+              defaultMessage="Service Token"
+            />
+          }
+          {...inputs.serviceTokenInput.formRowProps}
+        >
+          <EuiFieldText
+            fullWidth
+            {...inputs.serviceTokenInput.props}
+            placeholder={i18n.translate(
+              'xpack.fleet.settings.editOutputFlyout.remoteESHostPlaceholder',
+              {
+                defaultMessage: 'Specify service token',
+              }
+            )}
           />
-        }
-        {...inputs.serviceTokenInput.formRowProps}
-      >
-        <EuiFieldText
+        </EuiFormRow>
+      ) : (
+        <SecretFormRow
           fullWidth
-          {...inputs.serviceTokenInput.props}
-          placeholder={i18n.translate(
-            'xpack.fleet.settings.editOutputFlyout.remoteESHostPlaceholder',
-            {
-              defaultMessage: 'Specify service token',
-            }
-          )}
-        />
-      </EuiFormRow>
+          title={i18n.translate('xpack.fleet.settings.editOutputFlyout.serviceTokenLabel', {
+            defaultMessage: 'Service Token',
+          })}
+          {...inputs.serviceTokenSecretInput.formRowProps}
+          onUsePlainText={onUsePlainText}
+        >
+          <EuiFieldText
+            data-test-subj="serviceTokenSecretInput"
+            fullWidth
+            {...inputs.serviceTokenSecretInput.props}
+            placeholder={i18n.translate(
+              'xpack.fleet.settings.editOutputFlyout.remoteESHostPlaceholder',
+              {
+                defaultMessage: 'Specify service token',
+              }
+            )}
+          />
+        </SecretFormRow>
+      )}
       <EuiSpacer size="m" />
       <EuiCallOut
         title={

diff --git a/...ications/fleet/sections/settings/components/edit_output_flyout/output_form_validators.tsx b/...ications/fleet/sections/settings/components/edit_output_flyout/output_form_validators.tsx
@@ -270,6 +270,8 @@ export function validateServiceToken(value: string) {
   }
 }
 
+export const validateServiceTokenSecret = toSecretValidator(validateServiceToken);
+
 export function validateSSLCertificate(value: string) {
   if (!value || value === '') {
     return [