From 521b6eec30cb5f8e4a81307110760527d21fd154 Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Mon, 9 Sep 2024 16:52:07 +0300 Subject: [PATCH] [Automatic Import] Safely output the package manifest (#192316) ## Release note Fixes issues with rendering the package manifest in Automatic Import. ## Summary Previously the multiline output or special symbols in the user-provided strings, like description, were breaking YAML structure of the package manifest. The user would be confronted with a message like this, during the last step, after all the work of generating the integration was completed. The incorrect behavior can be observed in detail with a failing test in the first commit of the PR. In this PR, we change the manifest construction logic from template rendering into TypeScript code. As a result, all user-provided strings are correctly serialized. We keep as close as possible to the original manifest structure, also keeping the parameter names. --------- Co-authored-by: Elastic Machine --- .../build_integration.test.ts | 61 ++++++++++ .../integration_builder/build_integration.ts | 111 ++++++++++++++++-- .../manifest/package_manifest.yml.njk | 34 ------ 3 files changed, 159 insertions(+), 47 deletions(-) create mode 100644 x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts delete mode 100644 x-pack/plugins/integration_assistant/server/templates/manifest/package_manifest.yml.njk diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts new file mode 100644 index 0000000000000..b77f1fa77a1bb --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Integration } from '../../common'; +import { renderPackageManifestYAML } from './build_integration'; +import yaml from 'js-yaml'; + +describe('renderPackageManifestYAML', () => { + test('generates the package manifest correctly', () => { + const integration: Integration = { + title: 'Sample Integration', + name: 'sample-integration', + description: + ' This is a sample integration\n\nWith multiple lines and weird spacing. \n\n And more lines ', + logo: 'some-logo.png', + dataStreams: [ + { + name: 'data-stream-1', + title: 'Data Stream 1', + description: 'This is data stream 1', + inputTypes: ['filestream'], + rawSamples: ['{field: "value"}'], + pipeline: { + processors: [], + }, + docs: [], + samplesFormat: { name: 'ndjson', multiline: false }, + }, + { + name: 'data-stream-2', + title: 'Data Stream 2', + description: + 'This is data stream 2\nWith multiple lines of description\nBut otherwise, nothing special', + inputTypes: ['aws-cloudwatch'], + pipeline: { + processors: [], + }, + rawSamples: ['field="value"'], + docs: [], + samplesFormat: { name: 'structured' }, + }, + ], + }; + + const manifestContent = renderPackageManifestYAML(integration); + + // The manifest content must be parseable as YAML. + const manifest = yaml.safeLoad(manifestContent); + + expect(manifest).toBeDefined(); + expect(manifest.title).toBe(integration.title); + expect(manifest.name).toBe(integration.name); + expect(manifest.type).toBe('integration'); + expect(manifest.description).toBe(integration.description); + expect(manifest.icons).toBeTruthy(); + }); +}); diff --git a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts index 62972f6141c64..73113f6bf7b04 100644 --- a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts +++ b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts @@ -9,6 +9,7 @@ import AdmZip from 'adm-zip'; import nunjucks from 'nunjucks'; import { getDataPath } from '@kbn/utils'; import { join as joinPath } from 'path'; +import { safeDump } from 'js-yaml'; import type { DataStream, Integration } from '../../common'; import { createSync, ensureDirSync, generateUniqueId, removeDirSync } from '../util'; import { createAgentInput } from './agent'; @@ -18,7 +19,7 @@ import { createPipeline } from './pipeline'; const initialVersion = '1.0.0'; -export async function buildPackage(integration: Integration): Promise { +function configureNunjucks() { const templateDir = joinPath(__dirname, '../templates'); const agentTemplates = joinPath(templateDir, 'agent'); const manifestTemplates = joinPath(templateDir, 'manifest'); @@ -26,6 +27,10 @@ export async function buildPackage(integration: Integration): Promise { nunjucks.configure([templateDir, agentTemplates, manifestTemplates, systemTestTemplates], { autoescape: false, }); +} + +export async function buildPackage(integration: Integration): Promise { + configureNunjucks(); const workingDir = joinPath(getDataPath(), `integration-assistant-${generateUniqueId()}`); const packageDirectoryName = `${integration.name}-${initialVersion}`; @@ -116,7 +121,82 @@ async function createZipArchive(workingDir: string, packageDirectoryName: string return buffer; } -function createPackageManifest(packageDir: string, integration: Integration): void { +/* eslint-disable @typescript-eslint/naming-convention */ +/** + * Creates a package manifest dictionary. + * + * @param format_version - The format version of the package. + * @param package_title - The title of the package. + * @param package_name - The name of the package. + * @param package_version - The version of the package. + * @param package_description - The description of the package. + * @param package_logo - The package logo file name, if present. + * @param package_owner - The owner of the package. + * @param min_version - The minimum version of Kibana required for the package. + * @param inputs - An array of unique input objects containing type, title, and description. + * @returns The package manifest dictionary. + */ +function createPackageManifestDict( + format_version: string, + package_title: string, + package_name: string, + package_version: string, + package_description: string, + package_logo: string | undefined, + package_owner: string, + min_version: string, + inputs: Array<{ type: string; title: string; description: string }> +): { [key: string]: string | object } { + const data: { [key: string]: string | object } = { + format_version, + name: package_name, + title: package_title, + version: package_version, + description: package_description, + type: 'integration', + categories: ['security', 'iam'], + conditions: { + kibana: { + version: min_version, + }, + }, + policy_templates: [ + { + name: package_name, + title: package_title, + description: package_description, + inputs: inputs.map((input) => ({ + type: input.type, + title: `${input.title} : ${input.type}`, + description: input.description, + })), + }, + ], + owner: { + github: package_owner, + type: 'elastic', + }, + }; + + if (package_logo !== undefined && package_logo !== '') { + data.icons = { + src: '/img/logo.svg', + title: `${package_title} Logo`, + size: '32x32', + type: 'image/svg+xml', + }; + } + return data; +} +/* eslint-enable @typescript-eslint/naming-convention */ + +/** + * Render the package manifest for an integration. + * + * @param integration - The integration object. + * @returns The package manifest YAML rendered into a string. + */ +export function renderPackageManifestYAML(integration: Integration): string { const uniqueInputs: { [key: string]: { type: string; title: string; description: string } } = {}; integration.dataStreams.forEach((dataStream: DataStream) => { @@ -133,17 +213,22 @@ function createPackageManifest(packageDir: string, integration: Integration): vo const uniqueInputsList = Object.values(uniqueInputs); - const packageManifest = nunjucks.render('package_manifest.yml.njk', { - format_version: '3.1.4', - package_title: integration.title, - package_name: integration.name, - package_version: initialVersion, - package_description: integration.description, - package_logo: integration.logo, - package_owner: '@elastic/custom-integrations', - min_version: '^8.13.0', - inputs: uniqueInputsList, - }); + const packageData = createPackageManifestDict( + '3.1.4', // format_version + integration.title, // package_title + integration.name, // package_name + initialVersion, // package_version + integration.description, // package_description + integration.logo, // package_logo + '@elastic/custom-integrations', // package_owner + '^8.13.0', // min_version + uniqueInputsList // inputs + ); + + return safeDump(packageData); +} +function createPackageManifest(packageDir: string, integration: Integration): void { + const packageManifest = renderPackageManifestYAML(integration); createSync(joinPath(packageDir, 'manifest.yml'), packageManifest); } diff --git a/x-pack/plugins/integration_assistant/server/templates/manifest/package_manifest.yml.njk b/x-pack/plugins/integration_assistant/server/templates/manifest/package_manifest.yml.njk deleted file mode 100644 index fa7b8d8c66266..0000000000000 --- a/x-pack/plugins/integration_assistant/server/templates/manifest/package_manifest.yml.njk +++ /dev/null @@ -1,34 +0,0 @@ -format_version: "{{ format_version }}" -name: "{{ package_name }}" -title: | - {{ package_title }} -version: {{ package_version }} -description: | - {{ package_description }} -type: integration -categories: - - security - - iam -conditions: - kibana: - version: {{ min_version }} -{% if package_logo %}icons: - - src: /img/logo.svg - title: "{{ package_name }} Logo" - size: 32x32 - type: image/svg+xml{% endif %} -policy_templates: - - name: {{ package_name }} - title: | - {{ package_title }} - description: | - {{ package_description}} - inputs: {% for input in inputs %} - - type: {{ input.type }} - title: | - {{ input.title }} : {{ input.type }} - description: | - {{ input.description }} {% endfor %} -owner: - github: "{{ package_owner }}" - type: elastic \ No newline at end of file