From a5d571a2a98ffdede6b9877149fa1e8f36bf820d Mon Sep 17 00:00:00 2001 From: Paul Tavares <56442535+paul-tavares@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:12:56 -0400 Subject: [PATCH] [Security Solution][Endpoint] Add FTR API tests that validates creation of DOT indices (#197899) ## Summary - Adds new FTR API test suite for validating that DOT indices are created whenever a policy in fleet is created/updated - Renamed and moved `DEFAULT_DIAGNOSTIC_INDEX` `const` to security solution top-level `common` directory for better reuse - Moved utility function that builds an index name with the `namespace` included to top-level `common` directory for better reuse - Created some additional scripting methods in the Fleet services module for updating fleet policies --- .../ftr_security_serverless_configs.yml | 1 + .buildkite/ftr_security_stateful_configs.yml | 1 + .../common/endpoint/constants.ts | 3 + .../utils/index_name_utilities.test.ts | 19 +++ .../endpoint/utils/index_name_utilities.ts | 34 +++++ .../scripts/endpoint/common/fleet_services.ts | 130 +++++++++++++++++ .../handlers/create_policy_datastreams.ts | 21 +-- .../lib/telemetry_helpers.ts | 6 +- .../server/lib/telemetry/constants.ts | 2 - .../server/lib/telemetry/receiver.ts | 4 +- .../telemetry/tasks/timelines_diagnostic.ts | 5 +- .../configs/ess.config.ts | 22 +++ .../configs/serverless.config.ts | 22 +++ .../datastream_index_creation.ts | 133 ++++++++++++++++++ .../trial_license_complete_tier/index.ts | 56 ++++++++ .../services/endpoint_policy.ts | 60 +++++--- 16 files changed, 470 insertions(+), 49 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/endpoint/utils/index_name_utilities.test.ts create mode 100644 x-pack/plugins/security_solution/common/endpoint/utils/index_name_utilities.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/configs/ess.config.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/configs/serverless.config.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/datastream_index_creation.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/index.ts diff --git a/.buildkite/ftr_security_serverless_configs.yml b/.buildkite/ftr_security_serverless_configs.yml index cdb66caea4be7..6b1c222382687 100644 --- a/.buildkite/ftr_security_serverless_configs.yml +++ b/.buildkite/ftr_security_serverless_configs.yml @@ -106,6 +106,7 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/metadata/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/package/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/spaces/trial_license_complete_tier/configs/serverless.config.ts diff --git a/.buildkite/ftr_security_stateful_configs.yml b/.buildkite/ftr_security_stateful_configs.yml index 8f780e081b11f..d3aadb1b7491d 100644 --- a/.buildkite/ftr_security_stateful_configs.yml +++ b/.buildkite/ftr_security_stateful_configs.yml @@ -93,6 +93,7 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/metadata/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/package/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/ess.config.ts - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/spaces/trial_license_complete_tier/configs/ess.config.ts diff --git a/x-pack/plugins/security_solution/common/endpoint/constants.ts b/x-pack/plugins/security_solution/common/endpoint/constants.ts index 534d7e5c2b8a4..19aa53eca6649 100644 --- a/x-pack/plugins/security_solution/common/endpoint/constants.ts +++ b/x-pack/plugins/security_solution/common/endpoint/constants.ts @@ -55,6 +55,9 @@ export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*'; export const ENDPOINT_HEARTBEAT_INDEX = '.logs-endpoint.heartbeat-default'; export const ENDPOINT_HEARTBEAT_INDEX_PATTERN = '.logs-endpoint.heartbeat-*'; +// Endpoint diagnostics index +export const DEFAULT_DIAGNOSTIC_INDEX_PATTERN = '.logs-endpoint.diagnostic.collection-*' as const; + // File storage indexes supporting endpoint Upload/download export const FILE_STORAGE_METADATA_INDEX = getFileMetadataIndexName('endpoint'); export const FILE_STORAGE_DATA_INDEX = getFileDataIndexName('endpoint'); diff --git a/x-pack/plugins/security_solution/common/endpoint/utils/index_name_utilities.test.ts b/x-pack/plugins/security_solution/common/endpoint/utils/index_name_utilities.test.ts new file mode 100644 index 0000000000000..25eb35bf88872 --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/utils/index_name_utilities.test.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildIndexNameWithNamespace } from './index_name_utilities'; + +describe('index name utilities', () => { + describe('buildIndexNameWithNamespace()', () => { + test.each(['logs-endpoint.foo', 'logs-endpoint.foo-', 'logs-endpoint.foo-*'])( + `should build correct index name for: %s`, + (prefix) => { + expect(buildIndexNameWithNamespace(prefix, 'bar')).toEqual('logs-endpoint.foo-bar'); + } + ); + }); +}); diff --git a/x-pack/plugins/security_solution/common/endpoint/utils/index_name_utilities.ts b/x-pack/plugins/security_solution/common/endpoint/utils/index_name_utilities.ts new file mode 100644 index 0000000000000..80db5392f653f --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/utils/index_name_utilities.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Builds an index name that includes the `namespace` using the provided index name prefix or pattern. + * + * @param indexNamePrefixOrPattern + * @param namespace + * + * @example + * + * buildIndexNameWithNamespace('logs-foo.bar-*', 'default'); // == 'logs-foo.bar-default' + * buildIndexNameWithNamespace('logs-foo.bar', 'default'); // == 'logs-foo.bar-default' + * buildIndexNameWithNamespace('logs-foo.bar-', 'default'); // == 'logs-foo.bar-default' + */ +export const buildIndexNameWithNamespace = ( + indexNamePrefixOrPattern: string, + namespace: string +): string => { + if (indexNamePrefixOrPattern.endsWith('*')) { + const hasDash = indexNamePrefixOrPattern.endsWith('-*'); + return `${indexNamePrefixOrPattern.substring(0, indexNamePrefixOrPattern.length - 1)}${ + hasDash ? '' : '-' + }${namespace}`; + } + + return `${indexNamePrefixOrPattern}${ + indexNamePrefixOrPattern.endsWith('-') ? '' : '-' + }${namespace}`; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index 92c13a521ed2c..a07823194fa69 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -21,10 +21,12 @@ import type { GetAgentsResponse, GetInfoResponse, GetOneAgentPolicyResponse, + GetOnePackagePolicyResponse, GetPackagePoliciesRequest, GetPackagePoliciesResponse, PackagePolicy, PostFleetSetupResponse, + UpdatePackagePolicyResponse, } from '@kbn/fleet-plugin/common'; import { AGENT_API_ROUTES, @@ -39,6 +41,7 @@ import { PACKAGE_POLICY_API_ROUTES, PACKAGE_POLICY_SAVED_OBJECT_TYPE, SETUP_API_ROUTE, + packagePolicyRouteService, } from '@kbn/fleet-plugin/common'; import type { ToolingLog } from '@kbn/tooling-log'; import type { KbnClient } from '@kbn/test'; @@ -57,11 +60,14 @@ import type { GetEnrollmentAPIKeysResponse, GetOutputsResponse, PostAgentUnenrollResponse, + UpdateAgentPolicyRequest, + UpdateAgentPolicyResponse, } from '@kbn/fleet-plugin/common/types'; import semver from 'semver'; import axios from 'axios'; import { userInfo } from 'os'; import pRetry from 'p-retry'; +import { getPolicyDataForUpdate } from '../../../common/endpoint/service/policy'; import { fetchActiveSpace } from './spaces'; import { fetchKibanaStatus } from '../../../common/endpoint/utils/kibana_status'; import { isFleetServerRunning } from './fleet_server/fleet_server_services'; @@ -76,6 +82,7 @@ import { } from '../../../common/endpoint/data_loaders/utils'; import { catchAxiosErrorFormatAndThrow } from '../../../common/endpoint/format_axios_error'; import { FleetAgentGenerator } from '../../../common/endpoint/data_generators/fleet_agent_generator'; +import type { PolicyData } from '../../../common/endpoint/types'; const fleetGenerator = new FleetAgentGenerator(); const CURRENT_USERNAME = userInfo().username.toLowerCase(); @@ -101,6 +108,39 @@ export const randomAgentPolicyName = (() => { */ const isValidArtifactVersion = (version: string) => !!version.match(/^\d+\.\d+\.\d+(-SNAPSHOT)?$/); +const getAgentPolicyDataForUpdate = ( + agentPolicy: AgentPolicy +): UpdateAgentPolicyRequest['body'] => { + return pick(agentPolicy, [ + 'advanced_settings', + 'agent_features', + 'data_output_id', + 'description', + 'download_source_id', + 'fleet_server_host_id', + 'global_data_tags', + 'has_fleet_server', + 'id', + 'inactivity_timeout', + 'is_default', + 'is_default_fleet_server', + 'is_managed', + 'is_protected', + 'keep_monitoring_alive', + 'monitoring_diagnostics', + 'monitoring_enabled', + 'monitoring_http', + 'monitoring_output_id', + 'monitoring_pprof_enabled', + 'name', + 'namespace', + 'overrides', + 'space_ids', + 'supports_agentless', + 'unenroll_timeout', + ]) as UpdateAgentPolicyRequest['body']; +}; + export const checkInFleetAgent = async ( esClient: Client, agentId: string, @@ -1369,3 +1409,93 @@ export const enableFleetSpaceAwareness = memoize(async (kbnClient: KbnClient): P }) .catch(catchAxiosErrorFormatAndThrow); }); + +/** + * Fetches a single integratino policy by id + * @param kbnClient + * @param policyId + */ +export const fetchIntegrationPolicy = async ( + kbnClient: KbnClient, + policyId: string +): Promise => { + return kbnClient + .request({ + path: packagePolicyRouteService.getInfoPath(policyId), + method: 'GET', + headers: { 'elastic-api-version': '2023-10-31' }, + }) + .catch(catchAxiosErrorFormatAndThrow) + .then((response) => response.data.item); +}; + +/** + * Update a fleet integration policy (aka: package policy) + * @param kbnClient + */ +export const updateIntegrationPolicy = async ( + kbnClient: KbnClient, + /** The Integration policy id */ + id: string, + policyData: Partial, + /** If set to `true`, then `policyData` can be a partial set of updates and not the full policy data */ + patch: boolean = false +): Promise => { + let fullPolicyData = policyData; + + if (patch) { + const currentSavedPolicy = await fetchIntegrationPolicy(kbnClient, id); + fullPolicyData = getPolicyDataForUpdate(currentSavedPolicy as PolicyData); + Object.assign(fullPolicyData, policyData); + } + + return kbnClient + .request({ + path: packagePolicyRouteService.getUpdatePath(id), + method: 'PUT', + body: fullPolicyData, + headers: { 'elastic-api-version': '2023-10-31' }, + }) + .catch(catchAxiosErrorFormatAndThrow) + .then((response) => response.data.item); +}; + +/** + * Updates a Fleet agent policy + * @param kbnClient + * @param id + * @param policyData + * @param patch + */ +export const updateAgentPolicy = async ( + kbnClient: KbnClient, + /** Fleet Agent Policy ID */ + id: string, + /** The updated agent policy data. Could be a `partial` update if `patch` arguments below is true */ + policyData: Partial, + /** + * If set to `true`, the `policyData` provided on input will first be merged with the latest version + * of the policy and then the updated applied + */ + patch: boolean = false +): Promise => { + let fullPolicyData = policyData; + + if (patch) { + const currentSavedPolicy = await fetchAgentPolicy(kbnClient, id); + + fullPolicyData = getAgentPolicyDataForUpdate(currentSavedPolicy); + delete fullPolicyData.id; + Object.assign(fullPolicyData, policyData); + } + + return kbnClient + .request({ + path: agentPolicyRouteService.getUpdatePath(id), + method: 'PUT', + body: fullPolicyData, + headers: { 'elastic-api-version': '2023-10-31' }, + }) + .catch(catchAxiosErrorFormatAndThrow) + .then((response) => response.data.item); +}; diff --git a/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts index a113c68e4132d..e94bc71cd4fa7 100644 --- a/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts +++ b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_datastreams.ts @@ -6,33 +6,18 @@ */ import pMap from 'p-map'; +import { buildIndexNameWithNamespace } from '../../../common/endpoint/utils/index_name_utilities'; import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; import { catchAndWrapError } from '../../endpoint/utils'; import type { SimpleMemCacheInterface } from '../../endpoint/lib/simple_mem_cache'; import { SimpleMemCache } from '../../endpoint/lib/simple_mem_cache'; import { + DEFAULT_DIAGNOSTIC_INDEX_PATTERN, ENDPOINT_ACTION_RESPONSES_DS, ENDPOINT_HEARTBEAT_INDEX_PATTERN, } from '../../../common/endpoint/constants'; -import { DEFAULT_DIAGNOSTIC_INDEX } from '../../lib/telemetry/constants'; import { stringify } from '../../endpoint/utils/stringify'; -const buildIndexNameWithNamespace = ( - indexNamePrefixOrPattern: string, - namespace: string -): string => { - if (indexNamePrefixOrPattern.endsWith('*')) { - const hasDash = indexNamePrefixOrPattern.endsWith('-*'); - return `${indexNamePrefixOrPattern.substring(0, indexNamePrefixOrPattern.length - 1)}${ - hasDash ? '' : '-' - }${namespace}`; - } - - return `${indexNamePrefixOrPattern}${ - indexNamePrefixOrPattern.endsWith('-') ? '' : '-' - }${namespace}`; -}; - const cache = new SimpleMemCache({ // Cache of created Datastreams last for 12h, at which point it is checked again. // This is just a safeguard case (for whatever reason) the index is deleted @@ -81,7 +66,7 @@ export const createPolicyDataStreamsIfNeeded: PolicyDataStreamsCreator = async ( const indicesToCreate: string[] = Array.from( Object.values(policyNamespaces.integrationPolicy).reduce>((acc, namespaceList) => { for (const namespace of namespaceList) { - acc.add(buildIndexNameWithNamespace(DEFAULT_DIAGNOSTIC_INDEX, namespace)); + acc.add(buildIndexNameWithNamespace(DEFAULT_DIAGNOSTIC_INDEX_PATTERN, namespace)); acc.add(buildIndexNameWithNamespace(ENDPOINT_ACTION_RESPONSES_DS, namespace)); if (endpointServices.isServerless()) { diff --git a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts index d83ff1e910ca5..a36d5e1be38de 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/lib/telemetry_helpers.ts @@ -30,6 +30,7 @@ import { packagePolicyService } from '@kbn/fleet-plugin/server/services'; import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants'; import { DETECTION_TYPE, NAMESPACE_TYPE } from '@kbn/lists-plugin/common/constants.mock'; +import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants'; import { bulkInsert, updateTimestamps } from './helpers'; import { TelemetryEventsSender } from '../../lib/telemetry/sender'; import type { @@ -40,7 +41,6 @@ import type { SecurityTelemetryTask } from '../../lib/telemetry/task'; import { Plugin as SecuritySolutionPlugin } from '../../plugin'; import { AsyncTelemetryEventsSender } from '../../lib/telemetry/async_sender'; import { type ITelemetryReceiver, TelemetryReceiver } from '../../lib/telemetry/receiver'; -import { DEFAULT_DIAGNOSTIC_INDEX } from '../../lib/telemetry/constants'; import mockEndpointAlert from '../__mocks__/endpoint-alert.json'; import mockedRule from '../__mocks__/rule.json'; import fleetAgents from '../__mocks__/fleet-agents.json'; @@ -147,7 +147,7 @@ export function getTelemetryTask( } export async function createMockedEndpointAlert(esClient: ElasticsearchClient) { - const index = `${DEFAULT_DIAGNOSTIC_INDEX.replace('-*', '')}-001`; + const index = `${DEFAULT_DIAGNOSTIC_INDEX_PATTERN.replace('-*', '')}-001`; await esClient.indices.create({ index, body: { settings: { hidden: true } } }); @@ -223,7 +223,7 @@ export async function dropEndpointIndices(esClient: ElasticsearchClient) { } export async function cleanupMockedEndpointAlerts(esClient: ElasticsearchClient) { - const index = `${DEFAULT_DIAGNOSTIC_INDEX.replace('-*', '')}-001`; + const index = `${DEFAULT_DIAGNOSTIC_INDEX_PATTERN.replace('-*', '')}-001`; await esClient.indices.delete({ index }).catch(() => { // ignore errors diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts b/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts index 8e50e4590a72f..50e0e0be47cdd 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts @@ -27,8 +27,6 @@ export const INSIGHTS_CHANNEL = 'security-insights-v1'; export const TASK_METRICS_CHANNEL = 'task-metrics'; -export const DEFAULT_DIAGNOSTIC_INDEX = '.logs-endpoint.diagnostic.collection-*' as const; - export const DEFAULT_ADVANCED_POLICY_CONFIG_SETTINGS = { linux: { advanced: { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts index 22f85d19c83d8..4d2ff971eeb62 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -48,6 +48,7 @@ import type { } from '@kbn/fleet-plugin/server'; import type { ExceptionListClient } from '@kbn/lists-plugin/server'; import moment from 'moment'; +import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../common/endpoint/constants'; import type { ExperimentalFeatures } from '../../../common'; import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; import { @@ -85,7 +86,6 @@ import type { import { telemetryConfiguration } from './configuration'; import { ENDPOINT_METRICS_INDEX } from '../../../common/constants'; import { PREBUILT_RULES_PACKAGE_NAME } from '../../../common/detection_engine/constants'; -import { DEFAULT_DIAGNOSTIC_INDEX } from './constants'; import type { TelemetryLogger } from './telemetry_logger'; export interface ITelemetryReceiver { @@ -546,7 +546,7 @@ export class TelemetryReceiver implements ITelemetryReceiver { to: executeTo, } as LogMeta); - let pitId = await this.openPointInTime(DEFAULT_DIAGNOSTIC_INDEX); + let pitId = await this.openPointInTime(DEFAULT_DIAGNOSTIC_INDEX_PATTERN); let fetchMore = true; let searchAfter: SortResults | undefined; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts index 35f5abeac10af..ec401a093c348 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/timelines_diagnostic.ts @@ -6,11 +6,12 @@ */ import type { Logger } from '@kbn/core/server'; +import { DEFAULT_DIAGNOSTIC_INDEX_PATTERN } from '../../../../common/endpoint/constants'; import type { ITelemetryEventsSender } from '../sender'; import type { ITelemetryReceiver } from '../receiver'; import type { TaskExecutionPeriod } from '../task'; import type { ITaskMetricsService } from '../task_metrics.types'; -import { DEFAULT_DIAGNOSTIC_INDEX, TELEMETRY_CHANNEL_TIMELINE } from '../constants'; +import { TELEMETRY_CHANNEL_TIMELINE } from '../constants'; import { ranges, TelemetryTimelineFetcher, newTelemetryLogger } from '../helpers'; export function createTelemetryDiagnosticTimelineTaskConfig() { @@ -43,7 +44,7 @@ export function createTelemetryDiagnosticTimelineTaskConfig() { const { rangeFrom, rangeTo } = ranges(taskExecutionPeriod); const alerts = await receiver.fetchTimelineAlerts( - DEFAULT_DIAGNOSTIC_INDEX, + DEFAULT_DIAGNOSTIC_INDEX_PATTERN, rangeFrom, rangeTo ); diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/configs/ess.config.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/configs/ess.config.ts new file mode 100644 index 0000000000000..bee23188b35a7 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/configs/ess.config.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile( + require.resolve('../../../../../config/ess/config.base.edr_workflows.trial') + ); + + return { + ...functionalConfig.getAll(), + testFiles: [require.resolve('..')], + junit: { + reportName: 'EDR Workflows API - Policy Tests - ESS Env - Trial License', + }, + }; +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/configs/serverless.config.ts new file mode 100644 index 0000000000000..ea2e2ce875ad1 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/configs/serverless.config.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile( + require.resolve('../../../../../config/serverless/config.base.edr_workflows') + ); + + return { + ...functionalConfig.getAll(), + testFiles: [require.resolve('..')], + junit: { + reportName: 'EDR Workflows API - Policy Tests - Serverless Env - Complete', + }, + }; +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/datastream_index_creation.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/datastream_index_creation.ts new file mode 100644 index 0000000000000..bc029369f3fe4 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/datastream_index_creation.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { + DEFAULT_DIAGNOSTIC_INDEX_PATTERN, + ENDPOINT_ACTION_RESPONSES_DS, + ENDPOINT_HEARTBEAT_INDEX_PATTERN, +} from '@kbn/security-solution-plugin/common/endpoint/constants'; +import { buildIndexNameWithNamespace } from '@kbn/security-solution-plugin/common/endpoint/utils/index_name_utilities'; +import { + updateAgentPolicy, + updateIntegrationPolicy, +} from '@kbn/security-solution-plugin/scripts/endpoint/common/fleet_services'; +import { PolicyTestResourceInfo } from '../../../../../security_solution_endpoint/services/endpoint_policy'; +import { FtrProviderContext } from '../../../../ftr_provider_context_edr_workflows'; + +export default function ({ getService }: FtrProviderContext) { + const endpointPolicyTestResources = getService('endpointPolicyTestResources'); + const esClient = getService('es'); + const kbnClient = getService('kibanaServer'); + const log = getService('log'); + const retry = getService('retry'); + const config = getService('config'); + const isServerless = config.get('serverless'); + + // FIXME:PT Remove @skipInServerlessMKI and enable it for MKI + describe('@ess @serverless @skipInServerlessMKI Creation of DOT indices for elastic defend policies', function () { + let testData: PolicyTestResourceInfo; + + const getExpectedIndexList = (namespace: string): string[] => { + const indexList = [ + buildIndexNameWithNamespace(ENDPOINT_ACTION_RESPONSES_DS, namespace), + buildIndexNameWithNamespace(DEFAULT_DIAGNOSTIC_INDEX_PATTERN, namespace), + ]; + + if (isServerless) { + indexList.push(buildIndexNameWithNamespace(ENDPOINT_HEARTBEAT_INDEX_PATTERN, namespace)); + } + + return indexList; + }; + + beforeEach(async () => { + testData = await endpointPolicyTestResources.createPolicy({ + // Endpoint policy inherits namespace from Agent policy + integrationPolicyOverrides: { namespace: undefined }, + }); + }); + + afterEach(async () => { + if (testData) { + await testData.cleanup(); + // @ts-expect-error + testData = undefined; + } + }); + + it('should create indices when endpoint integration policy is created', async () => { + for (const indexName of getExpectedIndexList('default')) { + log.debug(`Checking that [${indexName}] exists`); + // The creation of the indices is done in the background and may not be done by + // the time the API call that created/updated the policy in fleet is returned - thus + // we use retry logic below + await retry.try(async () => { + expect(await esClient.indices.exists({ index: indexName })).to.be(true); + }); + } + }); + + it('should create new indices when endpoint policy is updated with new namespace', async () => { + const namespace = Math.random().toString(32).substring(2); + await updateIntegrationPolicy(kbnClient, testData.packagePolicy.id, { namespace }, true); + + for (const indexName of getExpectedIndexList(namespace)) { + log.debug(`Checking that [${indexName}] exists`); + await retry.try(async () => { + expect(await esClient.indices.exists({ index: indexName })).to.be(true); + }); + } + }); + + it('should create new indices when agent policy is updated with new namespace', async () => { + const namespace = Math.random().toString(32).substring(2); + await updateAgentPolicy(kbnClient, testData.agentPolicy.id, { namespace }, true); + + for (const indexName of getExpectedIndexList(namespace)) { + log.debug(`Checking that [${indexName}] exists`); + await retry.try(async () => { + expect(await esClient.indices.exists({ index: indexName })).to.be(true); + }); + } + }); + + it('should NOT create indices when agent policy is updated if endpoint policy explicitly has a namespace defined', async () => { + const namespace = Math.random().toString(32).substring(2); + await updateIntegrationPolicy(kbnClient, testData.packagePolicy.id, { namespace }, true); + + for (const indexName of getExpectedIndexList(namespace)) { + log.debug(`Checking that [${indexName}] exists`); + await retry.try(async () => { + expect(await esClient.indices.exists({ index: indexName })).to.be(true); + }); + } + + // Now update agent policy with new namespace and check that indices are NOT created + const namespace2 = Math.random().toString(32).substring(2); + await updateAgentPolicy(kbnClient, testData.agentPolicy.id, { namespace: namespace2 }, true); + + // Delay just a few seconds to ensure policy index logic has executed + await new Promise(async (resolve, reject) => { + try { + setTimeout(() => { + resolve(undefined); + }, 5000); + } catch (error) { + reject(error); + } + }); + + for (const indexName of getExpectedIndexList(namespace2)) { + log.debug(`Checking that [${indexName}] does NOT exists`); + await retry.try(async () => { + expect(await esClient.indices.exists({ index: indexName })).to.be(false); + }); + } + }); + }); +} diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/index.ts new file mode 100644 index 0000000000000..dba8ed191925e --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy/trial_license_complete_tier/index.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { getRegistryUrl as getRegistryUrlFromIngest } from '@kbn/fleet-plugin/server'; +import { isServerlessKibanaFlavor } from '@kbn/security-solution-plugin/common/endpoint/utils/kibana_status'; +import { FtrProviderContext } from '../../../../ftr_provider_context_edr_workflows'; +import { ROLE } from '../../../../config/services/security_solution_edr_workflows_roles_users'; + +export default function endpointAPIIntegrationTests(providerContext: FtrProviderContext) { + const { loadTestFile, getService } = providerContext; + + describe('Endpoint Policy', function () { + const ingestManager = getService('ingestManager'); + const rolesUsersProvider = getService('rolesUsersProvider'); + const kbnClient = getService('kibanaServer'); + const log = getService('log'); + const endpointRegistryHelpers = getService('endpointRegistryHelpers'); + + const roles = Object.values(ROLE); + before(async () => { + if (!endpointRegistryHelpers.isRegistryEnabled()) { + log.warning('These tests are being run with an external package registry'); + } + + const registryUrl = + endpointRegistryHelpers.getRegistryUrlFromTestEnv() ?? getRegistryUrlFromIngest(); + log.info(`Package registry URL for tests: ${registryUrl}`); + try { + await ingestManager.setup(); + } catch (err) { + log.warning(`Error setting up ingestManager: ${err}`); + } + + if (!(await isServerlessKibanaFlavor(kbnClient))) { + // create role/user + for (const role of roles) { + await rolesUsersProvider.createRole({ predefinedRole: role }); + await rolesUsersProvider.createUser({ name: role, roles: [role] }); + } + } + }); + + after(async () => { + if (!(await isServerlessKibanaFlavor(kbnClient))) { + // delete role/user + await rolesUsersProvider.deleteUsers(roles); + await rolesUsersProvider.deleteRoles(roles); + } + }); + + loadTestFile(require.resolve('./datastream_index_creation')); + }); +} diff --git a/x-pack/test/security_solution_endpoint/services/endpoint_policy.ts b/x-pack/test/security_solution_endpoint/services/endpoint_policy.ts index 335328db08ce4..8f24f116ca875 100644 --- a/x-pack/test/security_solution_endpoint/services/endpoint_policy.ts +++ b/x-pack/test/security_solution_endpoint/services/endpoint_policy.ts @@ -27,19 +27,19 @@ import { pkgKeyFromPackageInfo } from '@kbn/fleet-plugin/public/services/pkg_key import { EndpointError } from '@kbn/security-solution-plugin/common/endpoint/errors'; import { FtrProviderContext } from '../configs/ftr_provider_context'; -const INGEST_API_ROOT = '/api/fleet'; -const INGEST_API_AGENT_POLICIES = `${INGEST_API_ROOT}/agent_policies`; -const INGEST_API_AGENT_POLICIES_DELETE = `${INGEST_API_AGENT_POLICIES}/delete`; -const INGEST_API_PACKAGE_POLICIES = `${INGEST_API_ROOT}/package_policies`; -const INGEST_API_PACKAGE_POLICIES_DELETE = `${INGEST_API_PACKAGE_POLICIES}/delete`; +const FLEET_API_ROOT = '/api/fleet'; +const FLEET_API_AGENT_POLICIES = `${FLEET_API_ROOT}/agent_policies`; +const FLEET_API_AGENT_POLICIES_DELETE = `${FLEET_API_AGENT_POLICIES}/delete`; +const FLEET_API_PACKAGE_POLICIES = `${FLEET_API_ROOT}/package_policies`; +const FLEET_API_PACKAGE_POLICIES_DELETE = `${FLEET_API_PACKAGE_POLICIES}/delete`; /** * Holds information about the test resources created to support an Endpoint Policy */ export interface PolicyTestResourceInfo { - /** The Ingest agent policy created */ + /** The Fleet agent policy created */ agentPolicy: Immutable; - /** The Ingest Package Policy created and added to agent policy. + /** The Fleet Package Policy created and added to agent policy. * This is where Endpoint Policy is stored. */ packagePolicy: Immutable; @@ -87,7 +87,7 @@ export function EndpointPolicyTestResourcesProvider({ getService }: FtrProviderC .expect(200) .catch((error) => { return logSupertestApiErrorAndThrow( - `Unable to retrieve Endpoint package via Ingest!`, + `Unable to retrieve Endpoint package via Fleet!`, error ); }) @@ -139,7 +139,7 @@ export function EndpointPolicyTestResourcesProvider({ getService }: FtrProviderC let fullAgentPolicy: GetFullAgentPolicyResponse['item']; try { const apiResponse: { body: GetFullAgentPolicyResponse } = await supertest - .get(`${INGEST_API_AGENT_POLICIES}/${agentPolicyId}/full`) + .get(`${FLEET_API_AGENT_POLICIES}/${agentPolicyId}/full`) .expect(200); fullAgentPolicy = apiResponse.body.item; @@ -151,10 +151,16 @@ export function EndpointPolicyTestResourcesProvider({ getService }: FtrProviderC }, /** - * Creates an Ingest Agent policy and adds to it the Endpoint Package Policy that + * Creates a Fleet Agent policy and adds to it the Endpoint Package Policy that * stores the Policy configuration data */ - async createPolicy(): Promise { + async createPolicy({ + agentPolicyOverrides = {}, + integrationPolicyOverrides = {}, + }: Partial<{ + agentPolicyOverrides: Partial; + integrationPolicyOverrides: Partial; + }> = {}): Promise { // create Agent Policy let agentPolicy: CreateAgentPolicyResponse['item']; try { @@ -162,15 +168,16 @@ export function EndpointPolicyTestResourcesProvider({ getService }: FtrProviderC name: `East Coast ${uuidv4()}`, description: 'East Coast call center', namespace: 'default', + ...agentPolicyOverrides, }; const { body: createResponse }: { body: CreateAgentPolicyResponse } = await supertest - .post(INGEST_API_AGENT_POLICIES) + .post(FLEET_API_AGENT_POLICIES) .set('kbn-xsrf', 'xxx') .send(newAgentPolicyData) .expect(200); agentPolicy = createResponse.item; } catch (error) { - return logSupertestApiErrorAndThrow(`Unable to create Agent Policy via Ingest!`, error); + return logSupertestApiErrorAndThrow(`Unable to create Agent Policy via Fleet!`, error); } // Retrieve the Endpoint package information @@ -210,15 +217,16 @@ export function EndpointPolicyTestResourcesProvider({ getService }: FtrProviderC title: endpointPackageInfo?.title ?? '', version: endpointPackageInfo?.version ?? '', }, + ...integrationPolicyOverrides, }; const { body: createResponse }: { body: CreatePackagePolicyResponse } = await supertest - .post(INGEST_API_PACKAGE_POLICIES) + .post(FLEET_API_PACKAGE_POLICIES) .set('kbn-xsrf', 'xxx') .send(newPackagePolicyData) .expect(200); packagePolicy = createResponse.item; } catch (error) { - return logSupertestApiErrorAndThrow(`Unable to create Package Policy via Ingest!`, error); + return logSupertestApiErrorAndThrow(`Unable to create Package Policy via Fleet!`, error); } log.info( @@ -237,12 +245,16 @@ export function EndpointPolicyTestResourcesProvider({ getService }: FtrProviderC packagePolicyIds: [packagePolicy.id], }; await supertest - .post(INGEST_API_PACKAGE_POLICIES_DELETE) + .post(FLEET_API_PACKAGE_POLICIES_DELETE) .set('kbn-xsrf', 'xxx') .send(deletePackagePolicyData) .expect(200); + log.info(`Fleet Endpoint integration policy deleted: ${packagePolicy.id}`); } catch (error) { - logSupertestApiErrorAndThrow('Unable to delete Package Policy via Ingest!', error); + logSupertestApiErrorAndThrow( + `Unable to delete Endpoint Integration Policy [${packagePolicy.id}] via Fleet!`, + error + ); } // Delete Agent Policy @@ -251,12 +263,16 @@ export function EndpointPolicyTestResourcesProvider({ getService }: FtrProviderC agentPolicyId: agentPolicy.id, }; await supertest - .post(INGEST_API_AGENT_POLICIES_DELETE) + .post(FLEET_API_AGENT_POLICIES_DELETE) .set('kbn-xsrf', 'xxx') .send(deleteAgentPolicyData) .expect(200); + log.info(`Fleet Agent policy deleted: ${agentPolicy.id}`); } catch (error) { - logSupertestApiErrorAndThrow('Unable to delete Agent Policy via Ingest!', error); + logSupertestApiErrorAndThrow( + `Unable to delete Agent Policy [${agentPolicy.id}] via Fleet!`, + error + ); } }, }; @@ -271,7 +287,7 @@ export function EndpointPolicyTestResourcesProvider({ getService }: FtrProviderC try { const { body: packagePoliciesResponse }: { body: GetPackagePoliciesResponse } = await supertest - .get(INGEST_API_PACKAGE_POLICIES) + .get(FLEET_API_PACKAGE_POLICIES) .set('kbn-xsrf', 'xxx') .query({ kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.name: ${name}` }) .send() @@ -297,12 +313,12 @@ export function EndpointPolicyTestResourcesProvider({ getService }: FtrProviderC packagePolicyIds: [packagePolicyList[0].id], }; await supertest - .post(INGEST_API_PACKAGE_POLICIES_DELETE) + .post(FLEET_API_PACKAGE_POLICIES_DELETE) .set('kbn-xsrf', 'xxx') .send(deletePackagePolicyData) .expect(200); } catch (error) { - logSupertestApiErrorAndThrow('Unable to delete Package Policy via Ingest!', error); + logSupertestApiErrorAndThrow('Unable to delete Package Policy via Fleet!', error); } }, };