From fb6494d61c9edf421dd0c9e10448bdc553bf726b Mon Sep 17 00:00:00 2001 From: Tiago Vila Verde Date: Wed, 30 Oct 2024 23:14:26 +0100 Subject: [PATCH 1/2] [Entity Analytics] [Entity Store] Add audit logs (#196847) ## Summary This PR adds audit logs for the different actions that can be performed on the entity store engines. (cherry picked from commit 6c6ae68ded65bf8955d56d8c2007fb9320385411) # Conflicts: # x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts --- .../entity_store/auditing/actions.ts | 17 ++ .../entity_store/auditing/resources.ts | 18 ++ .../entity_store/entity_store_data_client.ts | 179 ++++++++++++++---- .../entity_analytics/utils/entity_store.ts | 3 + 4 files changed, 175 insertions(+), 42 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/actions.ts create mode 100644 x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/resources.ts diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/actions.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/actions.ts new file mode 100644 index 0000000000000..63d594a9711a3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/actions.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const EntityEngineActions = { + INIT: 'init', + START: 'start', + STOP: 'stop', + CREATE: 'create', + DELETE: 'delete', + EXECUTE: 'execute', +} as const; + +export type EntityEngineActions = (typeof EntityEngineActions)[keyof typeof EntityEngineActions]; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/resources.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/resources.ts new file mode 100644 index 0000000000000..67d33fb42dc93 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/resources.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const EntityStoreResource = { + ENTITY_ENGINE: 'entity_engine', + ENTITY_DEFINITION: 'entity_definition', + ENTITY_INDEX: 'entity_index', + INDEX_COMPONENT_TEMPLATE: 'index_component_template', + PLATFORM_PIPELINE: 'platform_pipeline', + FIELD_RETENTION_ENRICH_POLICY: 'field_retention_enrich_policy', + FIELD_RETENTION_ENRICH_POLICY_TASK: 'field_retention_enrich_policy_task', +} as const; + +export type EntityStoreResource = (typeof EntityStoreResource)[keyof typeof EntityStoreResource]; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index 8c0c87acdc6b1..8264c262b415d 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -10,6 +10,7 @@ import type { ElasticsearchClient, SavedObjectsClientContract, AuditLogger, + AuditEvent, AnalyticsServiceSetup, } from '@kbn/core/server'; import { EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client'; @@ -55,12 +56,15 @@ import { isPromiseRejected, } from './utils'; +import { EntityEngineActions } from './auditing/actions'; +import { EntityStoreResource } from './auditing/resources'; +import { AUDIT_CATEGORY, AUDIT_OUTCOME, AUDIT_TYPE } from '../audit'; +import type { EntityRecord, EntityStoreConfig } from './types'; import { ENTITY_ENGINE_INITIALIZATION_EVENT, ENTITY_ENGINE_RESOURCE_INIT_FAILURE_EVENT, } from '../../telemetry/event_based/events'; -import type { EntityRecord, EntityStoreConfig } from './types'; import { CRITICALITY_VALUES } from '../asset_criticality/constants'; interface EntityStoreClientOpts { @@ -130,7 +134,7 @@ export class EntityStoreDataClient { throw new Error('Task Manager is not available'); } - const { logger, config } = this.options; + const { config } = this.options; await this.riskScoreDataClient.createRiskScoreLatestIndex(); @@ -142,8 +146,13 @@ export class EntityStoreDataClient { 'Asset criticality data migration is required before initializing entity store. If this error persists, please restart Kibana.' ); } - logger.info( - `In namespace ${this.options.namespace}: Initializing entity store for ${entityType}` + + this.log('info', entityType, `Initializing entity store`); + this.audit( + EntityEngineActions.INIT, + EntityStoreResource.ENTITY_ENGINE, + entityType, + 'Initializing entity engine' ); const descriptor = await this.engineClient.init(entityType, { @@ -151,9 +160,7 @@ export class EntityStoreDataClient { fieldHistoryLength, indexPattern, }); - logger.debug(`Initialized engine for ${entityType}`); - // first create the entity definition without starting it - // so that the index template is created which we can add a component template to + this.log('debug', entityType, `Initialized engine saved object`); this.asyncSetup( entityType, @@ -163,9 +170,9 @@ export class EntityStoreDataClient { filter, config, pipelineDebugMode - ).catch((error) => { - logger.error(`There was an error during async setup of the Entity Store: ${error.message}`); - }); + ).catch((e) => + this.log('error', entityType, `Error during async setup of entity store: ${e.message}`) + ); return descriptor; } @@ -193,9 +200,6 @@ export class EntityStoreDataClient { }); const { entityManagerDefinition } = unitedDefinition; - const debugLog = (message: string) => - logger.debug(`[Entity Engine] [${entityType}] ${message}`); - try { // clean up any existing entity store await this.delete(entityType, taskManager, { deleteData: false, deleteEngine: false }); @@ -211,7 +215,7 @@ export class EntityStoreDataClient { }, installOnly: true, }); - debugLog(`Created entity definition`); + this.log(`debug`, entityType, `Created entity definition`); // the index must be in place with the correct mapping before the enrich policy is created // this is because the enrich policy will fail if the index does not exist with the correct fields @@ -219,14 +223,14 @@ export class EntityStoreDataClient { unitedDefinition, esClient, }); - debugLog(`Created entity index component template`); + this.log(`debug`, entityType, `Created entity index component template`); await createEntityIndex({ entityType, esClient, namespace, logger, }); - debugLog(`Created entity index`); + this.log(`debug`, entityType, `Created entity index`); // we must create and execute the enrich policy before the pipeline is created // this is because the pipeline will fail if the enrich index does not exist @@ -234,24 +238,24 @@ export class EntityStoreDataClient { unitedDefinition, esClient, }); - debugLog(`Created field retention enrich policy`); + this.log(`debug`, entityType, `Created field retention enrich policy`); + await executeFieldRetentionEnrichPolicy({ unitedDefinition, esClient, logger, }); - debugLog(`Executed field retention enrich policy`); + this.log(`debug`, entityType, `Executed field retention enrich policy`); await createPlatformPipeline({ debugMode: pipelineDebugMode, unitedDefinition, logger, esClient, }); - debugLog(`Created @platform pipeline`); + this.log(`debug`, entityType, `Created @platform pipeline`); // finally start the entity definition now that everything is in place const updated = await this.start(entityType, { force: true }); - debugLog(`Started entity definition`); // the task will execute the enrich policy on a schedule await startEntityStoreFieldRetentionEnrichTask({ @@ -259,7 +263,9 @@ export class EntityStoreDataClient { logger, taskManager, }); - logger.info(`Entity store initialized for ${entityType}`); + + this.log(`debug`, entityType, `Started entity store field retention enrich task`); + this.log(`info`, entityType, `Entity store initialized`); const setupEndTime = moment().utc().toISOString(); const duration = moment(setupEndTime).diff(moment(setupStartTime), 'seconds'); @@ -269,8 +275,14 @@ export class EntityStoreDataClient { return updated; } catch (err) { - this.options.logger.error( - `Error initializing entity store for ${entityType}: ${err.message}` + this.log(`error`, entityType, `Error initializing entity store: ${err.message}`); + + this.audit( + EntityEngineActions.INIT, + EntityStoreResource.ENTITY_ENGINE, + entityType, + 'Failed to initialize entity engine resources', + err ); this.options.telemetry?.reportEvent(ENTITY_ENGINE_RESOURCE_INIT_FAILURE_EVENT.eventType, { @@ -300,41 +312,54 @@ export class EntityStoreDataClient { } public async start(entityType: EntityType, options?: { force: boolean }) { + const { namespace } = this.options; const descriptor = await this.engineClient.get(entityType); if (!options?.force && descriptor.status !== ENGINE_STATUS.STOPPED) { throw new Error( - `In namespace ${this.options.namespace}: Cannot start Entity engine for ${entityType} when current status is: ${descriptor.status}` + `In namespace ${namespace}: Cannot start Entity engine for ${entityType} when current status is: ${descriptor.status}` ); } - this.options.logger.info( - `In namespace ${this.options.namespace}: Starting entity store for ${entityType}` - ); + this.log('info', entityType, `Starting entity store`); // startEntityDefinition requires more fields than the engine descriptor // provides so we need to fetch the full entity definition const fullEntityDefinition = await this.getExistingEntityDefinition(entityType); + this.audit( + EntityEngineActions.START, + EntityStoreResource.ENTITY_DEFINITION, + entityType, + 'Starting entity definition' + ); await this.entityClient.startEntityDefinition(fullEntityDefinition); + this.log('debug', entityType, `Started entity definition`); return this.engineClient.update(entityType, ENGINE_STATUS.STARTED); } public async stop(entityType: EntityType) { + const { namespace } = this.options; const descriptor = await this.engineClient.get(entityType); if (descriptor.status !== ENGINE_STATUS.STARTED) { throw new Error( - `In namespace ${this.options.namespace}: Cannot stop Entity engine for ${entityType} when current status is: ${descriptor.status}` + `In namespace ${namespace}: Cannot stop Entity engine for ${entityType} when current status is: ${descriptor.status}` ); } - this.options.logger.info( - `In namespace ${this.options.namespace}: Stopping entity store for ${entityType}` - ); + this.log('info', entityType, `Stopping entity store`); + // stopEntityDefinition requires more fields than the engine descriptor // provides so we need to fetch the full entity definition const fullEntityDefinition = await this.getExistingEntityDefinition(entityType); + this.audit( + EntityEngineActions.STOP, + EntityStoreResource.ENTITY_DEFINITION, + entityType, + 'Stopping entity definition' + ); await this.entityClient.stopEntityDefinition(fullEntityDefinition); + this.log('debug', entityType, `Stopped entity definition`); return this.engineClient.update(entityType, ENGINE_STATUS.STOPPED); } @@ -367,30 +392,46 @@ export class EntityStoreDataClient { frequency: `${config.frequency.asSeconds()}s`, }); const { entityManagerDefinition } = unitedDefinition; - logger.info(`In namespace ${namespace}: Deleting entity store for ${entityType}`); + + this.log('info', entityType, `Deleting entity store`); + this.audit( + EntityEngineActions.DELETE, + EntityStoreResource.ENTITY_ENGINE, + entityType, + 'Deleting entity engine' + ); + try { - try { - await this.entityClient.deleteEntityDefinition({ + await this.entityClient + .deleteEntityDefinition({ id: entityManagerDefinition.id, deleteData, - }); - } catch (e) { - logger.warn(`Error deleting entity definition for ${entityType}: ${e.message}`); - } + }) + // Swallowing the error as it is expected to fail if no entity definition exists + .catch((e) => + this.log(`warn`, entityType, `Error deleting entity definition: ${e.message}`) + ); + this.log('debug', entityType, `Deleted entity definition`); + await deleteEntityIndexComponentTemplate({ unitedDefinition, esClient, }); + this.log('debug', entityType, `Deleted entity index component template`); + await deletePlatformPipeline({ unitedDefinition, logger, esClient, }); + this.log('debug', entityType, `Deleted platform pipeline`); + await deleteFieldRetentionEnrichPolicy({ unitedDefinition, esClient, logger, }); + this.log('debug', entityType, `Deleted field retention enrich policy`); if (deleteData) { await deleteEntityIndex({ @@ -399,6 +440,7 @@ export class EntityStoreDataClient { namespace, logger, }); + this.log('debug', entityType, `Deleted entity index`); } if (descriptor && deleteEngine) { @@ -412,13 +454,22 @@ export class EntityStoreDataClient { logger, taskManager, }); + this.log('debug', entityType, `Deleted entity store field retention enrich task`); } return { deleted: true }; - } catch (e) { - logger.error(`Error deleting entity store for ${entityType}: ${e.message}`); - // TODO: should we set the engine status to error here? - throw e; + } catch (err) { + this.log(`error`, entityType, `Error deleting entity store: ${err.message}`); + + this.audit( + EntityEngineActions.DELETE, + EntityStoreResource.ENTITY_ENGINE, + entityType, + 'Failed to delete entity engine', + err + ); + + throw err; } } @@ -546,4 +597,48 @@ export class EntityStoreDataClient { errors: updateErrors, }; } + + private log( + level: Exclude, + entityType: EntityType, + msg: string + ) { + this.options.logger[level]( + `[Entity Engine] [entity.${entityType}] [namespace: ${this.options.namespace}] ${msg}` + ); + } + + private audit( + action: EntityEngineActions, + resource: EntityStoreResource, + entityType: EntityType, + msg: string, + error?: Error + ) { + // NOTE: Excluding errors, all auditing events are currently WRITE events, meaning the outcome is always UNKNOWN. + // This may change in the future, depending on the audit action. + const outcome = error ? AUDIT_OUTCOME.FAILURE : AUDIT_OUTCOME.UNKNOWN; + + const type = + action === EntityEngineActions.CREATE + ? AUDIT_TYPE.CREATION + : EntityEngineActions.DELETE + ? AUDIT_TYPE.DELETION + : AUDIT_TYPE.CHANGE; + + const category = AUDIT_CATEGORY.DATABASE; + + const message = error ? `${msg}: ${error.message}` : msg; + const event: AuditEvent = { + message: `[Entity Engine] [entity.${entityType}] ${message}`, + event: { + action: `${action}_${entityType}_${resource}`, + category, + outcome, + type, + }, + }; + + return this.options.auditLogger?.log(event); + } } diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts index 029103425af68..7ee32e20640d6 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts @@ -82,6 +82,9 @@ export const EntityStoreUtils = ( if (body.engines.every((engine: any) => engine.status === 'started')) { return true; } + if (body.engines.some((engine: any) => engine.status === 'error')) { + throw new Error(`Engines not started: ${JSON.stringify(body)}`); + } return false; } ); From 3b54587f99152b414a2fa92308683dd0e6748c13 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:44:17 +0000 Subject: [PATCH 2/2] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../entity_analytics/entity_store/entity_store_data_client.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index 0231873367f7f..9c9dee1a41035 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -265,7 +265,6 @@ export class EntityStoreDataClient { this.log(`debug`, entityType, `Started entity store field retention enrich task`); this.log(`info`, entityType, `Entity store initialized`); - const setupEndTime = moment().utc().toISOString(); const duration = moment(setupEndTime).diff(moment(setupStartTime), 'seconds'); this.options.telemetry?.reportEvent(ENTITY_ENGINE_INITIALIZATION_EVENT.eventType, { @@ -274,7 +273,6 @@ export class EntityStoreDataClient { return updated; } catch (err) { - this.audit( EntityEngineActions.INIT, EntityStoreResource.ENTITY_ENGINE,