[Entity Analytics] [Entity Store] Add audit logs

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/actions.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const EntityEngineActions = {
+  INIT: 'init',
+  START: 'start',
+  STOP: 'stop',
+  CREATE: 'create',
+  DELETE: 'delete',
+  EXECUTE: 'execute',
+} as const;
+
+export type EntityEngineActions = (typeof EntityEngineActions)[keyof typeof EntityEngineActions];

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/resources.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const EntityStoreResource = {
+  ENTITY_ENGINE: 'entity_engine',
+  ENTITY_DEFINITION: 'entity_definition',
+  ENTITY_INDEX: 'entity_index',
+  INDEX_COMPONENT_TEMPLATE: 'index_component_template',
+  PLATFORM_PIPELINE: 'platform_pipeline',
+  FIELD_RETENTION_ENRICH_POLICY: 'field_retention_enrich_policy',
+  FIELD_RETENTION_ENRICH_POLICY_TASK: 'field_retention_enrich_policy_task',
+} as const;
+
+export type EntityStoreResource = (typeof EntityStoreResource)[keyof typeof EntityStoreResource];

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts

@@ -11,6 +11,7 @@ import type {
   SavedObjectsClientContract,
   AuditLogger,
   IScopedClusterClient,
+  AuditEvent,
 } from '@kbn/core/server';
 import { EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client';
 import type { SortOrder } from '@elastic/elasticsearch/lib/api/types';
@@ -53,6 +54,11 @@ import {
   isPromiseFulfilled,
   isPromiseRejected,
 } from './utils';
+
+import { EntityEngineActions } from './auditing/actions';
+import { EntityStoreResource } from './auditing/resources';
+import { AUDIT_CATEGORY, AUDIT_OUTCOME, AUDIT_TYPE } from '../audit';
+
 import type { EntityRecord } from './types';
 import { CRITICALITY_VALUES } from '../asset_criticality/constants';
 
@@ -123,8 +129,6 @@ export class EntityStoreDataClient {
       throw new Error('Task Manager is not available');
     }
 
-    const { logger } = this.options;
-
     await this.riskScoreDataClient.createRiskScoreLatestIndex();
 
     const requiresMigration =
@@ -135,18 +139,22 @@ export class EntityStoreDataClient {
         'Asset criticality data migration is required before initializing entity store. If this error persists, please restart Kibana.'
       );
     }
-    logger.info(
-      `In namespace ${this.options.namespace}: Initializing entity store for ${entityType}`
+
+    this.log('info', entityType, `Initializing entity store`);
+    this.audit(
+      EntityEngineActions.INIT,
+      EntityStoreResource.ENTITY_ENGINE,
+      entityType,
+      'Initializing entity engine'
     );
 
     const descriptor = await this.engineClient.init(entityType, {
       filter,
       fieldHistoryLength,
       indexPattern,
     });
-    logger.debug(`Initialized engine for ${entityType}`);
-    // first create the entity definition without starting it
-    // so that the index template is created which we can add a component template to
+
+    this.log('debug', entityType, `Initialized engine`);
 
     this.asyncSetup(
       entityType,
@@ -155,9 +163,9 @@ export class EntityStoreDataClient {
       indexPattern,
       filter,
       pipelineDebugMode
-    ).catch((error) => {
-      logger.error(`There was an error during async setup of the Entity Store: ${error}`);
-    });
+    ).catch((e) =>
+      this.log('error', entityType, `Error during async setup of entity store: ${e.message}`)
+    );
 
     return descriptor;
   }
@@ -181,9 +189,6 @@ export class EntityStoreDataClient {
     });
     const { entityManagerDefinition } = unitedDefinition;
 
-    const debugLog = (message: string) =>
-      logger.debug(`[Entity Engine] [${entityType}] ${message}`);
-
     try {
       // clean up any existing entity store
       await this.delete(entityType, taskManager, { deleteData: false, deleteEngine: false });
@@ -199,60 +204,67 @@ export class EntityStoreDataClient {
         },
         installOnly: true,
       });
-      debugLog(`Created entity definition`);
+      this.log(`debug`, entityType, `Created entity definition`);
 
       // the index must be in place with the correct mapping before the enrich policy is created
       // this is because the enrich policy will fail if the index does not exist with the correct fields
       await createEntityIndexComponentTemplate({
         unitedDefinition,
         esClient: this.esClient,
       });
-      debugLog(`Created entity index component template`);
+      this.log(`debug`, entityType, `Created entity index component template`);
       await createEntityIndex({
         entityType,
         esClient: this.esClient,
         namespace,
         logger,
       });
-      debugLog(`Created entity index`);
+      this.log(`debug`, entityType, `Created entity index`);
 
       // we must create and execute the enrich policy before the pipeline is created
       // this is because the pipeline will fail if the enrich index does not exist
       await createFieldRetentionEnrichPolicy({
         unitedDefinition,
         esClient: this.esClient,
       });
-      debugLog(`Created field retention enrich policy`);
+      this.log(`debug`, entityType, `Created field retention enrich policy`);
+
       await executeFieldRetentionEnrichPolicy({
         unitedDefinition,
         esClient: this.esClient,
         logger,
       });
-      debugLog(`Executed field retention enrich policy`);
+      this.log(`debug`, entityType, `Executed field retention enrich policy`);
       await createPlatformPipeline({
         debugMode: pipelineDebugMode,
         unitedDefinition,
         logger,
         esClient: this.esClient,
       });
-      debugLog(`Created @platform pipeline`);
+      this.log(`debug`, entityType, `Created @platform pipeline`);
 
       // finally start the entity definition now that everything is in place
       const updated = await this.start(entityType, { force: true });
-      debugLog(`Started entity definition`);
 
       // the task will execute the enrich policy on a schedule
       await startEntityStoreFieldRetentionEnrichTask({
         namespace,
         logger,
         taskManager,
       });
-      logger.info(`Entity store initialized for ${entityType}`);
+      this.log(`debug`, entityType, `Started entity store field retention enrich task`);
+      this.log(`info`, entityType, `Entity store initialized`);
 
       return updated;
     } catch (err) {
-      this.options.logger.error(
-        `Error initializing entity store for ${entityType}: ${err.message}`
+      this.log(`error`, entityType, `Error initializing entity store: ${err.message}`);
+
+      this.audit(
+        EntityEngineActions.INIT,
+        EntityStoreResource.ENTITY_ENGINE,
+        entityType,
+        'Failed to initialize entity engine resources',
+        err
       );
 
       await this.engineClient.update(entityType, ENGINE_STATUS.ERROR);
@@ -278,41 +290,54 @@ export class EntityStoreDataClient {
   }
 
   public async start(entityType: EntityType, options?: { force: boolean }) {
+    const { namespace } = this.options;
     const descriptor = await this.engineClient.get(entityType);
     if (!options?.force && descriptor.status !== ENGINE_STATUS.STOPPED) {
       throw new Error(
-        `In namespace ${this.options.namespace}: Cannot start Entity engine for ${entityType} when current status is: ${descriptor.status}`
+        `In namespace ${namespace}: Cannot start Entity engine for ${entityType} when current status is: ${descriptor.status}`
       );
     }
 
-    this.options.logger.info(
-      `In namespace ${this.options.namespace}: Starting entity store for ${entityType}`
-    );
+    this.log('info', entityType, `Starting entity store`);
 
     // startEntityDefinition requires more fields than the engine descriptor
     // provides so we need to fetch the full entity definition
     const fullEntityDefinition = await this.getExistingEntityDefinition(entityType);
+    this.audit(
+      EntityEngineActions.START,
+      EntityStoreResource.ENTITY_DEFINITION,
+      entityType,
+      'Starting entity definition'
+    );
     await this.entityClient.startEntityDefinition(fullEntityDefinition);
+    this.log('debug', entityType, `Started entity definition`);
 
     return this.engineClient.update(entityType, ENGINE_STATUS.STARTED);
   }
 
   public async stop(entityType: EntityType) {
+    const { namespace } = this.options;
     const descriptor = await this.engineClient.get(entityType);
 
     if (descriptor.status !== ENGINE_STATUS.STARTED) {
       throw new Error(
-        `In namespace ${this.options.namespace}: Cannot stop Entity engine for ${entityType} when current status is: ${descriptor.status}`
+        `In namespace ${namespace}: Cannot stop Entity engine for ${entityType} when current status is: ${descriptor.status}`
       );
     }
 
-    this.options.logger.info(
-      `In namespace ${this.options.namespace}: Stopping entity store for ${entityType}`
-    );
+    this.log('info', entityType, `Stopping entity store`);
+
     // stopEntityDefinition requires more fields than the engine descriptor
     // provides so we need to fetch the full entity definition
     const fullEntityDefinition = await this.getExistingEntityDefinition(entityType);
+    this.audit(
+      EntityEngineActions.STOP,
+      EntityStoreResource.ENTITY_DEFINITION,
+      entityType,
+      'Stopping entity definition'
+    );
     await this.entityClient.stopEntityDefinition(fullEntityDefinition);
+    this.log('debug', entityType, `Stopped entity definition`);
 
     return this.engineClient.update(entityType, ENGINE_STATUS.STOPPED);
   }
@@ -342,30 +367,46 @@ export class EntityStoreDataClient {
       fieldHistoryLength: descriptor?.fieldHistoryLength ?? 10,
     });
     const { entityManagerDefinition } = unitedDefinition;
-    logger.info(`In namespace ${namespace}: Deleting entity store for ${entityType}`);
+
+    this.log('info', entityType, `Deleting entity store`);
+    this.audit(
+      EntityEngineActions.DELETE,
+      EntityStoreResource.ENTITY_ENGINE,
+      entityType,
+      'Deleting entity engine'
+    );
+
     try {
-      try {
-        await this.entityClient.deleteEntityDefinition({
+      await this.entityClient
+        .deleteEntityDefinition({
           id: entityManagerDefinition.id,
           deleteData,
-        });
-      } catch (e) {
-        logger.error(`Error deleting entity definition for ${entityType}: ${e.message}`);
-      }
+        })
+        // Swallowing the error as it is expected to fail if no entity definition exists
+        .catch((e) =>
+          this.log(`error`, entityType, `Error deleting entity definition: ${e.message}`)
+        );
+      this.log('debug', entityType, `Deleted entity definition`);
+
       await deleteEntityIndexComponentTemplate({
         unitedDefinition,
         esClient: this.esClient,
       });
+      this.log('debug', entityType, `Deleted entity index component template`);
+
       await deletePlatformPipeline({
         unitedDefinition,
         logger,
         esClient: this.esClient,
       });
+      this.log('debug', entityType, `Deleted platform pipeline`);
+
       await deleteFieldRetentionEnrichPolicy({
         unitedDefinition,
         esClient: this.esClient,
         logger,
       });
+      this.log('debug', entityType, `Deleted field retention enrich policy`);
 
       if (deleteData) {
         await deleteEntityIndex({
@@ -374,6 +415,7 @@ export class EntityStoreDataClient {
           namespace,
           logger,
         });
+        this.log('debug', entityType, `Deleted entity index`);
       }
 
       if (descriptor && deleteEngine) {
@@ -387,13 +429,22 @@ export class EntityStoreDataClient {
           logger,
           taskManager,
         });
+        this.log('debug', entityType, `Deleted entity store field retention enrich task`);
       }
 
       return { deleted: true };
-    } catch (e) {
-      logger.error(`Error deleting entity store for ${entityType}: ${e.message}`);
-      // TODO: should we set the engine status to error here?
-      throw e;
+    } catch (err) {
+      this.log(`error`, entityType, `Error deleting entity store: ${err.message}`);
+
+      this.audit(
+        EntityEngineActions.DELETE,
+        EntityStoreResource.ENTITY_ENGINE,
+        entityType,
+        'Failed to delete entity engine',
+        err
+      );
+
+      throw err;
     }
   }
 
@@ -521,4 +572,48 @@ export class EntityStoreDataClient {
       errors: updateErrors,
     };
   }
+
+  private log(
+    level: Exclude<keyof Logger, 'get' | 'log' | 'isLevelEnabled'>,
+    entityType: EntityType,
+    msg: string
+  ) {
+    this.options.logger[level](
+      `[Entity Engine] [entity.${entityType}] [namespace: ${this.options.namespace}] ${msg}`
+    );
+  }
+
+  private audit(
+    action: EntityEngineActions,
+    resource: EntityStoreResource,
+    entityType: EntityType,
+    msg: string,
+    error?: Error
+  ) {
+    // NOTE: Excluding errors, all auditing events are currently WRITE events, meaning the outcome is always UNKNOWN.
+    // This may change in the future, depending on the audit action.
+    const outcome = error ? AUDIT_OUTCOME.FAILURE : AUDIT_OUTCOME.UNKNOWN;
+
+    const type =
+      action === EntityEngineActions.CREATE
+        ? AUDIT_TYPE.CREATION
+        : EntityEngineActions.DELETE
+        ? AUDIT_TYPE.DELETION
+        : AUDIT_TYPE.CHANGE;
+
+    const category = AUDIT_CATEGORY.DATABASE;
+
+    const message = error ? `${msg}: ${error.message}` : msg;
+    const event: AuditEvent = {
+      message: `[Entity Engine] [entity.${entityType}] ${message}`,
+      event: {
+        action: `${action}_${entityType}_${resource}`,
+        category,
+        outcome,
+        type,
+      },
+    };
+
+    return this.options.auditLogger?.log(event);
+  }
 }