[8.16] [eem] use current user to delete indices (#195886) (#198492)

# Backport

This will backport the following commits from `main` to `8.16`:
 - [eem] use current user to delete indices (#195886) (9f291dc)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Kevin
Lacabane","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-16T19:03:48Z","message":"[eem]
use current user to delete indices (#195886)\n\nWe were trying to
cleanup `.entities` indices with the system user that\r\ndoes not have
necessary privileges. This failed silently because
of\r\n`ignore_unavailable:
true`","sha":"9f291dc55ec4c82380b81aa2fb9d7a8d84a1ff22"},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[]}]
BACKPORT-->

Co-authored-by: Kevin Lacabane <[email protected]>

x-pack/plugins/entity_manager/server/lib/entities/delete_index.ts

@@ -7,23 +7,16 @@
 
 import { ElasticsearchClient, Logger } from '@kbn/core/server';
 import { EntityDefinition } from '@kbn/entities-schema';
-import { generateHistoryIndexName, generateLatestIndexName } from './helpers/generate_component_id';
+import { generateLatestIndexName } from './helpers/generate_component_id';
 
 export async function deleteIndices(
   esClient: ElasticsearchClient,
   definition: EntityDefinition,
   logger: Logger
 ) {
   try {
-    const { indices: historyIndices } = await esClient.indices.resolveIndex({
-      name: `${generateHistoryIndexName(definition)}.*`,
-      expand_wildcards: 'all',
-    });
-    const indices = [
-      ...historyIndices.map(({ name }) => name),
-      generateLatestIndexName(definition),
-    ];
-    await esClient.indices.delete({ index: indices, ignore_unavailable: true });
+    const index = generateLatestIndexName(definition);
+    await esClient.indices.delete({ index, ignore_unavailable: true });
   } catch (e) {
     logger.error(`Unable to remove entity definition index [${definition.id}}]`);
     throw e;

x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts

@@ -10,63 +10,48 @@ import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 import { EntityDefinition } from '@kbn/entities-schema';
 import { Logger } from '@kbn/logging';
 import { deleteEntityDefinition } from './delete_entity_definition';
-import { deleteIndices } from './delete_index';
 import { deleteIngestPipelines } from './delete_ingest_pipeline';
-import { findEntityDefinitions } from './find_entity_definition';
 
 import { deleteTemplates } from '../manage_index_templates';
 
 import { stopTransforms } from './stop_transforms';
 
 import { deleteTransforms } from './delete_transforms';
+import { EntityClient } from '../entity_client';
 
 export async function uninstallEntityDefinition({
   definition,
   esClient,
   soClient,
   logger,
-  deleteData = false,
 }: {
   definition: EntityDefinition;
   esClient: ElasticsearchClient;
   soClient: SavedObjectsClientContract;
   logger: Logger;
-  deleteData?: boolean;
 }) {
   await stopTransforms(esClient, definition, logger);
   await deleteTransforms(esClient, definition, logger);
 
   await deleteIngestPipelines(esClient, definition, logger);
 
-  if (deleteData) {
-    await deleteIndices(esClient, definition, logger);
-  }
-
   await deleteTemplates(esClient, definition, logger);
 
   await deleteEntityDefinition(soClient, definition);
 }
 
 export async function uninstallBuiltInEntityDefinitions({
-  esClient,
-  soClient,
-  logger,
+  entityClient,
   deleteData = false,
 }: {
-  esClient: ElasticsearchClient;
-  soClient: SavedObjectsClientContract;
-  logger: Logger;
+  entityClient: EntityClient;
   deleteData?: boolean;
 }): Promise<EntityDefinition[]> {
-  const definitions = await findEntityDefinitions({
-    soClient,
-    esClient,
-    builtIn: true,
-  });
+  const { definitions } = await entityClient.getEntityDefinitions({ builtIn: true });
 
   await Promise.all(
-    definitions.map(async (definition) => {
-      await uninstallEntityDefinition({ definition, esClient, soClient, logger, deleteData });
+    definitions.map(async ({ id }) => {
+      await entityClient.deleteEntityDefinition({ id, deleteData });
     })
   );
 

x-pack/plugins/entity_manager/server/lib/entity_client.ts

@@ -7,7 +7,7 @@
 
 import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
-import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
 import { Logger } from '@kbn/logging';
 import {
   installEntityDefinition,
@@ -20,13 +20,14 @@ import { uninstallEntityDefinition } from './entities/uninstall_entity_definitio
 import { EntityDefinitionNotFound } from './entities/errors/entity_not_found';
 
 import { stopTransforms } from './entities/stop_transforms';
+import { deleteIndices } from './entities/delete_index';
 import { EntityDefinitionWithState } from './entities/types';
 import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict';
 
 export class EntityClient {
   constructor(
     private options: {
-      esClient: ElasticsearchClient;
+      clusterClient: IScopedClusterClient;
       soClient: SavedObjectsClientContract;
       logger: Logger;
     }
@@ -39,15 +40,16 @@ export class EntityClient {
     definition: EntityDefinition;
     installOnly?: boolean;
   }) {
+    const secondaryAuthClient = this.options.clusterClient.asSecondaryAuthUser;
     const installedDefinition = await installEntityDefinition({
       definition,
+      esClient: secondaryAuthClient,
       soClient: this.options.soClient,
-      esClient: this.options.esClient,
       logger: this.options.logger,
     });
 
     if (!installOnly) {
-      await startTransforms(this.options.esClient, installedDefinition, this.options.logger);
+      await startTransforms(secondaryAuthClient, installedDefinition, this.options.logger);
     }
 
     return installedDefinition;
@@ -60,10 +62,11 @@ export class EntityClient {
     id: string;
     definitionUpdate: EntityDefinitionUpdate;
   }) {
+    const secondaryAuthClient = this.options.clusterClient.asSecondaryAuthUser;
     const definition = await findEntityDefinitionById({
       id,
       soClient: this.options.soClient,
-      esClient: this.options.esClient,
+      esClient: secondaryAuthClient,
       includeState: true,
     });
 
@@ -87,22 +90,22 @@ export class EntityClient {
       definition,
       definitionUpdate,
       soClient: this.options.soClient,
-      esClient: this.options.esClient,
+      esClient: secondaryAuthClient,
       logger: this.options.logger,
     });
 
     if (shouldRestartTransforms) {
-      await startTransforms(this.options.esClient, updatedDefinition, this.options.logger);
+      await startTransforms(secondaryAuthClient, updatedDefinition, this.options.logger);
     }
     return updatedDefinition;
   }
 
   async deleteEntityDefinition({ id, deleteData = false }: { id: string; deleteData?: boolean }) {
-    const [definition] = await findEntityDefinitions({
+    const secondaryAuthClient = this.options.clusterClient.asSecondaryAuthUser;
+    const definition = await findEntityDefinitionById({
       id,
-      perPage: 1,
+      esClient: secondaryAuthClient,
       soClient: this.options.soClient,
-      esClient: this.options.esClient,
     });
 
     if (!definition) {
@@ -111,11 +114,20 @@ export class EntityClient {
 
     await uninstallEntityDefinition({
       definition,
-      deleteData,
+      esClient: secondaryAuthClient,
       soClient: this.options.soClient,
-      esClient: this.options.esClient,
       logger: this.options.logger,
     });
+
+    if (deleteData) {
+      // delete data with current user as system user does not have
+      // .entities privileges
+      await deleteIndices(
+        this.options.clusterClient.asCurrentUser,
+        definition,
+        this.options.logger
+      );
+    }
   }
 
   async getEntityDefinitions({
@@ -134,7 +146,7 @@ export class EntityClient {
     builtIn?: boolean;
   }) {
     const definitions = await findEntityDefinitions({
-      esClient: this.options.esClient,
+      esClient: this.options.clusterClient.asSecondaryAuthUser,
       soClient: this.options.soClient,
       page,
       perPage,
@@ -148,10 +160,18 @@ export class EntityClient {
   }
 
   async startEntityDefinition(definition: EntityDefinition) {
-    return startTransforms(this.options.esClient, definition, this.options.logger);
+    return startTransforms(
+      this.options.clusterClient.asSecondaryAuthUser,
+      definition,
+      this.options.logger
+    );
   }
 
   async stopEntityDefinition(definition: EntityDefinition) {
-    return stopTransforms(this.options.esClient, definition, this.options.logger);
+    return stopTransforms(
+      this.options.clusterClient.asSecondaryAuthUser,
+      definition,
+      this.options.logger
+    );
   }
 }

x-pack/plugins/entity_manager/server/plugin.ts

@@ -99,9 +99,9 @@ export class EntityManagerServerPlugin
     request: KibanaRequest;
     coreStart: CoreStart;
   }) {
-    const esClient = coreStart.elasticsearch.client.asScoped(request).asSecondaryAuthUser;
+    const clusterClient = coreStart.elasticsearch.client.asScoped(request);
     const soClient = coreStart.savedObjects.getScopedClient(request);
-    return new EntityClient({ esClient, soClient, logger: this.logger });
+    return new EntityClient({ clusterClient, soClient, logger: this.logger });
   }
 
   public start(

x-pack/plugins/entity_manager/server/routes/enablement/disable.ts

@@ -49,7 +49,7 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({
       deleteData: z.optional(BooleanFromString).default(false),
     }),
   }),
-  handler: async ({ context, response, params, logger, server }) => {
+  handler: async ({ context, request, response, params, logger, server, getScopedClient }) => {
     try {
       const esClientAsCurrentUser = (await context.core).elasticsearch.client.asCurrentUser;
       const canDisable = await canDisableEntityDiscovery(esClientAsCurrentUser);
@@ -62,15 +62,13 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({
         });
       }
 
-      const esClient = (await context.core).elasticsearch.client.asSecondaryAuthUser;
+      const entityClient = await getScopedClient({ request });
       const soClient = (await context.core).savedObjects.getClient({
         includedHiddenTypes: [EntityDiscoveryApiKeyType.name],
       });
 
       await uninstallBuiltInEntityDefinitions({
-        soClient,
-        esClient,
-        logger,
+        entityClient,
         deleteData: params.query.deleteData,
       });
 

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts

@@ -19,10 +19,11 @@ import type { EntityStoreConfig } from './types';
 
 describe('EntityStoreDataClient', () => {
   const mockSavedObjectClient = savedObjectsClientMock.create();
-  const esClientMock = elasticsearchServiceMock.createScopedClusterClient().asInternalUser;
+  const clusterClientMock = elasticsearchServiceMock.createScopedClusterClient();
+  const esClientMock = clusterClientMock.asCurrentUser;
   const loggerMock = loggingSystemMock.createLogger();
   const dataClient = new EntityStoreDataClient({
-    esClient: esClientMock,
+    clusterClient: clusterClientMock,
     logger: loggerMock,
     namespace: 'default',
     soClient: mockSavedObjectClient,