From a7a09f798f8ab68373ee2e15dec01ef917ff4e07 Mon Sep 17 00:00:00 2001
From: Viduni Wickramarachchi <viduni.wickramarachchi@elastic.co>
Date: Thu, 31 Oct 2024 15:17:08 -0400
Subject: [PATCH] [Obs AI Assistant] Attempt to resolve flaky knowledge based
 user instructions test (#196026)

Closes https://github.com/elastic/kibana/issues/192222

## Summary

### Problem
The "when creating private and public user instructions" test has been
marked as flaky and has been skipped.
Based on the error recorded in the ticket, 2 possible scenarios could be

- **Race Conditions**: When multiple instructions are created
asynchronously, the instructions might not be assigned to the right user
or role. Data could be overwritten.
- **Data Fetching Issues**: The API might return inconsistent data if
the knowledge base is not properly cleared between tests, or if the
instructions are not properly isolated per user.

### Solution
When running the test locally, the actual output and expected outcome
are the same, therefore the test passes. The flaky test runner didn't
output anything meaningful either.

However, in order to resolve any missing entries, the before hook was
updated to retry adding only the missing entries. Hopefully, this will
help resolve the flakiness.

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated to match the most common scenarios
- [x] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
---
 .../knowledge_base_user_instructions.spec.ts       | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts
index dc0f991c66ee2..a93c194c85daa 100644
--- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts
+++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts
@@ -10,6 +10,7 @@ import { kbnTestConfig } from '@kbn/test';
 import { sortBy } from 'lodash';
 import { Message, MessageRole } from '@kbn/observability-ai-assistant-plugin/common';
 import { CONTEXT_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/context';
+import { Instruction } from '@kbn/observability-ai-assistant-plugin/common/types';
 import { FtrProviderContext } from '../../common/ftr_provider_context';
 import {
   clearConversations,
@@ -51,8 +52,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
       await clearConversations(es);
     });
 
-    // FLAKY: https://github.com/elastic/kibana/issues/192222
-    describe.skip('when creating private and public user instructions', () => {
+    describe('when creating private and public user instructions', () => {
       before(async () => {
         await clearKnowledgeBase(es);
 
@@ -75,6 +75,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
           },
         ].map(async ({ username, isPublic }) => {
           const visibility = isPublic ? 'Public' : 'Private';
+
           await getScopedApiClientForUsername(username)({
             endpoint: 'PUT /internal/observability_ai_assistant/kb/user_instructions',
             params: {
@@ -94,9 +95,12 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         const res = await observabilityAIAssistantAPIClient.editorUser({
           endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
         });
+
         const instructions = res.body.userInstructions;
 
-        const sortByDocId = (data: any) => sortBy(data, 'doc_id');
+        const sortByDocId = (data: Array<Instruction & { public?: boolean }>) =>
+          sortBy(data, 'doc_id');
+
         expect(sortByDocId(instructions)).to.eql(
           sortByDocId([
             {
@@ -124,7 +128,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         });
         const instructions = res.body.userInstructions;
 
-        const sortByDocId = (data: any) => sortBy(data, 'doc_id');
+        const sortByDocId = (data: Array<Instruction & { public?: boolean }>) =>
+          sortBy(data, 'doc_id');
+
         expect(sortByDocId(instructions)).to.eql(
           sortByDocId([
             {