From b5b81fab6fc7ecf6b8b581422fd180a96a00d804 Mon Sep 17 00:00:00 2001 From: Eduardo de Valle Date: Tue, 21 May 2024 09:46:57 -0600 Subject: [PATCH 01/10] Cleaner version of code (bug not fixed yet) --- package-lock.json | 137 +++++++++++++++++++++++++++++++++++++--------- services/rag.ts | 66 +++------------------- 2 files changed, 121 insertions(+), 82 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1baa710..39c460c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "dependencies": { "@aws-sdk/credential-providers": "^3.556.0", "@dnd-kit/core": "^6.1.0", - "@headlessui/react": "^1.7.18", + "@headlessui/react": "^2.0.3", "@mantine/charts": "^7.9.1", "@mantine/core": "^7.9.1", "@mantine/dates": "^7.9.1", @@ -2018,25 +2018,28 @@ } }, "node_modules/@floating-ui/core": { - "version": "1.6.0", - "license": "MIT", + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.2.tgz", + "integrity": "sha512-+2XpQV9LLZeanU4ZevzRnGFg2neDeKHgFLjP6YLW+tly0IvrhqT4u8enLGjLH3qeh85g19xY5rsAusfwTdn5lg==", "dependencies": { - "@floating-ui/utils": "^0.2.1" + "@floating-ui/utils": "^0.2.0" } }, "node_modules/@floating-ui/dom": { - "version": "1.6.3", - "license": "MIT", + "version": "1.6.5", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.5.tgz", + "integrity": "sha512-Nsdud2X65Dz+1RHjAIP0t8z5e2ff/IRbei6BqFrl1urT8sDVzM1HMQ+R0XcU5ceRfyO3I6ayeqIfh+6Wb8LGTw==", "dependencies": { "@floating-ui/core": "^1.0.0", "@floating-ui/utils": "^0.2.0" } }, "node_modules/@floating-ui/react": { - "version": "0.26.12", - "license": "MIT", + "version": "0.26.16", + "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.26.16.tgz", + "integrity": "sha512-HEf43zxZNAI/E781QIVpYSF3K2VH4TTYZpqecjdsFkjsaU1EbaWcM++kw0HXFffj7gDUcBFevX8s0rQGQpxkow==", "dependencies": { - "@floating-ui/react-dom": "^2.0.0", + "@floating-ui/react-dom": "^2.1.0", "@floating-ui/utils": "^0.2.0", "tabbable": "^6.0.0" }, @@ -2046,10 +2049,11 @@ } }, "node_modules/@floating-ui/react-dom": { - "version": "2.0.8", - "license": "MIT", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.0.tgz", + "integrity": "sha512-lNzj5EQmEKn5FFKc04+zasr09h/uX8RtJRNj5gUXsSQIXHVWTVh+hVAg1vOMCexkX8EgvemMvIFpQfkosnVNyA==", "dependencies": { - "@floating-ui/dom": "^1.6.1" + "@floating-ui/dom": "^1.0.0" }, "peerDependencies": { "react": ">=16.8.0", @@ -2057,22 +2061,26 @@ } }, "node_modules/@floating-ui/utils": { - "version": "0.2.1", - "license": "MIT" + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.2.tgz", + "integrity": "sha512-J4yDIIthosAsRZ5CPYP/jQvUAQtlZTTD/4suA08/FEnlxqW3sKS9iAhgsa9VYLZ6vDHn/ixJgIqRQPotoBjxIw==" }, "node_modules/@headlessui/react": { - "version": "1.7.19", - "license": "MIT", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-2.0.3.tgz", + "integrity": "sha512-Xd1h0YZgfhxZ7W1w4TvK0/TZ1c4qaX4liYVUkAXqW1HCLcXSqnMeYAUGJS/BBroBAUL9HErjyFcRpCWRQZ/0lA==", "dependencies": { - "@tanstack/react-virtual": "^3.0.0-beta.60", - "client-only": "^0.0.1" + "@floating-ui/react": "^0.26.13", + "@react-aria/focus": "^3.16.2", + "@react-aria/interactions": "^3.21.1", + "@tanstack/react-virtual": "3.5.0" }, "engines": { "node": ">=10" }, "peerDependencies": { - "react": "^16 || ^17 || ^18", - "react-dom": "^16 || ^17 || ^18" + "react": "^18", + "react-dom": "^18" } }, "node_modules/@httptoolkit/websocket-stream": { @@ -3290,6 +3298,83 @@ "node": ">=16" } }, + "node_modules/@react-aria/focus": { + "version": "3.17.1", + "resolved": "https://registry.npmjs.org/@react-aria/focus/-/focus-3.17.1.tgz", + "integrity": "sha512-FLTySoSNqX++u0nWZJPPN5etXY0WBxaIe/YuL/GTEeuqUIuC/2bJSaw5hlsM6T2yjy6Y/VAxBcKSdAFUlU6njQ==", + "dependencies": { + "@react-aria/interactions": "^3.21.3", + "@react-aria/utils": "^3.24.1", + "@react-types/shared": "^3.23.1", + "@swc/helpers": "^0.5.0", + "clsx": "^2.0.0" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0" + } + }, + "node_modules/@react-aria/interactions": { + "version": "3.21.3", + "resolved": "https://registry.npmjs.org/@react-aria/interactions/-/interactions-3.21.3.tgz", + "integrity": "sha512-BWIuf4qCs5FreDJ9AguawLVS0lV9UU+sK4CCnbCNNmYqOWY+1+gRXCsnOM32K+oMESBxilAjdHW5n1hsMqYMpA==", + "dependencies": { + "@react-aria/ssr": "^3.9.4", + "@react-aria/utils": "^3.24.1", + "@react-types/shared": "^3.23.1", + "@swc/helpers": "^0.5.0" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0" + } + }, + "node_modules/@react-aria/ssr": { + "version": "3.9.4", + "resolved": "https://registry.npmjs.org/@react-aria/ssr/-/ssr-3.9.4.tgz", + "integrity": "sha512-4jmAigVq409qcJvQyuorsmBR4+9r3+JEC60wC+Y0MZV0HCtTmm8D9guYXlJMdx0SSkgj0hHAyFm/HvPNFofCoQ==", + "dependencies": { + "@swc/helpers": "^0.5.0" + }, + "engines": { + "node": ">= 12" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0" + } + }, + "node_modules/@react-aria/utils": { + "version": "3.24.1", + "resolved": "https://registry.npmjs.org/@react-aria/utils/-/utils-3.24.1.tgz", + "integrity": "sha512-O3s9qhPMd6n42x9sKeJ3lhu5V1Tlnzhu6Yk8QOvDuXf7UGuUjXf9mzfHJt1dYzID4l9Fwm8toczBzPM9t0jc8Q==", + "dependencies": { + "@react-aria/ssr": "^3.9.4", + "@react-stately/utils": "^3.10.1", + "@react-types/shared": "^3.23.1", + "@swc/helpers": "^0.5.0", + "clsx": "^2.0.0" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0" + } + }, + "node_modules/@react-stately/utils": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/@react-stately/utils/-/utils-3.10.1.tgz", + "integrity": "sha512-VS/EHRyicef25zDZcM/ClpzYMC5i2YGN6uegOeQawmgfGjb02yaCX0F0zR69Pod9m2Hr3wunTbtpgVXvYbZItg==", + "dependencies": { + "@swc/helpers": "^0.5.0" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0" + } + }, + "node_modules/@react-types/shared": { + "version": "3.23.1", + "resolved": "https://registry.npmjs.org/@react-types/shared/-/shared-3.23.1.tgz", + "integrity": "sha512-5d+3HbFDxGZjhbMBeFHRQhexMFt4pUce3okyRtUVKbbedQFUrtXSBg9VszgF2RTeQDKDkMCIQDtz5ccP/Lk1gw==", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0" + } + }, "node_modules/@rushstack/eslint-patch": { "version": "1.10.2", "dev": true, @@ -3897,10 +3982,11 @@ } }, "node_modules/@tanstack/react-virtual": { - "version": "3.4.0", - "license": "MIT", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.5.0.tgz", + "integrity": "sha512-rtvo7KwuIvqK9zb0VZ5IL7fiJAEnG+0EiFZz8FUOs+2mhGqdGmjKIaT1XU7Zq0eFqL0jonLlhbayJI/J2SA/Bw==", "dependencies": { - "@tanstack/virtual-core": "3.4.0" + "@tanstack/virtual-core": "3.5.0" }, "funding": { "type": "github", @@ -3912,8 +3998,9 @@ } }, "node_modules/@tanstack/virtual-core": { - "version": "3.4.0", - "license": "MIT", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.5.0.tgz", + "integrity": "sha512-KnPRCkQTyqhanNC0K63GBG3wA8I+D1fQuVnAvcBF8f13akOKeQp1gSbu6f77zCxhEk727iV5oQnbHLYzHrECLg==", "funding": { "type": "github", "url": "https://github.com/sponsors/tannerlinsley" diff --git a/services/rag.ts b/services/rag.ts index 6a7f67c..617d88f 100644 --- a/services/rag.ts +++ b/services/rag.ts @@ -39,10 +39,6 @@ interface FeedbackRecords { }; } -/* - This function calculates the cosine similarity between the query and the resources, - it returns the resources with the highest similarity -*/ async function cosine_similarity(feedback: string) { const resourcesSimilarity = []; const allResources = await db.select().from(pipResource); @@ -77,7 +73,6 @@ async function cosine_similarity(feedback: string) { return resourcesId; } -// This function creates the tasks for the user based on the feedback received async function create_tasks(feedback: string) { const openai = new OpenAI({ apiKey: process.env.OPENAI_KEY, @@ -116,8 +111,6 @@ async function create_tasks(feedback: string) { return cleanedTasks; } -// This function processes the open feedback of the user and returns a summary of the feedback -// feedback async function process_open_feedback(userFeedback: FeedbackRecords) { const openai = new OpenAI({ apiKey: process.env.OPENAI_KEY, @@ -197,7 +190,6 @@ async function process_open_feedback(userFeedback: FeedbackRecords) { return summarizedFeedbackCategories; } -// This function performs the analysis of the closed feedback async function process_closed_feedback(answers: [string, number][]) { const ordinaryPerformance = 7; const userCategories = { goodCategories: [], badCategories: [] }; @@ -231,26 +223,6 @@ async function process_closed_feedback(answers: [string, number][]) { return userCategories; } -/* - This function groups all the feedback received in a custom structure. - Output: - { - coworkerId_1: { - coworkersFeedback: { - coworkerId_2: { - openFeedback: [[questionName, comment], ...], - closedFeedback: [[questionName, answer], ...] - }, - }, - feedbackSummary: { - positive: {}, - negative: {}, - biased: {}, - notUseful: {} - } - } - } -*/ async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { const feedbackRecords: FeedbackRecords = {}; @@ -336,18 +308,9 @@ async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { return feedbackRecords; } -// This function creates the embeddings of all the new resources without embeddings async function set_resources_embeddings() { - /* - Embeddings models: - "text-embedding-3-small" - maxEmbeddingSize=1536 - "text-embedding-ada-002" - maxEmbeddingSize=1536 - "text-embedding-3-large" - maxEmbeddingSize=3072 - */ - try { - // resources without embeddings - const resources = await db + const noEmbeddingResources = await db .select() .from(pipResource) .where(sql`embedding::text = '[]'`); @@ -356,7 +319,7 @@ async function set_resources_embeddings() { apiKey: process.env.OPENAI_KEY, }); - for (let resource of resources) { + for (let resource of noEmbeddingResources) { const description = resource.description as string; const response = await openai.embeddings.create({ @@ -378,7 +341,6 @@ async function set_resources_embeddings() { } } -// This function analyzes the feedback of the user and creates new resources depending on the answers export async function ruler_analysis() { const session = await auth(); const userId = session?.user?.id; @@ -387,10 +349,9 @@ export async function ruler_analysis() { // recommend resources only if the mood of the user is negative, check the previous resources recommended to experiment } -// This function is triggered by the manager when the sprint survey is closed +// Main function export async function feedback_analysis(sprintSurveyIds: number[]) { for (let sprintSurveyId of sprintSurveyIds) { - // first check that the sprint survey hasnt been processed const processed = await db .select({ processed: sprintSurvey.processed }) .from(sprintSurvey) @@ -399,8 +360,7 @@ export async function feedback_analysis(sprintSurveyIds: number[]) { const notProcessed = !processed[0].processed; if (notProcessed) { - // get all the unique users that belong to the project of the sprint survey - const uniqueUsers = await db + const uniqueProjectUsers = await db .select({ userId: projectMember.userId, }) @@ -409,12 +369,10 @@ export async function feedback_analysis(sprintSurveyIds: number[]) { .innerJoin(projectMember, eq(projectMember.projectId, project.id)) .where(eq(sprintSurvey.id, sprintSurveyId)); - const ids = uniqueUsers.map((user) => user.userId as string); - - // feedback ordered, this structure is important to detect similarities between the feedback + const ids = uniqueProjectUsers.map((user) => user.userId as string); const orderedFeedback = await group_feedback(sprintSurveyId, ids); - // iterate through all the feedback of the sprint survey and analyze it + // iterate through each unique user of the project for (let userId of Object.keys(orderedFeedback)) { for (let coworkerId of Object.keys( orderedFeedback[userId]["coworkersFeedback"], @@ -423,29 +381,26 @@ export async function feedback_analysis(sprintSurveyIds: number[]) { orderedFeedback[userId]["coworkersFeedback"][coworkerId] .openFeedback.length > 0 ) { - // iterate through all the open feedback received from the coworker const userFeedbackSummary = await process_open_feedback( orderedFeedback[userId], ); - // add to the primary structure all the summaries of the previous function and the coworker who made the feedback const negativeFeedbackMap = orderedFeedback[userId].feedbackSummary.negative; + + // summarize the overall feedback received by the cowoeker for (let feedback of feedbackSummary) { if (feedback in negativeFeedbackMap!) { - // the summary emotion already exists, push the actual user in that element orderedFeedback[userId].feedbackSummary.negative[feedback].push( coworkerId, ); } else { - // the summary emotion doesnt exist, create a new element with the coworker orderedFeedback[userId].feedbackSummary.negative[feedback] = [ coworkerId, ]; } } } else { - // no open feedback received from the coworker, process closed feedback const answers = orderedFeedback[userId]["coworkersFeedback"][coworkerId] .closedFeedback; @@ -463,7 +418,6 @@ export async function feedback_analysis(sprintSurveyIds: number[]) { }, ); - // sort the feedback suggestions by the number of suggestions in descending order feedbackSuggestions.sort((a, b) => b[0] - a[0]); // get the feedback classification with the most suggestions @@ -483,7 +437,7 @@ export async function feedback_analysis(sprintSurveyIds: number[]) { // create tasks with the feedback received const tasks = await create_tasks(feedbackComment); - // insert the selected resources for the user + // insert the selected tasks and resources for (let task of tasks) { const [title, description] = task.split(":"); await db.insert(pipTask).values({ @@ -494,7 +448,6 @@ export async function feedback_analysis(sprintSurveyIds: number[]) { }); } - // insert the tasks for the user for (let resource of resources) { await db.insert(userResource).values({ userId: userId, @@ -503,7 +456,6 @@ export async function feedback_analysis(sprintSurveyIds: number[]) { } } - // mark the sprint survey as processed await db .update(sprintSurvey) .set({ processed: true }) From 9de9795e1e754be93504a3fc5cd9185845d3e6dd Mon Sep 17 00:00:00 2001 From: Eduardo de Valle Date: Tue, 21 May 2024 10:00:55 -0600 Subject: [PATCH 02/10] Bugs fixed, refactor of code functionality is left --- services/rag.ts | 49 +++++++++++++------------------------------------ 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/services/rag.ts b/services/rag.ts index 617d88f..b745452 100644 --- a/services/rag.ts +++ b/services/rag.ts @@ -20,8 +20,8 @@ import { interface FeedbackCategory { [coworkerId: string]: { - openFeedback: Array<[string, string]>; - closedFeedback: Array<[string, number]>; + openFeedback: Array<[number, string]>; + closedFeedback: Array<[number, number]>; }; } @@ -111,7 +111,7 @@ async function create_tasks(feedback: string) { return cleanedTasks; } -async function process_open_feedback(userFeedback: FeedbackRecords) { +async function process_open_feedback(userFeedback: string) { const openai = new OpenAI({ apiKey: process.env.OPENAI_KEY, }); @@ -153,7 +153,7 @@ async function process_open_feedback(userFeedback: FeedbackRecords) { }, { role: "user", - content: feedback, + content: userFeedback, }, ], }); @@ -190,35 +190,11 @@ async function process_open_feedback(userFeedback: FeedbackRecords) { return summarizedFeedbackCategories; } -async function process_closed_feedback(answers: [string, number][]) { +async function process_closed_feedback(answers: [number, number][]) { const ordinaryPerformance = 7; const userCategories = { goodCategories: [], badCategories: [] }; - // the good and bad categories must not contradict themselves across different questions - const categoriesPerQuestion = { - X: { goodCategories: [], badCategories: [] }, - Y: { goodCategories: [], badCategories: [] }, - Z: { goodCategories: [], badCategories: [] }, - }; - - type QuestionName = keyof typeof categoriesPerQuestion; - - for (let answer of answers) { - let questionName: QuestionName = answer[0] as QuestionName; - let answerValue = answer[1]; - - if (answerValue > ordinaryPerformance) { - // asign all the great categories to the user - let questionCategories = - categoriesPerQuestion[questionName]["goodCategories"]; - userCategories["goodCategories"].push(...questionCategories); - } else if (answerValue < ordinaryPerformance) { - // asign all the bad categories to the user - let questionCategories = - categoriesPerQuestion[questionName]["badCategories"]; - userCategories["badCategories"].push(...questionCategories); - } - } + // get the classifications of each closed question return userCategories; } @@ -230,7 +206,7 @@ async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { .select({ userId: sprintSurveyAnswerCoworkers.userId, coworkerId: sprintSurveyAnswerCoworkers.coworkerId, - questionName: sprintSurveyAnswerCoworkers.questionName, + questionId: sprintSurveyAnswerCoworkers.questionId, comment: sprintSurveyAnswerCoworkers.comment, }) .from(sprintSurvey) @@ -249,7 +225,7 @@ async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { .select({ userId: sprintSurveyAnswerCoworkers.userId, coworkerId: sprintSurveyAnswerCoworkers.coworkerId, - questionName: sprintSurveyAnswerCoworkers.questionName, + questionId: sprintSurveyAnswerCoworkers.questionId, answer: sprintSurveyAnswerCoworkers.answer, }) .from(sprintSurvey) @@ -293,7 +269,7 @@ async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { if (record.coworkerId !== null && record.coworkerId !== undefined) { feedbackRecords[record.userId!]["coworkersFeedback"][ record.coworkerId - ].openFeedback.push([record.questionName!, record.comment!]); + ].openFeedback.push([record.questionId!, record.comment!]); } }); @@ -301,7 +277,7 @@ async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { if (record.coworkerId !== null && record.coworkerId !== undefined) { feedbackRecords[record.userId!]["coworkersFeedback"][ record.coworkerId - ].closedFeedback.push([record.questionName!, record.answer!]); + ].closedFeedback.push([record.questionId!, record.answer!]); } }); @@ -382,14 +358,15 @@ export async function feedback_analysis(sprintSurveyIds: number[]) { .openFeedback.length > 0 ) { const userFeedbackSummary = await process_open_feedback( - orderedFeedback[userId], + orderedFeedback[userId]["coworkersFeedback"][coworkerId] + .openFeedback[0][1], ); const negativeFeedbackMap = orderedFeedback[userId].feedbackSummary.negative; // summarize the overall feedback received by the cowoeker - for (let feedback of feedbackSummary) { + for (let feedback of userFeedbackSummary) { if (feedback in negativeFeedbackMap!) { orderedFeedback[userId].feedbackSummary.negative[feedback].push( coworkerId, From af58a1e58508ed7ba2b939b5f558291cecf4bb9c Mon Sep 17 00:00:00 2001 From: Eduardo de Valle Date: Tue, 21 May 2024 10:13:47 -0600 Subject: [PATCH 03/10] Feedback processing one by one The function no longer receives an array of sprint surveys left, it only receives the sprint survey --- db/schema.ts | 2 +- services/rag.ts | 197 +++++++++++++++++++++++------------------------- 2 files changed, 97 insertions(+), 102 deletions(-) diff --git a/db/schema.ts b/db/schema.ts index b504b83..785c195 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -179,7 +179,7 @@ export const sprintSurveyQuestion = pgTable("sprint_survey_question", { questionId: integer("question_id").references(() => question.id), }); -export const resourcePositiveSkill = pgTable("resource_positive_skill", { +export const pipResourcePositiveSkill = pgTable("pip_resource_positive_skill", { pipResourceId: integer("pip_resource_id").references(() => pipResource.id), positiveSkillId: integer("positive_skill_id").references( () => positiveSkill.id, diff --git a/services/rag.ts b/services/rag.ts index b745452..8d3c76b 100644 --- a/services/rag.ts +++ b/services/rag.ts @@ -326,117 +326,112 @@ export async function ruler_analysis() { } // Main function -export async function feedback_analysis(sprintSurveyIds: number[]) { - for (let sprintSurveyId of sprintSurveyIds) { - const processed = await db - .select({ processed: sprintSurvey.processed }) +export async function feedback_analysis(sprintSurveyId: number) { + const processed = await db + .select({ processed: sprintSurvey.processed }) + .from(sprintSurvey) + .where(eq(sprintSurvey.id, sprintSurveyId)); + + const notProcessed = !processed[0].processed; + + if (notProcessed) { + const uniqueProjectUsers = await db + .select({ + userId: projectMember.userId, + }) .from(sprintSurvey) + .innerJoin(project, eq(project.id, sprintSurvey.projectId)) + .innerJoin(projectMember, eq(projectMember.projectId, project.id)) .where(eq(sprintSurvey.id, sprintSurveyId)); - const notProcessed = !processed[0].processed; - - if (notProcessed) { - const uniqueProjectUsers = await db - .select({ - userId: projectMember.userId, - }) - .from(sprintSurvey) - .innerJoin(project, eq(project.id, sprintSurvey.projectId)) - .innerJoin(projectMember, eq(projectMember.projectId, project.id)) - .where(eq(sprintSurvey.id, sprintSurveyId)); - - const ids = uniqueProjectUsers.map((user) => user.userId as string); - const orderedFeedback = await group_feedback(sprintSurveyId, ids); - - // iterate through each unique user of the project - for (let userId of Object.keys(orderedFeedback)) { - for (let coworkerId of Object.keys( - orderedFeedback[userId]["coworkersFeedback"], - )) { - if ( + const ids = uniqueProjectUsers.map((user) => user.userId as string); + const orderedFeedback = await group_feedback(sprintSurveyId, ids); + + // iterate through each unique user of the project + for (let userId of Object.keys(orderedFeedback)) { + for (let coworkerId of Object.keys( + orderedFeedback[userId]["coworkersFeedback"], + )) { + if ( + orderedFeedback[userId]["coworkersFeedback"][coworkerId].openFeedback + .length > 0 + ) { + const userFeedbackSummary = await process_open_feedback( orderedFeedback[userId]["coworkersFeedback"][coworkerId] - .openFeedback.length > 0 - ) { - const userFeedbackSummary = await process_open_feedback( - orderedFeedback[userId]["coworkersFeedback"][coworkerId] - .openFeedback[0][1], - ); - - const negativeFeedbackMap = - orderedFeedback[userId].feedbackSummary.negative; - - // summarize the overall feedback received by the cowoeker - for (let feedback of userFeedbackSummary) { - if (feedback in negativeFeedbackMap!) { - orderedFeedback[userId].feedbackSummary.negative[feedback].push( - coworkerId, - ); - } else { - orderedFeedback[userId].feedbackSummary.negative[feedback] = [ - coworkerId, - ]; - } + .openFeedback[0][1], + ); + + const negativeFeedbackMap = + orderedFeedback[userId].feedbackSummary.negative; + + // summarize the overall feedback received by the cowoeker + for (let feedback of userFeedbackSummary) { + if (feedback in negativeFeedbackMap!) { + orderedFeedback[userId].feedbackSummary.negative[feedback].push( + coworkerId, + ); + } else { + orderedFeedback[userId].feedbackSummary.negative[feedback] = [ + coworkerId, + ]; } - } else { - const answers = - orderedFeedback[userId]["coworkersFeedback"][coworkerId] - .closedFeedback; - const feedbackSummary = await process_closed_feedback(answers); } + } else { + const answers = + orderedFeedback[userId]["coworkersFeedback"][coworkerId] + .closedFeedback; + const feedbackSummary = await process_closed_feedback(answers); } - // all feedback summarized, now get the classifications of negative feedback with the most suggestions - const feedbackSuggestions: [number, string][] = []; - Object.keys(orderedFeedback[userId].feedbackSummary.negative).forEach( - (key) => { - feedbackSuggestions.push([ - orderedFeedback[userId].feedbackSummary.negative[key].length, - key, - ]); - }, - ); - - feedbackSuggestions.sort((a, b) => b[0] - a[0]); - - // get the feedback classification with the most suggestions - const feedbackToCreate = feedbackSuggestions[0][1]; - - // get a comment with that classification to recommend resources and tasks - const newCoworkerId = - orderedFeedback[userId].feedbackSummary.negative[feedbackToCreate][0]; - - const feedbackComment = - orderedFeedback[userId].coworkersFeedback[newCoworkerId] - .openFeedback[0][1]; - - // get the resources with the highest similarity to the feedback - const resources = await cosine_similarity(feedbackComment); - - // create tasks with the feedback received - const tasks = await create_tasks(feedbackComment); - - // insert the selected tasks and resources - for (let task of tasks) { - const [title, description] = task.split(":"); - await db.insert(pipTask).values({ - userId: userId, - title: title, - description: description, - isDone: false, - }); - } + } - for (let resource of resources) { - await db.insert(userResource).values({ - userId: userId, - resourceId: resource, - }); - } + // all feedback summarized, now get the classifications of negative feedback with the most suggestions + const feedbackSuggestions: [number, string][] = []; + Object.keys(orderedFeedback[userId].feedbackSummary.negative).forEach( + (key) => { + feedbackSuggestions.push([ + orderedFeedback[userId].feedbackSummary.negative[key].length, + key, + ]); + }, + ); + + feedbackSuggestions.sort((a, b) => b[0] - a[0]); + + const feedbackToCreate = feedbackSuggestions[0][1]; + + const newCoworkerId = + orderedFeedback[userId].feedbackSummary.negative[feedbackToCreate][0]; + + const feedbackComment = + orderedFeedback[userId].coworkersFeedback[newCoworkerId] + .openFeedback[0][1]; + + // select the best tasks and resource + const resources = await cosine_similarity(feedbackComment); + + const tasks = await create_tasks(feedbackComment); + + for (let task of tasks) { + const [title, description] = task.split(":"); + await db.insert(pipTask).values({ + userId: userId, + title: title, + description: description, + isDone: false, + }); } - await db - .update(sprintSurvey) - .set({ processed: true }) - .where(eq(sprintSurvey.id, sprintSurveyId)); + for (let resource of resources) { + await db.insert(userResource).values({ + userId: userId, + resourceId: resource, + }); + } } + + await db + .update(sprintSurvey) + .set({ processed: true }) + .where(eq(sprintSurvey.id, sprintSurveyId)); } } From 472e72b2a84b7b745e8d9fc47af984a41413678e Mon Sep 17 00:00:00 2001 From: Eduardo de Valle Date: Tue, 21 May 2024 11:53:32 -0600 Subject: [PATCH 04/10] Interface of open feedback is no longer an array, it's a string --- services/rag.ts | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/services/rag.ts b/services/rag.ts index 8d3c76b..9056305 100644 --- a/services/rag.ts +++ b/services/rag.ts @@ -20,7 +20,7 @@ import { interface FeedbackCategory { [coworkerId: string]: { - openFeedback: Array<[number, string]>; + openFeedback: string; closedFeedback: Array<[number, number]>; }; } @@ -259,7 +259,7 @@ async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { for (let coworkerId of filteredCoworkers) { feedbackRecords[userId]["coworkersFeedback"][coworkerId] = { - openFeedback: [], + openFeedback: "", closedFeedback: [], }; } @@ -269,7 +269,7 @@ async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { if (record.coworkerId !== null && record.coworkerId !== undefined) { feedbackRecords[record.userId!]["coworkersFeedback"][ record.coworkerId - ].openFeedback.push([record.questionId!, record.comment!]); + ].openFeedback = record.comment!; } }); @@ -352,36 +352,6 @@ export async function feedback_analysis(sprintSurveyId: number) { for (let coworkerId of Object.keys( orderedFeedback[userId]["coworkersFeedback"], )) { - if ( - orderedFeedback[userId]["coworkersFeedback"][coworkerId].openFeedback - .length > 0 - ) { - const userFeedbackSummary = await process_open_feedback( - orderedFeedback[userId]["coworkersFeedback"][coworkerId] - .openFeedback[0][1], - ); - - const negativeFeedbackMap = - orderedFeedback[userId].feedbackSummary.negative; - - // summarize the overall feedback received by the cowoeker - for (let feedback of userFeedbackSummary) { - if (feedback in negativeFeedbackMap!) { - orderedFeedback[userId].feedbackSummary.negative[feedback].push( - coworkerId, - ); - } else { - orderedFeedback[userId].feedbackSummary.negative[feedback] = [ - coworkerId, - ]; - } - } - } else { - const answers = - orderedFeedback[userId]["coworkersFeedback"][coworkerId] - .closedFeedback; - const feedbackSummary = await process_closed_feedback(answers); - } } // all feedback summarized, now get the classifications of negative feedback with the most suggestions From cd4748d1839f0a4ddf196c67b3883d8d4452e273 Mon Sep 17 00:00:00 2001 From: Eduardo de Valle Date: Thu, 23 May 2024 09:44:42 -0600 Subject: [PATCH 05/10] Double check in each user of sprint survey Safety double check if the user has been checked in case of a failure in the middle of the survey analysis --- services/rag.ts | 121 ++++++++++++++++++++++++++++++------------------ 1 file changed, 75 insertions(+), 46 deletions(-) diff --git a/services/rag.ts b/services/rag.ts index 9056305..7ae1581 100644 --- a/services/rag.ts +++ b/services/rag.ts @@ -2,7 +2,7 @@ import dotenv from "dotenv"; import OpenAI from "openai"; import db from "@/db/drizzle"; -import { and, eq, gte, lte, isNull, sql } from "drizzle-orm"; +import { and, count, eq, gte, lte, isNull, sql } from "drizzle-orm"; import { auth } from "@/auth"; import similarity from "compute-cosine-similarity"; @@ -25,7 +25,7 @@ interface FeedbackCategory { }; } -interface FeedbackSummary { +interface FeedbackClassifications { positive: { [sentiment: string]: string[] }; negative: { [sentiment: string]: string[] }; biased: { [sentiment: string]: string[] }; @@ -35,12 +35,12 @@ interface FeedbackSummary { interface FeedbackRecords { [userId: string]: { coworkersFeedback: FeedbackCategory; - feedbackSummary: FeedbackSummary; + feedbackClassifications: FeedbackClassifications; }; } async function cosine_similarity(feedback: string) { - const resourcesSimilarity = []; + const resourcesSimilarity: [GLfloat, Number][] = []; const allResources = await db.select().from(pipResource); const openai = new OpenAI({ @@ -57,7 +57,10 @@ async function cosine_similarity(feedback: string) { // calculate the cosine similarity between the feedback and the resources for (let resource of allResources) { - var resourceSimilarity = similarity(feedbackEmbedding, resource.embedding!); + var resourceSimilarity: GLfloat = similarity( + feedbackEmbedding, + resource.embedding!, + )!; resourcesSimilarity.push([resourceSimilarity, resource.id]); } @@ -249,7 +252,7 @@ async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { feedbackRecords[userId] = { coworkersFeedback: {}, - feedbackSummary: { + feedbackClassifications: { positive: {}, negative: {}, biased: {}, @@ -327,14 +330,14 @@ export async function ruler_analysis() { // Main function export async function feedback_analysis(sprintSurveyId: number) { - const processed = await db + const processedSurvey = await db .select({ processed: sprintSurvey.processed }) .from(sprintSurvey) .where(eq(sprintSurvey.id, sprintSurveyId)); - const notProcessed = !processed[0].processed; + const notProcessedSurvey = !processedSurvey[0].processed; - if (notProcessed) { + if (notProcessedSurvey) { const uniqueProjectUsers = await db .select({ userId: projectMember.userId, @@ -347,55 +350,81 @@ export async function feedback_analysis(sprintSurveyId: number) { const ids = uniqueProjectUsers.map((user) => user.userId as string); const orderedFeedback = await group_feedback(sprintSurveyId, ids); - // iterate through each unique user of the project + // iterate through each unique user of the project and read the feedback received for (let userId of Object.keys(orderedFeedback)) { - for (let coworkerId of Object.keys( - orderedFeedback[userId]["coworkersFeedback"], - )) { - } - - // all feedback summarized, now get the classifications of negative feedback with the most suggestions - const feedbackSuggestions: [number, string][] = []; - Object.keys(orderedFeedback[userId].feedbackSummary.negative).forEach( - (key) => { + let userTasksCount = await db + .select({ count: count() }) + .from(pipTask) + .where( + and( + eq(pipTask.userId, userId), + eq(pipTask.sprintSurveyId, sprintSurveyId), + ), + ); + + let userResourcesCount = await db + .select({ count: count() }) + .from(userResource) + .where( + and( + eq(userResource.userId, userId), + eq(userResource.sprintSurveyId, sprintSurveyId), + ), + ); + + // safety double check if the user has been checked in case of a failure in the middle of the survey analysis + if (userTasksCount[0].count == 0 || userResourcesCount[0].count == 0) { + for (let coworkerId of Object.keys( + orderedFeedback[userId]["coworkersFeedback"], + )) { + } + + // all feedback summarized, now get the classifications of negative feedback with the most suggestions + const feedbackSuggestions: [number, string][] = []; + Object.keys( + orderedFeedback[userId].feedbackClassifications.negative, + ).forEach((key) => { feedbackSuggestions.push([ - orderedFeedback[userId].feedbackSummary.negative[key].length, + orderedFeedback[userId].feedbackClassifications.negative[key] + .length, key, ]); - }, - ); + }); - feedbackSuggestions.sort((a, b) => b[0] - a[0]); + feedbackSuggestions.sort((a, b) => b[0] - a[0]); - const feedbackToCreate = feedbackSuggestions[0][1]; + const feedbackToCreate = feedbackSuggestions[0][1]; - const newCoworkerId = - orderedFeedback[userId].feedbackSummary.negative[feedbackToCreate][0]; + const newCoworkerId = + orderedFeedback[userId].feedbackClassifications.negative[ + feedbackToCreate + ][0]; - const feedbackComment = - orderedFeedback[userId].coworkersFeedback[newCoworkerId] - .openFeedback[0][1]; + const feedbackComment = + orderedFeedback[userId].coworkersFeedback[newCoworkerId] + .openFeedback[0][1]; - // select the best tasks and resource - const resources = await cosine_similarity(feedbackComment); + // select the best tasks and resource + const resources = await cosine_similarity(feedbackComment); - const tasks = await create_tasks(feedbackComment); + const tasks = await create_tasks(feedbackComment); - for (let task of tasks) { - const [title, description] = task.split(":"); - await db.insert(pipTask).values({ - userId: userId, - title: title, - description: description, - isDone: false, - }); - } + for (let task of tasks) { + const [title, description] = task.split(":"); + await db.insert(pipTask).values({ + userId: userId, + title: title, + description: description, + isDone: false, + }); + } - for (let resource of resources) { - await db.insert(userResource).values({ - userId: userId, - resourceId: resource, - }); + for (let resource of resources) { + await db.insert(userResource).values({ + userId: userId, + resourceId: resource, + }); + } } } From 8b3917fb79ffef375e23c6c455bbf38eb845a517 Mon Sep 17 00:00:00 2001 From: Eduardo de Valle Date: Fri, 24 May 2024 13:59:55 -0600 Subject: [PATCH 06/10] First lecture of closed questions --- db/schema.ts | 3 +++ services/rag.ts | 42 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/db/schema.ts b/db/schema.ts index 785c195..e565b84 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -111,6 +111,9 @@ export const userResource = pgTable( resourceId: serial("resource_id").references(() => pipResource.id, { onDelete: "cascade", }), + sprintSurveyId: integer("sprint_survey_id").references( + () => sprintSurvey.id, + ), }, // composite primary key on (userId, resourceId) ); diff --git a/services/rag.ts b/services/rag.ts index 7ae1581..16fb831 100644 --- a/services/rag.ts +++ b/services/rag.ts @@ -16,6 +16,8 @@ import { sprintSurveyAnswerCoworkers, sprintSurveyAnswerProject, userResource, + question, + questionPositiveSkill, } from "@/db/schema"; interface FeedbackCategory { @@ -29,7 +31,6 @@ interface FeedbackClassifications { positive: { [sentiment: string]: string[] }; negative: { [sentiment: string]: string[] }; biased: { [sentiment: string]: string[] }; - notUseful: { [sentiment: string]: string[] }; } interface FeedbackRecords { @@ -256,7 +257,6 @@ async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { positive: {}, negative: {}, biased: {}, - notUseful: {}, }, }; @@ -287,6 +287,36 @@ async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { return feedbackRecords; } +async function getFeedbackClassifications(coworkersFeedback: FeedbackCategory) { + const feedbackClassifications: FeedbackClassifications = { + positive: {}, + negative: {}, + biased: {}, + }; + + for (let coworkerId of Object.keys(coworkersFeedback)) { + let closedFeedback = coworkersFeedback[coworkerId].closedFeedback; + let positivePerformanceCount = 0; + let negativePerformanceCount = 0; + for (let answer of closedFeedback) { + if (answer[1] >= 8) { + positivePerformanceCount++; + // get the positive skills of the question + let questionPositiveSkills = await db + .select({ + skill: questionPositiveSkill.skill, + }) + .from(); + } else { + negativePerformanceCount++; + // get the positive skills of the question + } + } + } + + return feedbackClassifications; +} + async function set_resources_embeddings() { try { const noEmbeddingResources = await db @@ -374,10 +404,10 @@ export async function feedback_analysis(sprintSurveyId: number) { // safety double check if the user has been checked in case of a failure in the middle of the survey analysis if (userTasksCount[0].count == 0 || userResourcesCount[0].count == 0) { - for (let coworkerId of Object.keys( - orderedFeedback[userId]["coworkersFeedback"], - )) { - } + orderedFeedback[userId]["feedbackClassifications"] = + await getFeedbackClassifications( + orderedFeedback[userId].coworkersFeedback, + ); // all feedback summarized, now get the classifications of negative feedback with the most suggestions const feedbackSuggestions: [number, string][] = []; From 4adb61ef55d91f308a91f9bcebc2cb81eece1b76 Mon Sep 17 00:00:00 2001 From: Eduardo de Valle Date: Sun, 26 May 2024 15:44:31 -0600 Subject: [PATCH 07/10] Update in the algorithm of feedback analysis --- services/rag.ts | 152 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 111 insertions(+), 41 deletions(-) diff --git a/services/rag.ts b/services/rag.ts index 16fb831..47583fd 100644 --- a/services/rag.ts +++ b/services/rag.ts @@ -9,17 +9,22 @@ import similarity from "compute-cosine-similarity"; import { pipTask, pipResource, + positiveSkill, project, projectMember, rulerEmotion, sprintSurvey, sprintSurveyAnswerCoworkers, sprintSurveyAnswerProject, + sprintSurveyQuestion, userResource, question, + questionNegativeSkill, questionPositiveSkill, } from "@/db/schema"; +// =============== FEEDBACK INTERFACES =============== + interface FeedbackCategory { [coworkerId: string]: { openFeedback: string; @@ -40,7 +45,16 @@ interface FeedbackRecords { }; } -async function cosine_similarity(feedback: string) { +// =============== QUESTIONS SKILLS INTERFACES =============== + +interface QuestionSkills { + [questionId: number]: { + positive: number[]; + negative: number[]; + }; +} + +async function cosineSimilarity(feedback: string) { const resourcesSimilarity: [GLfloat, Number][] = []; const allResources = await db.select().from(pipResource); @@ -77,7 +91,7 @@ async function cosine_similarity(feedback: string) { return resourcesId; } -async function create_tasks(feedback: string) { +async function createTasks(feedback: string) { const openai = new OpenAI({ apiKey: process.env.OPENAI_KEY, }); @@ -115,7 +129,7 @@ async function create_tasks(feedback: string) { return cleanedTasks; } -async function process_open_feedback(userFeedback: string) { +async function processOpenFeedback(userFeedback: string) { const openai = new OpenAI({ apiKey: process.env.OPENAI_KEY, }); @@ -144,9 +158,6 @@ async function process_open_feedback(userFeedback: string) { notUseful: Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. """`; - const summarizationInstructions: string = - "El siguiente es un párrafo con comentarios negativos o críticas constructivas hacia una persona, no son ataques personales. Quiero que resumas todo el párrafo en algunas categorías, pero solo puedes usar las que yo te diga sin usar opciones afuera de esa lista y debes regresar las categorías separadas por comas y sin espacios entre cada separador, solo regresa las categorías identificadas, no todas las mencionadas, no respondas ni expliques tu procedimiento, limítate a cumplir con las instrucciones especificadas. Las clasificaciones son: Mala gestión de tiempo, Sin inteligencia emocional, Prepotencia, Soberbia, Poca creatividad, Sin iniciativa, Mala comunicación, Mal trabajo en equipo, Falta de ética, y Sin razonamiento crítico. Un ejemplo del resultado que debes regresar es `Sin iniciativa,Mala gestión de tiempo,Mal trabajo en equipo,Mala comunicación`"; - // classify the feedback into 4 categories: positive, negative, biased, not useful const classifiedFeedback = await openai.chat.completions.create({ model: "gpt-4", @@ -173,28 +184,10 @@ async function process_open_feedback(userFeedback: string) { } } - // summarize the negative feedback to be compared with all the feedback given to the same user - const summarizedFeedback = await openai.chat.completions.create({ - model: "gpt-4", - messages: [ - { - role: "system", - content: summarizationInstructions, - }, - { - role: "user", - content: negativeFeedback, - }, - ], - }); - - const summarizedFeedbackCategories = - summarizedFeedback.choices[0].message.content!.split(","); - return summarizedFeedbackCategories; } -async function process_closed_feedback(answers: [number, number][]) { +async function processClosedFeedback(answers: [number, number][]) { const ordinaryPerformance = 7; const userCategories = { goodCategories: [], badCategories: [] }; @@ -203,7 +196,7 @@ async function process_closed_feedback(answers: [number, number][]) { return userCategories; } -async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { +async function orderFeedback(sprintSurveyId: number, uniqueWorkers: string[]) { const feedbackRecords: FeedbackRecords = {}; const coworkersOpenFeedback = await db @@ -287,37 +280,110 @@ async function group_feedback(sprintSurveyId: number, uniqueWorkers: string[]) { return feedbackRecords; } -async function getFeedbackClassifications(coworkersFeedback: FeedbackCategory) { +async function getFeedbackClassifications( + coworkersFeedback: FeedbackCategory, + questionsSkills: QuestionSkills, +) { const feedbackClassifications: FeedbackClassifications = { positive: {}, negative: {}, biased: {}, }; + // read the feedback of each coworker and classify it for (let coworkerId of Object.keys(coworkersFeedback)) { let closedFeedback = coworkersFeedback[coworkerId].closedFeedback; let positivePerformanceCount = 0; let negativePerformanceCount = 0; + // read all the closed feedback of the coworker for (let answer of closedFeedback) { if (answer[1] >= 8) { + // add the positive skills of the question positivePerformanceCount++; - // get the positive skills of the question - let questionPositiveSkills = await db - .select({ - skill: questionPositiveSkill.skill, - }) - .from(); + let questionPositiveSkills = questionsSkills[answer[0]].positive; + for (let positiveSkillId of questionPositiveSkills) { + if (positiveSkillId in feedbackClassifications.positive) { + // the skill already exists in the classification, check if the coworker is already there + if ( + !feedbackClassifications.positive[positiveSkillId].includes( + coworkerId, + ) + ) { + feedbackClassifications.positive[positiveSkillId].push( + coworkerId, + ); + } + } else { + // the skill does not exist in the classification, add the coworker + feedbackClassifications.positive[positiveSkillId] = [coworkerId]; + } + } } else { + // add the negative skills of the question negativePerformanceCount++; - // get the positive skills of the question + let questionNegativeSkills = questionsSkills[answer[0]].negative; + for (let negativeSkillId of questionNegativeSkills) { + if (negativeSkillId in feedbackClassifications.negative) { + // the skill already exists in the classification, check if the coworker is already there + if ( + !feedbackClassifications.negative[negativeSkillId].includes( + coworkerId, + ) + ) { + feedbackClassifications.negative[negativeSkillId].push( + coworkerId, + ); + } + } else { + // the skill does not exist in the classification, add the coworker + feedbackClassifications.negative[negativeSkillId] = [coworkerId]; + } + } } } + + // the user has negative performance, an analysis of the comment is needed + if (negativePerformanceCount > 1) { + } } return feedbackClassifications; } -async function set_resources_embeddings() { +async function getQuestionsSkills(sprintSurveyId: number) { + const questionsSkills: QuestionSkills = {}; + const questions = await db + .select({ + questionId: sprintSurveyQuestion.questionId, + }) + .from(sprintSurveyQuestion) + .where(eq(sprintSurveyQuestion.sprintSurveyId, sprintSurveyId)); + + for (let question of questions) { + const positiveSkills = await db + .select({ + skill: questionPositiveSkill.positiveSkillId, + }) + .from(questionPositiveSkill) + .where(eq(questionPositiveSkill.questionId, question.questionId!)); + + const negativeSkills = await db + .select({ + skill: questionNegativeSkill.negativeSkillId, + }) + .from(questionNegativeSkill) + .where(eq(questionNegativeSkill.questionId, question.questionId!)); + + questionsSkills[question.questionId!] = { + positive: positiveSkills.map((skill) => skill.skill as number), + negative: negativeSkills.map((skill) => skill.skill as number), + }; + } + + return questionsSkills; +} + +async function setResourcesEmbeddings() { try { const noEmbeddingResources = await db .select() @@ -350,7 +416,7 @@ async function set_resources_embeddings() { } } -export async function ruler_analysis() { +export async function rulerAnalysis() { const session = await auth(); const userId = session?.user?.id; if (!userId) throw new Error("You must be signed in"); @@ -378,7 +444,8 @@ export async function feedback_analysis(sprintSurveyId: number) { .where(eq(sprintSurvey.id, sprintSurveyId)); const ids = uniqueProjectUsers.map((user) => user.userId as string); - const orderedFeedback = await group_feedback(sprintSurveyId, ids); + const orderedFeedback = await orderFeedback(sprintSurveyId, ids); + const questionsSkills = await getQuestionsSkills(sprintSurveyId); // iterate through each unique user of the project and read the feedback received for (let userId of Object.keys(orderedFeedback)) { @@ -402,11 +469,12 @@ export async function feedback_analysis(sprintSurveyId: number) { ), ); - // safety double check if the user has been checked in case of a failure in the middle of the survey analysis + // safety double check if the user has been checked in case of a failure in the middle of a previous survey analysis if (userTasksCount[0].count == 0 || userResourcesCount[0].count == 0) { orderedFeedback[userId]["feedbackClassifications"] = await getFeedbackClassifications( orderedFeedback[userId].coworkersFeedback, + questionsSkills, ); // all feedback summarized, now get the classifications of negative feedback with the most suggestions @@ -434,10 +502,10 @@ export async function feedback_analysis(sprintSurveyId: number) { orderedFeedback[userId].coworkersFeedback[newCoworkerId] .openFeedback[0][1]; - // select the best tasks and resource - const resources = await cosine_similarity(feedbackComment); + // select the best tasks and resources + const resources = await cosineSimilarity(feedbackComment); - const tasks = await create_tasks(feedbackComment); + const tasks = await createTasks(feedbackComment); for (let task of tasks) { const [title, description] = task.split(":"); @@ -455,6 +523,8 @@ export async function feedback_analysis(sprintSurveyId: number) { resourceId: resource, }); } + + // set the strengths and weaknesses of the user } } From 752930af65e369ef5d8db5976ab8d5376fdbe714 Mon Sep 17 00:00:00 2001 From: Eduardo de Valle Date: Sun, 26 May 2024 16:30:31 -0600 Subject: [PATCH 08/10] RAG function updated Now the same RAG function can be used by different values for different tables for similarity search --- services/rag.ts | 47 ++++++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/services/rag.ts b/services/rag.ts index 47583fd..36deb32 100644 --- a/services/rag.ts +++ b/services/rag.ts @@ -45,7 +45,7 @@ interface FeedbackRecords { }; } -// =============== QUESTIONS SKILLS INTERFACES =============== +// =============== OTHER INTERFACES =============== interface QuestionSkills { [questionId: number]: { @@ -54,9 +54,16 @@ interface QuestionSkills { }; } -async function cosineSimilarity(feedback: string) { - const resourcesSimilarity: [GLfloat, Number][] = []; - const allResources = await db.select().from(pipResource); +interface EmbeddingRecord { + id: number; + embedding: GLfloat[] | null; +} + +async function cosineSimilarity( + baseMessage: string, + embeddingRecords: EmbeddingRecord[], +) { + const recordsSimilarity: [GLfloat, Number][] = []; const openai = new OpenAI({ apiKey: process.env.OPENAI_KEY, @@ -64,31 +71,26 @@ async function cosineSimilarity(feedback: string) { const response = await openai.embeddings.create({ model: "text-embedding-3-small", - input: feedback, + input: baseMessage, encoding_format: "float", }); - const feedbackEmbedding = response.data[0].embedding; + const baseEmbedding = response.data[0].embedding; - // calculate the cosine similarity between the feedback and the resources - for (let resource of allResources) { - var resourceSimilarity: GLfloat = similarity( - feedbackEmbedding, - resource.embedding!, + // calculate the cosine similarity between the base string and all the records + for (let record of embeddingRecords) { + var recordSimilarity: GLfloat = similarity( + baseEmbedding, + record.embedding!, )!; - resourcesSimilarity.push([resourceSimilarity, resource.id]); + recordsSimilarity.push([recordSimilarity, record.id]); } - // sort the resources by similarity in descending order - resourcesSimilarity.sort((a, b) => b[0]! - a[0]!); - - resourcesSimilarity.splice(5); - - // return only the resources ids - const resourcesId: number[] = resourcesSimilarity + // return only the IDs of the records + const recordsId: number[] = recordsSimilarity .map(([_, second]) => second) .filter((value): value is number => value !== null); - return resourcesId; + return recordsId; } async function createTasks(feedback: string) { @@ -503,7 +505,10 @@ export async function feedback_analysis(sprintSurveyId: number) { .openFeedback[0][1]; // select the best tasks and resources - const resources = await cosineSimilarity(feedbackComment); + const allResources: EmbeddingRecord[] = await db + .select({ id: pipResource.id, embedding: pipResource.embedding }) + .from(pipResource); + const resources = await cosineSimilarity(feedbackComment, allResources); const tasks = await createTasks(feedbackComment); From 9f12f8feeee54be1841e9a7d8e9c6585f11a5fe5 Mon Sep 17 00:00:00 2001 From: Eduardo de Valle Date: Mon, 27 May 2024 00:06:05 -0600 Subject: [PATCH 09/10] Changes in the main algorithm of feedback analysis --- services/rag.ts | 102 +++++++++++++++++++++++++++++++----------------- 1 file changed, 67 insertions(+), 35 deletions(-) diff --git a/services/rag.ts b/services/rag.ts index 36deb32..c12dc1e 100644 --- a/services/rag.ts +++ b/services/rag.ts @@ -132,35 +132,38 @@ async function createTasks(feedback: string) { } async function processOpenFeedback(userFeedback: string) { + // string cleaning + userFeedback = userFeedback.replaceAll("positive:", ""); + userFeedback = userFeedback.replaceAll("negative:", ""); + userFeedback = userFeedback.replaceAll("biased:", ""); + userFeedback = userFeedback.replaceAll(" ", " "); + const openai = new OpenAI({ apiKey: process.env.OPENAI_KEY, }); const classificationInstructions: string = `El siguiente query es un párrafo que contiene feedback de un usuario a otro. Realiza las siguientes instrucciones: 1. Identifica las ideas claves de todo el párrafo, sepáralo por cada idea tomando en cuenta la coherencia entre las oraciones y las ideas que expresan guardando la conexión del asunto, puedes ignorar signos de puntuación si una misma oración une ideas diferentes, pero tu prioridad es separar las oraciones con ideas atómicas. - 2. De las oraciones agrupadas por ideas, sepáralas en las siguientes 4 clasificaciones de sentimientos, tomando en cuenta la descripción de cada una: + 2. De las oraciones agrupadas por ideas, sepáralas en las siguientes 3 clasificaciones de sentimientos, tomando en cuenta la descripción de cada una: * positive: cualquier cumplido, elogio o felicitación a la persona que recibe el comentario o a su desempeño en el trabajo. Las críticas constructivas si bien son una forma saludable de dar retroalimentación no cuentan como comentario positivo porque destacan una necesidad de la persona evaluada. * negative: cualquier comentario relacionado con críticas constructivas o áreas de mejora en las habilidades de la persona y en su desempeño laboral. Si hay un comentario relacionado con la inteligencia emocional o una crítica a al carácter de la persona sé muy cauteloso y presta atención si el comentario es objetivo y si habla con hechos, ya que es posible que pueda involucrar un ataque personal, tómalo como un comentario constructivo si brinda hechos e información de forma objetiva. * biased: cualquier comentario o crítica relacionada con la raza, color de piel, creencias, sexo, preferencias sexuales de la persona evaluada o comentarios con insultos y ataques personales. No es lo mismo que un comentario negativo porque no es imparcial. - * notUseful: cualquier comentario que no sea positivo, ni negativo, ni sesgado, ni aporte ningún tipo de valor a las clasificaciones previas. - 3. En cada una de las 4 clasificaciones de sentimiento une todas las oraciones en un párrafo. Al unir las oraciones de cada clasificación debe haber una conexión clara entre las ideas, pero es posible que en la unión no haya coherencia gramatical o por signos de puntuación, si ese es el caso puedes modificar ligeramente las palabras o signos para unir todas las oraciones en un párrafo de la clasificación en cuestión, pero no alteres el contenido del mensaje que expresan. - 4. Separa las 4 clasificaciones con el separador "\n\n" que solo puede aparecer entre la clasificación de cada sentimiento, no en el párrafo formado de oraciones de cada clasificación. + 3. En cada una de las 3 clasificaciones de sentimiento une todas las oraciones en un párrafo. Al unir las oraciones de cada clasificación debe haber una conexión clara entre las ideas, pero es posible que en la unión no haya coherencia gramatical o por signos de puntuación, si ese es el caso puedes modificar ligeramente las palabras o signos para unir todas las oraciones en un párrafo de la clasificación en cuestión, pero no alteres el contenido del mensaje que expresan. + 4. Separa las 3 clasificaciones con el separador "\n\n" que solo puede aparecer entre la clasificación de cada sentimiento, no en el párrafo formado de oraciones de cada clasificación. 5. Si hay clasificaciones de sentimientos que no cuentan con ninguna oración porque ninguna cayó en esa categoría, aun así incluye el nombre del sentimiento con el separador definido en el paso anterior. 6. Es importante que en tu respuesta el orden de las clasificaciones de sentimientos sea el mismo en que los presenté en el paso 2. 7. No respondas ni expliques tu procedimiento, limítate a cumplir con las instrucciones especificadas con la estructura especificada, haz el análisis de todo el contenido del texto sin dejar oraciones sin procesar. - Este es un ejemplo del resultado esperado, la categoría 'Sesgado' se encuentra vacía porque ningún comentario encajó en las instrucciones proporcionadas de ese sentimiento y así se deben representar las categorías cuando ningún comentario pertenezca a ella, recuerda que es solo un ejemplo, pon atención en la estructura, no tanto en el contenido: + Este es un ejemplo del resultado esperado, la categoría 'biased' se encuentra vacía porque ningún comentario encajó en las instrucciones proporcionadas de ese sentimiento y así se deben representar las categorías cuando ningún comentario pertenezca a ella, recuerda que es solo un ejemplo, pon atención en la estructura, no tanto en el contenido: """ positive: Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. \n\n negative: Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat \n\n biased: - \n\n - notUseful: Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. """`; - // classify the feedback into 4 categories: positive, negative, biased, not useful + // classify the feedback into 3 categories: positive, negative, biased const classifiedFeedback = await openai.chat.completions.create({ model: "gpt-4", messages: [ @@ -179,23 +182,32 @@ async function processOpenFeedback(userFeedback: string) { const sentiments = classifiedFeedback.choices[0].message.content!.split("\n\n"); const cleanedSentiments = sentiments.filter((element) => element !== ""); + + const feedbackClassifications: FeedbackClassifications = { + positive: {}, + negative: {}, + biased: {}, + }; + + var positiveFeedback: string = ""; var negativeFeedback: string = ""; + var biasedFeedback: string = ""; + for (let sentiment of cleanedSentiments) { - if (sentiment.includes("negative:")) { + if (sentiment.includes("positive:")) { + positiveFeedback = sentiment.substring(10); + } else if (sentiment.includes("negative:")) { negativeFeedback = sentiment.substring(10); + } else if (sentiment.includes("biased:")) { + biasedFeedback = sentiment.substring(8); } } - return summarizedFeedbackCategories; -} + // analyze the skills in all the classifications -async function processClosedFeedback(answers: [number, number][]) { - const ordinaryPerformance = 7; - const userCategories = { goodCategories: [], badCategories: [] }; + // add the skills, the coworker is not needed at this stage - // get the classifications of each closed question - - return userCategories; + return feedbackClassifications; } async function orderFeedback(sprintSurveyId: number, uniqueWorkers: string[]) { @@ -346,6 +358,31 @@ async function getFeedbackClassifications( // the user has negative performance, an analysis of the comment is needed if (negativePerformanceCount > 1) { + let coworkerFeedback = coworkersFeedback[coworkerId].openFeedback; + if (coworkerFeedback !== "") { + let openFeedbackClassifications = + await processOpenFeedback(coworkerFeedback); + for (let sentiment of Object.keys( + openFeedbackClassifications, + ) as (keyof FeedbackClassifications)[]) { + for (let skill of Object.keys( + openFeedbackClassifications[sentiment], + )) { + // check if the skill already exists in the classification + if (skill in feedbackClassifications[sentiment]) { + // the skill already exists in the classification, check if the coworker is already there + if ( + !feedbackClassifications[sentiment][skill].includes(coworkerId) + ) { + feedbackClassifications[sentiment][skill].push(coworkerId); + } + } else { + // the skill does not exist in the classification, add the coworker + feedbackClassifications[sentiment][skill] = [coworkerId]; + } + } + } + } } } @@ -480,35 +517,30 @@ export async function feedback_analysis(sprintSurveyId: number) { ); // all feedback summarized, now get the classifications of negative feedback with the most suggestions - const feedbackSuggestions: [number, string][] = []; + const userNegativeSkills: [number, number][] = []; // [coworkersCount, negativeSkillId] + + // get the negative skills associated Object.keys( orderedFeedback[userId].feedbackClassifications.negative, - ).forEach((key) => { - feedbackSuggestions.push([ - orderedFeedback[userId].feedbackClassifications.negative[key] - .length, - key, + ).forEach((negativeSkill) => { + userNegativeSkills.push([ + orderedFeedback[userId].feedbackClassifications.negative + .negativeSkill.length, + Number(negativeSkill), ]); }); - feedbackSuggestions.sort((a, b) => b[0] - a[0]); - - const feedbackToCreate = feedbackSuggestions[0][1]; + // sort the negative skills by the number of coworkers that suggested them in descending order + userNegativeSkills.sort((a, b) => b[0] - a[0]); - const newCoworkerId = - orderedFeedback[userId].feedbackClassifications.negative[ - feedbackToCreate - ][0]; - - const feedbackComment = - orderedFeedback[userId].coworkersFeedback[newCoworkerId] - .openFeedback[0][1]; + // get the strings of the negative skills // select the best tasks and resources const allResources: EmbeddingRecord[] = await db .select({ id: pipResource.id, embedding: pipResource.embedding }) .from(pipResource); - const resources = await cosineSimilarity(feedbackComment, allResources); + + const resources = await selectTasks(); const tasks = await createTasks(feedbackComment); From 267e6992bcf711b3077353e2f72e7cc66afcbe01 Mon Sep 17 00:00:00 2001 From: Eduardo de Valle Date: Mon, 27 May 2024 16:11:54 -0600 Subject: [PATCH 10/10] Reduce for loops using sets instead of arrays --- services/rag.ts | 75 ++++++++++++++++++------------------------------- 1 file changed, 27 insertions(+), 48 deletions(-) diff --git a/services/rag.ts b/services/rag.ts index c12dc1e..0c4ddb5 100644 --- a/services/rag.ts +++ b/services/rag.ts @@ -183,31 +183,23 @@ async function processOpenFeedback(userFeedback: string) { classifiedFeedback.choices[0].message.content!.split("\n\n"); const cleanedSentiments = sentiments.filter((element) => element !== ""); - const feedbackClassifications: FeedbackClassifications = { - positive: {}, - negative: {}, - biased: {}, + const commentClassifications = { + positive: "", + negative: "", + biased: "", }; - var positiveFeedback: string = ""; - var negativeFeedback: string = ""; - var biasedFeedback: string = ""; - for (let sentiment of cleanedSentiments) { if (sentiment.includes("positive:")) { - positiveFeedback = sentiment.substring(10); + commentClassifications.positive = sentiment.substring(10); } else if (sentiment.includes("negative:")) { - negativeFeedback = sentiment.substring(10); + commentClassifications.negative = sentiment.substring(10); } else if (sentiment.includes("biased:")) { - biasedFeedback = sentiment.substring(8); + commentClassifications.biased = sentiment.substring(8); } } - // analyze the skills in all the classifications - - // add the skills, the coworker is not needed at this stage - - return feedbackClassifications; + return commentClassifications; } async function orderFeedback(sprintSurveyId: number, uniqueWorkers: string[]) { @@ -307,6 +299,7 @@ async function getFeedbackClassifications( // read the feedback of each coworker and classify it for (let coworkerId of Object.keys(coworkersFeedback)) { let closedFeedback = coworkersFeedback[coworkerId].closedFeedback; + let coworkerRecommendations = new Set([]); let positivePerformanceCount = 0; let negativePerformanceCount = 0; // read all the closed feedback of the coworker @@ -360,33 +353,21 @@ async function getFeedbackClassifications( if (negativePerformanceCount > 1) { let coworkerFeedback = coworkersFeedback[coworkerId].openFeedback; if (coworkerFeedback !== "") { - let openFeedbackClassifications = - await processOpenFeedback(coworkerFeedback); - for (let sentiment of Object.keys( - openFeedbackClassifications, - ) as (keyof FeedbackClassifications)[]) { - for (let skill of Object.keys( - openFeedbackClassifications[sentiment], - )) { - // check if the skill already exists in the classification - if (skill in feedbackClassifications[sentiment]) { - // the skill already exists in the classification, check if the coworker is already there - if ( - !feedbackClassifications[sentiment][skill].includes(coworkerId) - ) { - feedbackClassifications[sentiment][skill].push(coworkerId); - } - } else { - // the skill does not exist in the classification, add the coworker - feedbackClassifications[sentiment][skill] = [coworkerId]; - } - } + let sentimentsClassified = await processOpenFeedback(coworkerFeedback); + if (sentimentsClassified.negative !== "") { + const allResources: EmbeddingRecord[] = await db + .select({ id: pipResource.id, embedding: pipResource.embedding }) + .from(pipResource); + const recommendedResources = await cosineSimilarity( + sentimentsClassified.negative, + allResources, + ); } } } } - return feedbackClassifications; + return [feedbackClassifications, ragRecomendations]; } async function getQuestionsSkills(sprintSurveyId: number) { @@ -510,11 +491,14 @@ export async function feedback_analysis(sprintSurveyId: number) { // safety double check if the user has been checked in case of a failure in the middle of a previous survey analysis if (userTasksCount[0].count == 0 || userResourcesCount[0].count == 0) { - orderedFeedback[userId]["feedbackClassifications"] = - await getFeedbackClassifications( - orderedFeedback[userId].coworkersFeedback, - questionsSkills, - ); + let coworkerRecommendations = new Set([]); + [ + orderedFeedback[userId]["feedbackClassifications"], + coworkerRecommendations, + ] = await getFeedbackClassifications( + orderedFeedback[userId].coworkersFeedback, + questionsSkills, + ); // all feedback summarized, now get the classifications of negative feedback with the most suggestions const userNegativeSkills: [number, number][] = []; // [coworkersCount, negativeSkillId] @@ -535,11 +519,6 @@ export async function feedback_analysis(sprintSurveyId: number) { // get the strings of the negative skills - // select the best tasks and resources - const allResources: EmbeddingRecord[] = await db - .select({ id: pipResource.id, embedding: pipResource.embedding }) - .from(pipResource); - const resources = await selectTasks(); const tasks = await createTasks(feedbackComment);