try: use openAI file ids to refer to the concatenated files; also pri…

…nt them server side to be inspected

src/lib/infrastructure/server/config/openai/openai-utils.ts

@@ -158,18 +158,18 @@ export const DATA_FILE_FORMATS: string[] = [".json", ".txt"];
 
 export const WEBSAT_VECTOR_STORE_FILE_FORMATS = OPENAI_VECTOR_STORE_SUPPORTED_FILE_FORMATS.filter((format) => !DATA_FILE_FORMATS.includes(format));
 
-export const generateOpenAiAssistantInstructions = (researchContextTitle: string, researchContextDescription: string, jsonFileName: string | undefined, txtFileName: string | undefined): string => {
+export const generateOpenAiAssistantInstructions = (researchContextTitle: string, researchContextDescription: string, jsonFileId: string | undefined, txtFileId: string | undefined): string => {
   // concatenated-json-files.json
   // concatenated-txt-files.txt
 
   let jsonFileInstructions = "";
-  if (jsonFileName !== undefined) {
-    jsonFileInstructions = ` The file '${jsonFileName}' contains a concatenation of JSON data. Within this file, there are comments that indicate the source files of the JSON data. When a user references a specific JSON source file, please search for the data that is between the start of file comment which refers to that source file name, and the next end of file comment.`;
+  if (jsonFileId !== undefined) {
+    jsonFileInstructions = ` The file '${jsonFileId}' contains a concatenation of JSON data. Within this file, there are markers that indicate the source files of the JSON data. The markers have the following structure: \"### START OF FILE: '\${file.name}' ###\" and \"### END OF FILE ###\". When a user references a specific JSON source file, please search for the data that is between the start of file comment which refers to that source file name, and the next end of file comment.`;
   }
 
   let txtFileInstructions = "";
-  if (txtFileName !== undefined) {
-    txtFileInstructions = ` The file '${txtFileName}' contains a concatenation of TXT data. Within this file, there are comments that indicate the source files of the TXT data. When a user references a specific TXT source file, please search for the data that is between the start of file comment which refers to that source file name, and the next end of file comment.`;
+  if (txtFileId !== undefined) {
+    txtFileInstructions = ` The file '${txtFileId}' contains a concatenation of TXT data. Within this file, there are markers that indicate the source files of the TXT data. The markers have the following structure: \"### START OF FILE: '\${file.name}' ###\" and \"### END OF FILE ###\". When a user references a specific TXT source file, please search for the data that is between the start of file comment which refers to that source file name, and the next end of file comment.`;
   }
 
   const instructions = `You are an expert data analyst specialized working in the research context with title \"${researchContextTitle}\". This research context has the following description \"${researchContextDescription}\". You have also been assigned some files and you have access to scraped data and some results produced by us in your vector store. Some of these are images in different formats (e.g., JPG, JPEG, PNG, etc.), assigned to you via normal code interpreter.${jsonFileInstructions}${txtFileInstructions} Other files, containing key data, have been assigned to you via a vector store. Please consider files from both sources anytime the user asks you about files you have access to, and name which sources you're drawing from.

src/lib/infrastructure/server/gateway/openai-agent-gateway.ts

@@ -29,14 +29,14 @@ export default class OpenAIAgentGateway implements AgentGatewayOutputPort<TOpenA
   }
 
   async createAgent(researchContextTitle: string, researchContextDescription: string, vectorStoreID: string, additionalFiles?: RemoteFile[], agentSystemInstructions?: string): Promise<TCreateAgentDTO> {
-    const jsonFile = additionalFiles?.find((file) => file.relativePath.endsWith(".json"))?.name;
-    const txtFile = additionalFiles?.find((file) => file.relativePath.endsWith(".txt"))?.name;
+    const jsonFileId = additionalFiles?.find((file) => file.relativePath.endsWith(".json"))?.id;
+    const txtFileId = additionalFiles?.find((file) => file.relativePath.endsWith(".txt"))?.id;
 
     let instructions: string;
     if (agentSystemInstructions) {
       instructions = agentSystemInstructions;
     } else {
-      instructions = generateOpenAiAssistantInstructions(researchContextTitle, researchContextDescription, jsonFile, txtFile);
+      instructions = generateOpenAiAssistantInstructions(researchContextTitle, researchContextDescription, jsonFileId, txtFileId);
     }
 
     const model = "gpt-4o";

src/lib/infrastructure/server/gateway/openai-vector-store-gateway.ts

@@ -106,8 +106,8 @@ export default class OpenAIVectorStoreGateway implements VectorStoreOutputPort {
             return `### START OF FILE: '${file.name}' ###\n${fileContent}\n### END OF FILE ###\n`;
           })
           .join("\n");
-
-        this.logger.info({ concatenatedJSONFileContent }, "DEBUG: Concatenated JSON file content");
+        // print the concatenated JSON file content to a file
+        fs.writeFileSync("concatenated-json-files.json", concatenatedJSONFileContent);
       }
 
       let concatenatedTxtFileContent = null;
@@ -118,8 +118,8 @@ export default class OpenAIVectorStoreGateway implements VectorStoreOutputPort {
             return `### START OF FILE: '${file.name}' ###\n${fileContent}\n### END OF FILE ###\n`;
           })
           .join("\n");
-
-        this.logger.info({ concatenatedTxtFileContent }, "DEBUG: Concatenated TXT file content");
+        // print the concatenated TXT file content to a file
+        fs.writeFileSync("concatenated-txt-files.txt", concatenatedTxtFileContent);
       }
 
       const scratchDir = process.env.SCRATCH_DIR ?? "/tmp";