diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt index 4a722b1c4..8f0067550 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt @@ -49,7 +49,7 @@ class ConversationSpec : } """" - | GPT_3_5_TURBO model has 4097 max context length + | GPT_3_5_TURBO model has 16385 max context length | when the number of token in the conversation is greater than | the space allotted for the message history in the prompt configuration | the number of messages in the request must have fewer messages than @@ -80,7 +80,7 @@ class ConversationSpec : val memories = vectorStore.memories(model, conversationId, totalTokens) // The messages in the request doesn't contain the message response - val messagesSizePlusMessageResponse = lastRequest.messages.size + 1 + val messagesSizePlusMessageResponse = lastRequest.messages.size messagesSizePlusMessageResponse shouldBeLessThan memories.size } diff --git a/tokenizer/src/commonMain/kotlin/com/xebia/functional/tokenizer/ModelType.kt b/tokenizer/src/commonMain/kotlin/com/xebia/functional/tokenizer/ModelType.kt index 3c0fee90b..4a4c5f4f4 100644 --- a/tokenizer/src/commonMain/kotlin/com/xebia/functional/tokenizer/ModelType.kt +++ b/tokenizer/src/commonMain/kotlin/com/xebia/functional/tokenizer/ModelType.kt @@ -38,6 +38,7 @@ sealed class ModelType( val DEFAULT_SPLITTER_MODEL = GPT_3_5_TURBO val functionSpecific: List = listOf( GPT_3_5_TURBO_0613, + GPT_3_5_TURBO_0125, GPT_3_5_16K_0613_TURBO_FUNCTIONS, GPT_4_32K_0314_FUNCTIONS, GPT_4_0613, @@ -50,6 +51,7 @@ sealed class ModelType( GPT_3_5_TURBO, GPT_3_5_TURBO_0301, GPT_3_5_TURBO_0613, + GPT_3_5_TURBO_0125, GPT_3_5_TURBO_FUNCTIONS, GPT_3_5_TURBO_16_K, GPT_3_5_TURBO_16_K_1106, @@ -103,11 +105,16 @@ sealed class ModelType( ) : ModelType(name, encodingType, maxContextLength) object GPT_3_5_TURBO : - ModelType("gpt-3.5-turbo", CL100K_BASE, 4097, tokensPerMessage = 4, tokensPerName = 0, tokenPadding = 5) + ModelType("gpt-3.5-turbo", CL100K_BASE, 16385, tokensPerMessage = 4, tokensPerName = 0, tokenPadding = 5) + object GPT_3_5_TURBO_0125 : + ModelType("gpt-3.5-turbo-0125", CL100K_BASE, 16385, tokensPerMessage = 4, tokensPerName = 0, tokenPadding = 5) + + @Deprecated("Use GPT_3_5_TURBO_0125 instead") object GPT_3_5_TURBO_0301 : ModelType("gpt-3.5-turbo-0301", CL100K_BASE, 4097, tokensPerMessage = 4, tokensPerName = 0, tokenPadding = 5) + @Deprecated("Use GPT_3_5_TURBO_0125 instead") object GPT_3_5_TURBO_0613 : ModelType("gpt-3.5-turbo-0613", CL100K_BASE, 4097, tokensPerMessage = 3, tokensPerName = 2, tokenPadding = 5)