Add gpt-3.5-turbo-0125 (#60)

* add `gpt-3.5-turbo-0125` * fix formatting issues
zurawiki · Mar 20, 2024 · a88091f · a88091f
1 parent d9dcfd8
commit a88091f
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 0 deletions.
diff --git a/tiktoken-rs/src/model.rs b/tiktoken-rs/src/model.rs
@@ -44,6 +44,9 @@ pub fn get_context_size(model: &str) -> usize {
     if starts_with_any!(model, "gpt-4") {
         return 8192;
     }
+    if starts_with_any!(model, "gpt-3.5-turbo-0125") {
+        return 16_385;
+    }
     if starts_with_any!(model, "gpt-3.5-turbo-1106") {
         return 16_385;
     }

diff --git a/tiktoken-rs/src/tokenizer.rs b/tiktoken-rs/src/tokenizer.rs
@@ -145,6 +145,10 @@ mod tests {
             get_tokenizer("gpt-4-1106-preview"),
             Some(Tokenizer::Cl100kBase)
         );
+        assert_eq!(
+            get_tokenizer("gpt-3.5-turbo-0125"),
+            Some(Tokenizer::Cl100kBase),
+        );
         assert_eq!(
             get_tokenizer("gpt-3.5-turbo-1106"),
             Some(Tokenizer::Cl100kBase),