From 9e6c82e208b86c251557f702d97013bf59a3e446 Mon Sep 17 00:00:00 2001
From: Mikael Souza <mikael.souza.cc@gmail.com>
Date: Mon, 5 Feb 2024 17:05:00 -0400
Subject: [PATCH] add `gpt-3.5-turbo-0125`

---
 tiktoken-rs/src/model.rs     | 3 +++
 tiktoken-rs/src/tokenizer.rs | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/tiktoken-rs/src/model.rs b/tiktoken-rs/src/model.rs
index e32a0cb..f0c98ab 100644
--- a/tiktoken-rs/src/model.rs
+++ b/tiktoken-rs/src/model.rs
@@ -44,6 +44,9 @@ pub fn get_context_size(model: &str) -> usize {
     if starts_with_any!(model, "gpt-4") {
         return 8192;
     }
+    if starts_with_any!(model, "gpt-3.5-turbo-0125"){
+        return 16_385;
+    }
     if starts_with_any!(model, "gpt-3.5-turbo-1106") {
         return 16_385;
     }
diff --git a/tiktoken-rs/src/tokenizer.rs b/tiktoken-rs/src/tokenizer.rs
index fc6f4a3..08bd0cb 100644
--- a/tiktoken-rs/src/tokenizer.rs
+++ b/tiktoken-rs/src/tokenizer.rs
@@ -145,6 +145,10 @@ mod tests {
             get_tokenizer("gpt-4-1106-preview"),
             Some(Tokenizer::Cl100kBase)
         );
+        assert_eq!(
+            get_tokenizer("gpt-3.5-turbo-0125"),
+            Some(Tokenizer::Cl100kBase),
+        );
         assert_eq!(
             get_tokenizer("gpt-3.5-turbo-1106"),
             Some(Tokenizer::Cl100kBase),