From e8511b0fb06d7d81dfc4974740190af65e86ad00 Mon Sep 17 00:00:00 2001 From: Congyu WANG Date: Wed, 25 Sep 2024 21:56:57 +0800 Subject: [PATCH] add support for chatgpt-4o-latest --- tiktoken-rs/src/tokenizer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tiktoken-rs/src/tokenizer.rs b/tiktoken-rs/src/tokenizer.rs index 4846e2d..2cd51a5 100644 --- a/tiktoken-rs/src/tokenizer.rs +++ b/tiktoken-rs/src/tokenizer.rs @@ -30,6 +30,7 @@ pub enum Tokenizer { const MODEL_PREFIX_TO_TOKENIZER: &[(&str, Tokenizer)] = &[ // chat + ("chatgpt-4o-latest", Tokenizer::O200kBase), ("gpt-4o-", Tokenizer::O200kBase), ("gpt-4-", Tokenizer::Cl100kBase), ("gpt-3.5-turbo-", Tokenizer::Cl100kBase),