Add length of second text added tokens check

huggingface · Aug 5, 2023 · 37bb179 · 37bb179
1 parent d05bbb7
commit 37bb179
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs
@@ -934,7 +934,11 @@ where
                 (encoding, pair_encoding)
             }
         };
-        let original_length = encoding.len();
+        let original_length = if let Some(second_encoding) = &pair_encoding {
+            encoding.len() + second_encoding.len()
+        } else {
+            encoding.len()
+        };
 
         // 2. Then We post process
         let final_encoding = if let Some(processor) = &self.post_processor {