Only check added tokens when add_special_tokens is true

huggingface · Aug 5, 2023 · 212ac33 · 212ac33
1 parent 37bb179
commit 212ac33
Showing 1 changed file with 9 additions and 7 deletions.
diff --git a/tokenizers/src/tokenizer/mod.rs b/tokenizers/src/tokenizer/mod.rs
@@ -956,13 +956,15 @@ where
             }
             encodings.pop().unwrap()
         };
-        assert_eq!(
-            final_encoding.len() - self.get_n_added_tokens(is_pair),
-            original_length,
-            "Processor should add {} tokens but instead added {}!",
-            self.get_n_added_tokens(is_pair),
-            final_encoding.len() - original_length
-        );
+        if add_special_tokens {
+            assert_eq!(
+                final_encoding.len() - self.get_n_added_tokens(is_pair),
+                original_length,
+                "Processor should add {} tokens but instead added {}!",
+                self.get_n_added_tokens(is_pair),
+                final_encoding.len() - original_length
+            )
+        };
 
         // 3. Then we pad if needed
         let [final_encoding] = if let Some(params) = &self.padding {