Update the tests.

huggingface · Aug 6, 2024 · 5026d82 · 5026d82
1 parent 378e4e4
commit 5026d82
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 2 deletions.
diff --git a/tokenizers/src/models/bpe/serialization.rs b/tokenizers/src/models/bpe/serialization.rs
@@ -178,10 +178,14 @@ mod test {
             .build()
             .unwrap();
 
+        let legacy = r#"{"type":"BPE","dropout":null,"unk_token":"<unk>","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"<unk>":0,"a":1,"b":2,"ab":3},"merges":["a b"]}"#;
+        let legacy = serde_json::from_str(&legacy).unwrap();
+        assert_eq!(bpe, legacy);
+
         let data = serde_json::to_string(&bpe).unwrap();
         assert_eq!(
             data,
-            r#"{"type":"BPE","dropout":null,"unk_token":"<unk>","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"<unk>":0,"a":1,"b":2,"ab":3},"merges":["a b"]}"#
+            r#"{"type":"BPE","dropout":null,"unk_token":"<unk>","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"<unk>":0,"a":1,"b":2,"ab":3},"merges":[["a","b"]]}"#
         );
         let reconstructed = serde_json::from_str(&data).unwrap();
 

diff --git a/tokenizers/src/models/mod.rs b/tokenizers/src/models/mod.rs
@@ -312,11 +312,14 @@ mod tests {
             .unwrap();
 
         let model = ModelWrapper::BPE(bpe);
+        let legacy = r#"{"type":"BPE","dropout":null,"unk_token":"<unk>","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"<unk>":0,"a":1,"b":2,"ab":3},"merges":["a b"]}"#;
+        let legacy = serde_json::from_str(&legacy).unwrap();
+        assert_eq!(model, legacy);
 
         let data = serde_json::to_string(&model).unwrap();
         assert_eq!(
             data,
-            r#"{"type":"BPE","dropout":null,"unk_token":"<unk>","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"<unk>":0,"a":1,"b":2,"ab":3},"merges":["a b"]}"#
+            r#"{"type":"BPE","dropout":null,"unk_token":"<unk>","continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":true,"vocab":{"<unk>":0,"a":1,"b":2,"ab":3},"merges":[["a","b"]]}"#
         );
         let reconstructed = serde_json::from_str(&data).unwrap();
         assert_eq!(model, reconstructed);