Adding a few tests for decoder deserialization.

huggingface · Aug 6, 2024 · 2d27761 · 2d27761
1 parent adc82cb
commit 2d27761
Showing 1 changed file with 28 additions and 1 deletion.
diff --git a/tokenizers/src/decoders/mod.rs b/tokenizers/src/decoders/mod.rs
@@ -94,6 +94,33 @@ mod tests {
     #[test]
     fn decoder_serialization_no_decode() {
         let json = r#"{"type":"Sequence","decoders":[{},{"type":"Metaspace","replacement":"▁","prepend_scheme":"always"}]}"#;
-        assert!(serde_json::from_str::<DecoderWrapper>(json).is_err());
+        let parse = serde_json::from_str::<DecoderWrapper>(json);
+        match parse {
+            Err(err) => assert_eq!(
+                format!("{err}"),
+                "data did not match any variant of untagged enum DecoderWrapper"
+            ),
+            _ => panic!("Expected error"),
+        }
+
+        let json = r#"{"replacement":"▁","prepend_scheme":"always"}"#;
+        let parse = serde_json::from_str::<DecoderWrapper>(json);
+        match parse {
+            Err(err) => assert_eq!(
+                format!("{err}"),
+                "data did not match any variant of untagged enum DecoderWrapper"
+            ),
+            _ => panic!("Expected error"),
+        }
+
+        let json = r#"{"type":"Sequence","prepend_scheme":"always"}"#;
+        let parse = serde_json::from_str::<DecoderWrapper>(json);
+        match parse {
+            Err(err) => assert_eq!(
+                format!("{err}"),
+                "data did not match any variant of untagged enum DecoderWrapper"
+            ),
+            _ => panic!("Expected error"),
+        }
     }
 }