Handle when precompiled charsmap is empty

huggingface · Jul 28, 2023 · 3ee82bd · 3ee82bd
1 parent c2664ae
commit 3ee82bd
Showing 1 changed file with 11 additions and 6 deletions.
diff --git a/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py b/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py
@@ -173,12 +173,17 @@ def from_spm(filename: str):
 
         tokenizer = Tokenizer(Unigram(vocab, unk_id, byte_fallback))
 
-        tokenizer.normalizer = normalizers.Sequence(
-            [
-                normalizers.Precompiled(precompiled_charsmap),
-                normalizers.Replace(Regex(" {2,}"), " "),
-            ]
-        )
+        if precompiled_charsmap:
+            tokenizer.normalizer = normalizers.Sequence(
+                [
+                    normalizers.Precompiled(precompiled_charsmap),
+                    normalizers.Replace(Regex(" {2,}"), " "),
+                ]
+            )
+        else:
+            tokenizer.normalizer = normalizers.Sequence(
+                [normalizers.Replace(Regex(" {2,}"), " ")]
+            )
         tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space)
         tokenizer.decoder = decoders.Metaspace(replacement=replacement, add_prefix_space=add_prefix_space)