diff --git a/fastembed/sparse/utils/tokenizer.py b/fastembed/sparse/utils/tokenizer.py index 1ba2be82..88b6c059 100644 --- a/fastembed/sparse/utils/tokenizer.py +++ b/fastembed/sparse/utils/tokenizer.py @@ -5,7 +5,7 @@ class SimpleTokenizer: - def tokenize(text: str) -> list[str]: + def tokenize(text: str) -> List[str]: text = re.sub(r"[^\w]", " ", text.lower()) text = re.sub(r"\s+", " ", text)