remove unknown token from known vocab (#440)

in FilterCorpusRemoveUnknownWordSegmentsJob
rwth-i6 · Aug 25, 2023 · 6b06f97 · 6b06f97
1 parent 8c25c89
commit 6b06f97
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/corpus/filter.py b/corpus/filter.py
@@ -241,6 +241,12 @@ def maybe_to_lower(s):
         with open_func(lex_path, "rt") as f:
             lex_root = ET.parse(f)
         vocabulary = set([maybe_to_lower(o.text.strip() if o.text else "") for o in lex_root.findall(".//orth")])
+        vocabulary -= {
+            maybe_to_lower(o.text.strip() if o.text else "")
+            for l in lex_root.findall(".//lemma")
+            if l.attrib.get("special") == "unknown"
+            for o in l.findall(".//orth")
+        }
 
         c = corpus.Corpus()
         c.load(self.corpus.get_path())