Use the custom eos/bos tokens passed to the bytelevel tokenizer

Signed-off-by: John St John <[email protected]>
NVIDIA · Feb 27, 2025 · 44d08b5 · 44d08b5
1 parent 8ca01eb
commit 44d08b5
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/nemo/collections/common/tokenizers/bytelevel_tokenizers.py b/nemo/collections/common/tokenizers/bytelevel_tokenizers.py
@@ -167,21 +167,21 @@ def pad_id(self):
         """
         Get the padding ID.
         """
-        return 256
+        return self._pad_id
 
     @property
     def bos_id(self):
         """
         Get the beginning-of-sequence ID.
         """
-        return 257
+        return self._bos_id
 
     @property
     def eos_id(self):
         """
         Get the end-of-sequence ID.
         """
-        return 258
+        return self._eos_id
 
     @property
     def unk_id(self):