🆚 Distinguish padding and eos when they differ (#2793)

huggingface · Feb 7, 2025 · 2241f17 · 2241f17
1 parent cf97133
commit 2241f17
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/trl/scripts/sft.py b/trl/scripts/sft.py
@@ -84,7 +84,8 @@ def main(script_args, training_args, model_args):
     tokenizer = AutoTokenizer.from_pretrained(
         model_args.model_name_or_path, trust_remote_code=model_args.trust_remote_code, use_fast=True
     )
-    tokenizer.pad_token = tokenizer.eos_token
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
 
     ################
     # Dataset