templateミスってた

yuiseki · Mar 28, 2024 · d5cb6b3 · d5cb6b3
1 parent c980304
commit d5cb6b3
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 179 deletions.
diff --git a/src/dataset/prepare.py b/src/dataset/prepare.py
diff --git a/src/train.py b/src/train.py
@@ -36,7 +36,7 @@ def simple_template_for_pretrain(input) -> str:
     # inputから、2つ以上連続する改行を除去する
     input = "\n".join([line for line in input.splitlines() if line.strip() != ""])
     template = f"""\
-    {input}
+    {input}\
     """
     # Remove any leading whitespace characters from each line in the template.
     template = "\n".join([line.lstrip() for line in template.splitlines()])
@@ -50,7 +50,7 @@ def simple_template_for_train(input, output) -> str:
     <|im_end|>
     <|im_start|>assistant
     {output}
-    <|im_end|>
+    <|im_end|>\
     """
     # Remove any leading whitespace characters from each line in the template.
     template = "\n".join([line.lstrip() for line in template.splitlines()])
@@ -65,7 +65,7 @@ def hint_template_for_train(hint, question, answer):
     <|im_end|>
     <|im_start|>assistant
     {answer}
-    <|im_end|>
+    <|im_end|>\
     """
     # Remove any leading whitespace characters from each line in the template.
     template = "\n".join([line.lstrip() for line in template.splitlines()])
@@ -80,7 +80,7 @@ def context_template_for_train(context, question, answer):
     <|im_end|>
     <|im_start|>assistant
     {answer}
-    <|im_end|>
+    <|im_end|>\
     """
     # Remove any leading whitespace characters from each line in the template.
     template = "\n".join([line.lstrip() for line in template.splitlines()])
@@ -98,7 +98,7 @@ def context_hint_template_for_train(hint, context, question, answer):
     <|im_end|>
     <|im_start|>assistant
     {answer}
-    <|im_end|>
+    <|im_end|>\
     """
     # Remove any leading whitespace characters from each line in the template.
     template = "\n".join([line.lstrip() for line in template.splitlines()])

diff --git a/src/train/tokenizer.py b/src/train/tokenizer.py