Hermes Checkpoint creation

.gitignore

@@ -827,3 +827,4 @@ models/
 # Do not ignore scripts directory
 !scripts/
 scripts/MergeKit
+scripts/test.py

scripts/data_prep.py

@@ -80,11 +80,11 @@ def prepare_dataset(
 
 if __name__ == "__main__":
     dataset_name = "teknium/OpenHermes-2.5"
-    tokenizer_name = "microsoft/Phi-3-mini-4k-instruct"
+    tokenizer_name = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
 
-    tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct", padding=True, truncation=True)
+    tokenizer = AutoTokenizer.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0", padding=True, truncation=True)
     tokenizer.pad_token = tokenizer.eos_token
-    tokenizer.padding_side = "left"
+    tokenizer.padding_side = "right"
 
     prepared_dataset = prepare_dataset(dataset_name, tokenizer, sample_percentage=1)
 

scripts/sft.py

@@ -44,7 +44,7 @@ def initialize_model_and_tokenizer():
 
 def load_dataset():
     try:
-        dataset = load_from_disk("data/sft_mmlu")
+        dataset = load_from_disk("data/sft")
         logger.info("Dataset loaded from disk.")
         return dataset
     except Exception as e:
@@ -108,7 +108,7 @@ def main():
         model, tokenizer = initialize_model_and_tokenizer()
 
         dataset = load_dataset()
-        train_dataset = dataset["auxiliary_train"]
+        train_dataset = dataset["train"]
         eval_dataset = dataset["test"]
 
         trainer = prepare_trainer(model, tokenizer, train_dataset, eval_dataset)