Skip to content

Commit

Permalink
Add average mmlu accuracy
Browse files Browse the repository at this point in the history
Signed-off-by: Akshay Sonawane <[email protected]>
  • Loading branch information
apsonawane committed Feb 1, 2025
1 parent 4f3be13 commit 459c70f
Showing 1 changed file with 5 additions and 0 deletions.
5 changes: 5 additions & 0 deletions src/lemonade/tools/mmlu.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,11 @@ def run(
self.status_stats.append(stat_name)

# Save accuracy results to CSV file
acc_avg = np.mean([accuracy_data["Accuracy"] for accuracy_data in summary_data])
avg_stat_name = "avg_accuracy"
state.save_stat(avg_stat_name, float(acc_avg) * 100)
state.save_stat("accuracy_units", "%")
self.status_stats.append(avg_stat_name)
summary_df = pd.DataFrame(summary_data)
summary_df.to_csv(
os.path.join(model_results_dir, "summary_results.csv"), index=False
Expand Down

0 comments on commit 459c70f

Please sign in to comment.