[eval] Add IMO problems with exact answers (#1528)

openai · Jul 13, 2024 · 234bcde · 234bcde
1 parent d3dc890
commit 234bcde
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 0 deletions.
diff --git a/evals/registry/data/imo_exact_answers/samples.jsonl b/evals/registry/data/imo_exact_answers/samples.jsonl
diff --git a/evals/registry/evals/imo_exact_answers.yaml b/evals/registry/evals/imo_exact_answers.yaml
@@ -0,0 +1,9 @@
+imo_exact_answers:
+  id: imo_exact_answers.dev.v0
+  description: A small set of IMO problems that have exact answers (e.g. yes/no, number, fraction).
+  metrics: [accuracy]
+
+imo_exact_answers.dev.v0:
+  class: evals.elsuite.basic.includes:Includes
+  args:
+    samples_jsonl: imo_exact_answers/samples.jsonl