Fix fewshot_config sampler constructor error

EvolvingLMMs-Lab · Mar 27, 2024 · 5dca4e1 · 5dca4e1
1 parent 2571960
commit 5dca4e1
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/lmms_eval/api/task.py b/lmms_eval/api/task.py
@@ -553,9 +553,12 @@ def __init__(self, model_name) -> None:  # TODO no super() call here
         if self.config.task == "flickr30k_test":
             pass  # TODO: for test, will delete later
         if self.config.fewshot_config is not None:
-            random_seed = self.config.fewshot_config.get("random_seed", 1234)
-            sampler_function = samplers.get_sampler(self.config.fewshot_config.get("sampler", "default") if self.config.fewshot_config else "default")
-            self.sampler = sampler_function(list(self.fewshot_docs()), self, rnd=random.Random(random_seed))
+            try:
+                random_seed = self.config.fewshot_config.get("random_seed", 1234)
+                sampler_constructor = samplers.get_sampler(self.config.fewshot_config.get("sampler", "default") if self.config.fewshot_config else "default")
+                self.sampler = sampler_constructor(list(self.fewshot_docs()), self, rnd=random.Random(random_seed))
+            except Exception as e:
+                eval_logger.error(f"Error in fewshot_config: {e}")
 
         if self.has_test_docs():
             self.task_docs = self.test_docs()