Pin eager attn in torch-ort backend (#219)

optimum_benchmark/backends/torch_ort/backend.py

@@ -82,6 +82,9 @@ def automodel_kwargs(self) -> Dict[str, Any]:
         if self.config.torch_dtype is not None:
             kwargs["torch_dtype"] = getattr(torch, self.config.torch_dtype)
 
+        if self.config.attn_implementation is not None:
+            kwargs["attn_implementation"] = self.config.attn_implementation
+
         return kwargs
 
     def train(

optimum_benchmark/backends/torch_ort/config.py

@@ -8,12 +8,14 @@
 @dataclass
 class TorchORTConfig(BackendConfig):
     name: str = "torch-ort"
-    version: Optional[str] = torch_ort_version
+    version: Optional[str] = torch_ort_version()
     _target_: str = "optimum_benchmark.backends.torch_ort.backend.TorchORTBackend"
 
     # load options
     no_weights: bool = False
     torch_dtype: Optional[str] = None
+    # sdpa, which has became default of many architectures, fails with torch ort
+    attn_implementation: Optional[str] = "eager"
 
     # peft options
     peft_type: Optional[str] = None

tests/configs/_text_encoders_.yaml

@@ -3,4 +3,4 @@ hydra:
   sweeper:
     params:
       backend.task: fill-mask,text-classification,token-classification,question-answering
-      backend.model: hf-internal-testing/tiny-random-bert,hf-internal-testing/tiny-random-roberta
+      backend.model: hf-internal-testing/tiny-random-BertModel,hf-internal-testing/tiny-random-RobertaModel