diff --git a/requirements.txt b/requirements.txt
index 444da1912..e3acb5335 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ typing
 jax>=0.4.10
 jaxlib>=0.4.10
 flax
-fjformer>=0.0.1
+fjformer>=0.0.5
 transformers>=4.33.0
 einops
 optax
diff --git a/setup.py b/setup.py
index ee87ab029..88dac54fa 100644
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@
         "jax>=0.4.10",
         "jaxlib>=0.4.10",
         "flax",
-        "fjformer>=0.0.1",
+        "fjformer>=0.0.5",
         "transformers>=4.33.0",
         "einops>=0.6.1",
         "optax",