change initial value of rezero parameter to 1e-3 per @fcampagne sugge…

…stion
lucidrains · Nov 17, 2020 · aa726b5 · aa726b5
1 parent 5e9e9b8
commit aa726b5
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/performer_pytorch/performer_pytorch.py b/performer_pytorch/performer_pytorch.py
@@ -190,7 +190,7 @@ def forward(self, q, k, v):
 class ReZero(nn.Module):
     def __init__(self, fn):
         super().__init__()
-        self.g = nn.Parameter(torch.zeros(1))
+        self.g = nn.Parameter(torch.tensor(1e-3))
         self.fn = fn
 
     def forward(self, x, **kwargs):

diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'performer-pytorch',
   packages = find_packages(exclude=['examples']),
-  version = '0.9.1',
+  version = '0.9.2',
   license='MIT',
   description = 'Performer - Pytorch',
   author = 'Phil Wang',