Merge pull request #1144 from SwordYork/master

fix inconsistent decay of beta1 in Adam
mila-iqia · Jan 4, 2017 · e89cf93 · e89cf93
2 parents 8e81bfe + a31ef0b
commit e89cf93
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/blocks/algorithms/__init__.py b/blocks/algorithms/__init__.py
@@ -877,10 +877,10 @@ def compute_step(self, parameter, previous_step):
         add_role(time, ALGORITHM_BUFFER)
 
         t1 = time + 1
+        beta_1t = 1. - (1. - self.beta1) * self.decay_factor ** (t1 - 1)
         learning_rate = (self.learning_rate *
                          tensor.sqrt((1. - (1. - self.beta2)**t1)) /
-                         (1. - (1. - self.beta1)**t1))
-        beta_1t = 1 - (1 - self.beta1) * self.decay_factor ** (t1 - 1)
+                         (1. - (1. - beta_1t)**t1))
         mean_t = beta_1t * previous_step + (1. - beta_1t) * mean
         variance_t = (self.beta2 * tensor.sqr(previous_step) +
                       (1. - self.beta2) * variance)