jacobian

commit 5eba72c2749bf36143f8396f9c3f2230be5b02e5
parent 73a26f8eee33bd1c54ca618d83b24f190d6206ed
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sat,  1 Aug 2020 18:32:12 -0700

Adamax goes boom

Diffstat:
M
example.cpp
 | 
2
+-
M
src/bpnn.cpp
 | 
9
+++++----

2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -18,7 +18,7 @@ double bench(int batch_sz)
   net.add_layer(4, "linear");
   net.add_layer(5, "relu");
   net.add_layer(2, "linear");
-  net.init_optimizer("adam", 0.9, 0.999, 0.000001);
+  net.init_optimizer("adamax", 0.9, 0.999, 0.000001);
   net.initialize();
   //net.grad_check();
   std::vector<float> vals;
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -118,10 +118,11 @@ void Network::init_optimizer(char* name, ...)
     float epsilon = va_arg(args, double);
     va_end(args);
     // TODO: Add bias correction (requires figuring out measuring t)
+    // TODO: cwiseProduct here is sketchy, look into me
     update = [this, beta1, beta2, epsilon](std::vector<Eigen::MatrixXf> deltas, int i) {
-      *layers[length-2-i].weights -= learning_rate * ((layers[length-2-i].v->cwiseSqrt()).array()+epsilon).pow(-1).cwiseProduct(layers[length-2-i].m->array()).matrix();
       *layers[length-2-i].m = (beta1 * *layers[length-2-i].m) + ((1-beta1)*deltas[i]);
       *layers[length-2-i].v = (beta2 * *layers[length-2-i].v) + (1-beta2)*(deltas[i].cwiseProduct(deltas[i]));
+      *layers[length-2-i].weights -= learning_rate * ((layers[length-2-i].v->cwiseSqrt()).array()+epsilon).pow(-1).cwiseProduct(layers[length-2-i].m->array()).matrix();
     };
   }
   else if (strcmp(name, "adamax") == 0) {
@@ -129,13 +130,13 @@ void Network::init_optimizer(char* name, ...)
     float beta2 = va_arg(args, double);
     float epsilon = va_arg(args, double);
     va_end(args);
-    // TODO Add bias correction for m (requires figuring out measuring t)
+    // TODO: Add bias correction for m (requires figuring out measuring t)
     update = [this, beta1, beta2, epsilon](std::vector<Eigen::MatrixXf> deltas, int i) {
-      *layers[length-2-i].weights -= *layers[length-2-i].m * learning_rate * layers[length-2-i].v->array().pow(-1).matrix();
       *layers[length-2-i].m = (beta1 * *layers[length-2-i].m) + ((1-beta1)*deltas[i]);
-      // FIXME Use of .sum() here is incredibly questionable. Do this correctly.
+      // FIXME: Use of .sum() here is incredibly questionable. Do this correctly.
       if ((beta2 * *layers[length-2-i].v).sum() > deltas[i].array().abs().sum()) *layers[length-2-i].v = (beta2 * *layers[length-2-i].v);
       else *layers[length-2-i].v = deltas[i].array().abs().matrix();
+      *layers[length-2-i].weights -= learning_rate * (layers[length-2-i].v->array().pow(-1).cwiseProduct(layers[length-2-i].m->array())).matrix();
     };
   }
 }