jacobian

commit 73a26f8eee33bd1c54ca618d83b24f190d6206ed
parent 5a54e69a9ffb25a62db86b8b44166b20755279ed
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sat,  1 Aug 2020 18:08:48 -0700

Adam no longer bugs out

Diffstat:
M
example.cpp
 | 
2
+-
M
src/bpnn.cpp
 | 
2
+-

2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -18,7 +18,7 @@ double bench(int batch_sz)
   net.add_layer(4, "linear");
   net.add_layer(5, "relu");
   net.add_layer(2, "linear");
-  //  net.init_optimizer("momentum", 0.9);
+  net.init_optimizer("adam", 0.9, 0.999, 0.000001);
   net.initialize();
   //net.grad_check();
   std::vector<float> vals;
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -119,7 +119,7 @@ void Network::init_optimizer(char* name, ...)
     va_end(args);
     // TODO: Add bias correction (requires figuring out measuring t)
     update = [this, beta1, beta2, epsilon](std::vector<Eigen::MatrixXf> deltas, int i) {
-      *layers[length-2-i].weights -= *layers[length-2-i].m * learning_rate * ((layers[length-2-i].v->cwiseSqrt()).array()+epsilon).pow(-1).matrix();
+      *layers[length-2-i].weights -= learning_rate * ((layers[length-2-i].v->cwiseSqrt()).array()+epsilon).pow(-1).cwiseProduct(layers[length-2-i].m->array()).matrix();
       *layers[length-2-i].m = (beta1 * *layers[length-2-i].m) + ((1-beta1)*deltas[i]);
       *layers[length-2-i].v = (beta2 * *layers[length-2-i].v) + (1-beta2)*(deltas[i].cwiseProduct(deltas[i]));
     };