jacobian

commit 861f8eaab3805620755b46e00587b57838968e51
parent af578071cf826a2cd819e101000342a9ae9ccbf9
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sat,  1 Aug 2020 12:55:36 -0700

Added demon + more NaNs

Diffstat:
M
example.cpp
 | 
2
+-
M
src/bpnn.cpp
 | 
26
+++++++++++++++++++-------

2 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -18,7 +18,7 @@ double bench(int batch_sz)
   net.add_layer(4, "linear");
   net.add_layer(5, "relu");
   net.add_layer(2, "linear");
-  net.init_optimizer("adam", 0.9, 0.999, pow(10,-8));
+  net.init_optimizer("demon", 0.9, 50);
   net.initialize();
   std::vector<float> vals;
   for (int i = 0; i < 50; i++) {
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -94,14 +94,25 @@ void Network::init_optimizer(char* name, ...)
   va_list args;
   va_start(args, name);
   if (strcmp(name, "momentum") == 0) {
-    float mu = va_arg(args, double);
+    float beta = va_arg(args, double);
     va_end(args);
-    update = [this, mu](std::vector<Eigen::MatrixXf> deltas, int i) {
-      *layers[length-2-i].weights -= (mu * *layers[length-2-i].m) + (learning_rate * deltas[i]);
+    update = [this, beta](std::vector<Eigen::MatrixXf> deltas, int i) {
+      *layers[length-2-i].weights -= (beta * *layers[length-2-i].m) + (learning_rate * deltas[i]);
       *layers[length-2-i].m = (learning_rate * deltas[i]);
     };
   }
-  if (strcmp(name, "adam") == 0) {
+  else if (strcmp(name, "demon") == 0) {
+    float beta_init = va_arg(args, double);
+    float max_ep = va_arg(args, int);
+    float beta = beta_init;
+    va_end(args);
+    update = [this, max_ep, beta_init, beta](std::vector<Eigen::MatrixXf> deltas, int i) mutable {
+      beta = ((1-beta) * beta_init * (1-epochs)/max_ep)/(1-beta_init);
+      *layers[length-2-i].weights -= (beta * *layers[length-2-i].m) + (learning_rate * deltas[i]);
+      *layers[length-2-i].m = (learning_rate * deltas[i]);
+    };
+  }
+  else if (strcmp(name, "adam") == 0) {
     float beta1 = va_arg(args, double);
     float beta2 = va_arg(args, double);
     float epsilon = va_arg(args, double);
@@ -113,16 +124,17 @@ void Network::init_optimizer(char* name, ...)
       *layers[length-2-i].v = (beta2 * *layers[length-2-i].v) + (1-beta2)*(deltas[i].cwiseProduct(deltas[i]));
     };
   }
-  if (strcmp(name, "adamax") == 0) {
+  else if (strcmp(name, "adamax") == 0) {
     float beta1 = va_arg(args, double);
     float beta2 = va_arg(args, double);
     float epsilon = va_arg(args, double);
     va_end(args);
-    // TODO: Add bias correction for m (requires figuring out measuring t)
+    // TODO Add bias correction for m (requires figuring out measuring t)
     update = [this, beta1, beta2, epsilon](std::vector<Eigen::MatrixXf> deltas, int i) {
       *layers[length-2-i].weights -= *layers[length-2-i].m * learning_rate * layers[length-2-i].v->array().pow(-1).matrix();
       *layers[length-2-i].m = (beta1 * *layers[length-2-i].m) + ((1-beta1)*deltas[i]);
-      if ((beta2 * *layers[length-2-i].v) > deltas[i].array().abs().matrix()) *layers[length-2-i].v = (beta2 * *layers[length-2-i].v);
+      // FIXME Use of .sum() here is incredibly questionable. Do this correctly.
+      if ((beta2 * *layers[length-2-i].v).sum() > deltas[i].array().abs().sum()) *layers[length-2-i].v = (beta2 * *layers[length-2-i].v);
       else *layers[length-2-i].v = deltas[i].array().abs().matrix();
     };
   }