jacobian

commit b6906f8e411660ac32986172027f2c5a603ebd38
parent d2a826849f5ed95c55b39e5f3367fd4eaffaa1c6
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sat,  1 Aug 2020 19:31:48 -0700

Update decays (they're broken now)

Diffstat:
M
example.cpp
 | 
1
+
M
src/bpnn.cpp
 | 
35
+++++++++++++++++++++++++----------
M
src/bpnn.hpp
 | 
4
++--
M
src/optimizers.cpp
 | 
4
+---

4 files changed, 29 insertions(+), 15 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -19,6 +19,7 @@ double bench(int batch_sz)
   net.add_layer(5, "relu");
   net.add_layer(2, "linear");
   net.init_optimizer("demon", 0.9, 50);
+  net.init_decay("step", 1, 2);
   net.initialize();
   //net.grad_check();
   std::vector<float> vals;
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -62,31 +62,46 @@ Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, in
   instances = total_instances - test_instances;
   data = fopen(TRAIN_PATH, "r");
   test_data = fopen(TEST_PATH, "r");
-  decay = [](float lr, float t) -> float {
-    return lr;
+  decay = [this]() -> void {
+    learning_rate = learning_rate;
   };
   update = [this](std::vector<Eigen::MatrixXf> deltas, int i) {
     *layers[length-2-i].weights -= (learning_rate * deltas[i]);
   };
 }
 
-void Network::init_decay(char* type, float a_0, float k)
+void Network::init_decay(char* type, ...)
 {
+  va_list args;
+  va_start(args, type);
   if (strcmp(type, "step") == 0) {
-    decay = [a_0, k](float lr, float t) -> float {
-      return lr/k;
+    float a_0 = va_arg(args, double);
+    float k = va_arg(args, double);
+    decay = [this, a_0, k]() -> void {
+      learning_rate = a_0 * learning_rate/k;
     };
   }
   if (strcmp(type, "exp") == 0) {
-    decay = [a_0, k](float lr, float t) -> float {
-      return a_0 * exp(-k*t);
+    float a_0 = va_arg(args, double);
+    float k = va_arg(args, double);
+    decay = [this, a_0, k]() -> void {
+      learning_rate = a_0 * exp(-k * epochs);
     };
   }
   if (strcmp(type, "frac") == 0) {
-    decay = [a_0, k](float lr, float t) -> float {
-      return a_0/(1+(k*t));
+    float a_0 = va_arg(args, double);
+    float k = va_arg(args, double);
+    decay = [this, a_0, k]() -> void {
+      learning_rate = a_0 / (1+(k * epochs));
     };
   }
+  if (strcmp(type, "linear") == 0) {
+    int max_ep = va_arg(args, double);
+    decay = [this, max_ep]() -> void {
+      learning_rate = 1 - epochs/max_ep;
+    };
+  }
+  va_end(args);
 }
 
 #include "optimizers.cpp"
@@ -535,7 +550,7 @@ void Network::train()
   printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n", epochs, epoch_cost, epoch_acc, val_cost, val_acc);
   batches=1;
   rewind(data);
-  learning_rate = decay(learning_rate, epochs);
+  decay();
   epochs++;
 }
 
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -61,13 +61,13 @@ public:
   int batches = 0;
   Eigen::MatrixXf* labels;
 
-  std::function<float(float, float)> decay;
+  std::function<void(void)> decay;
   std::function<void(std::vector<Eigen::MatrixXf>, int)> update;
 
   Network(char* path, int batch_sz, float learn_rate, float bias_rate, int regularization, float l, float ratio);
   void add_layer(int nodes, char* activation);
   void add_prelu_layer(int nodes, float a);
-  void init_decay(char* type, float a_0, float k);
+  void init_decay(char* type, ...);
   void init_optimizer(char* name, ...);
   void initialize();
   void grad_check();
diff --git a/src/optimizers.cpp b/src/optimizers.cpp
@@ -22,7 +22,6 @@ void Network::init_optimizer(char* name, ...)
     float max_ep = va_arg(args, int);
     float beta = beta_init;
     int prev_epoch = -1;
-    va_end(args);
     update = [this, max_ep, prev_epoch, beta_init, beta](std::vector<Eigen::MatrixXf> deltas, int i) mutable {
       if (epochs > prev_epoch) {
         beta = beta_init * (1-(epochs/max_ep)) / ((beta_init * (1-(epochs/max_ep))) + (1-beta_init));
@@ -36,7 +35,6 @@ void Network::init_optimizer(char* name, ...)
     float beta1 = va_arg(args, double);
     float beta2 = va_arg(args, double);
     float epsilon = va_arg(args, double);
-    va_end(args);
     // TODO: Add bias correction (requires figuring out measuring t)
     // TODO: cwiseProduct here is sketchy, look into me
     update = [this, beta1, beta2, epsilon](std::vector<Eigen::MatrixXf> deltas, int i) {
@@ -49,7 +47,6 @@ void Network::init_optimizer(char* name, ...)
     float beta1 = va_arg(args, double);
     float beta2 = va_arg(args, double);
     float epsilon = va_arg(args, double);
-    va_end(args);
     // TODO: Add bias correction for m (requires figuring out measuring t)
     update = [this, beta1, beta2, epsilon](std::vector<Eigen::MatrixXf> deltas, int i) {
       *layers[length-2-i].m = (beta1 * *layers[length-2-i].m) + ((1-beta1)*deltas[i]);
@@ -59,4 +56,5 @@ void Network::init_optimizer(char* name, ...)
       *layers[length-2-i].weights -= learning_rate * (layers[length-2-i].v->array().pow(-1).cwiseProduct(layers[length-2-i].m->array())).matrix();
     };
   }
+  va_end(args);
 }