jacobian

commit 6613b279c2baa0a8aa7fcfc3a399a080960953e3
parent b89d6ff25dc72dc9b1092e9ada64d2688da10953
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Mon,  3 Aug 2020 17:10:23 -0700

Add nesterov momentum

Diffstat:
M
example.cpp
 | 
2
+-
M
src/bpnn.cpp
 | 
8
++++++--
M
src/bpnn.hpp
 | 
1
+
M
src/optimizers.cpp
 | 
12
++++++++++++

4 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -17,6 +17,7 @@ double bench(int batch_sz)
   net.add_layer(4, "linear");
   net.add_layer(5, "relu");
   net.add_layer(2, "linear");
+  net.init_optimizer("nesterov", 0.9);
   net.initialize();
   for (int i = 0; i < 50; i++) {
     net.train();
@@ -27,6 +28,5 @@ double bench(int batch_sz)
 
 int main()
 {
-  sleep(40);
   bench(16);
 }
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -65,6 +65,9 @@ Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, in
   decay = [this]() -> void {
     learning_rate = learning_rate;
   };
+  grad_calc = [this](std::vector<Eigen::MatrixXf> gradients, int i, int counter) -> void {
+    gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
+  };
   update = [this](std::vector<Eigen::MatrixXf> deltas, int i) {
     *layers[length-2-i].weights -= (learning_rate * deltas[i]);
   };
@@ -377,7 +380,8 @@ void Network::backpropagate()
   int counter = 1;
   for (int i = length-2; i >= 1; i--) {
     // TODO: Find nice way to add this
-    // *layers[i].weights-((learning_rate * *layers[i].weights) + (0.9 * *layers[i].v))).transpose()
+    // (*layers[i].weights-((learning_rate * *layers[i].weights) + (0.9 * *layers[i].v))).transpose()
+    grad_calc(gradients, counter, i);
     gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
     deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
     counter++;
@@ -546,7 +550,7 @@ void Network::train()
   epoch_acc = 1.0/((float) instances/batch_size) * acc_sum;
   epoch_cost = 1.0/((float) instances/batch_size) * cost_sum;
   validate(VAL_PATH);
-  //  printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n", epochs, epoch_cost, epoch_acc, val_cost, val_acc);
+  printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n", epochs, epoch_cost, epoch_acc, val_cost, val_acc);
   batches=1;
   rewind(data);
   decay();
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -65,6 +65,7 @@ public:
   Eigen::MatrixXf* labels;
 
   std::function<void(void)> decay;
+  std::function<void(std::vector<Eigen::MatrixXf>, int, int)> grad_calc;
   std::function<void(std::vector<Eigen::MatrixXf>, int)> update;
 
   Network(char* path, int batch_sz, float learn_rate, float bias_rate, int regularization, float l, float ratio);
diff --git a/src/optimizers.cpp b/src/optimizers.cpp
@@ -17,6 +17,18 @@ void Network::init_optimizer(char* name, ...)
       *layers[length-2-i].m = (learning_rate * deltas[i]);
     };
   }
+  // TODO: split into functions to remove reundant code
+  if (strcmp(name, "nesterov") == 0) {
+    float beta = va_arg(args, double);
+    va_end(args);
+    grad_calc = [this](std::vector<Eigen::MatrixXf> gradients, int i, int counter) -> void {
+      gradients.push_back((gradients[counter-1] * (*layers[i].weights-((learning_rate * *layers[i].weights) + (0.9 * *layers[i].v))).transpose()).cwiseProduct(*layers[i].dZ));
+    };
+    update = [this, beta](std::vector<Eigen::MatrixXf> deltas, int i) {
+      *layers[length-2-i].weights -= (beta * *layers[length-2-i].m) + (learning_rate * deltas[i]);
+      *layers[length-2-i].m = (learning_rate * deltas[i]);
+    };
+  }
   else if (strcmp(name, "demon") == 0) {
     float beta_init = va_arg(args, double);
     float max_ep = va_arg(args, int);