jacobian

commit 94a504a457a856654cc9edd8fc8a2b99604246cb
parent 54787847d8e9cf6fe1512ef6bb29478ce4b27f8b
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Fri, 31 Jul 2020 16:14:17 -0700

Testing momentum - weight updates exploding...

Diffstat:
M
src/bpnn.cpp
 | 
20
++++++++++++++------
M
src/bpnn.hpp
 | 
1
+

2 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -38,6 +38,7 @@ Layer::Layer(int batch_sz, int nodes, float a)
 
 void Layer::init_weights(Layer next)
 {
+  v = new Eigen::MatrixXf (contents->cols(), next.contents->cols());
   weights = new Eigen::MatrixXf (contents->cols(), next.contents->cols());
   int nodes = weights->cols();
   int n = contents->cols() + next.contents->cols();
@@ -46,6 +47,7 @@ void Layer::init_weights(Layer next)
     std::random_device rd;
     std::mt19937 gen(rd()); 
     (*weights)((int)i / nodes, i%nodes) = d(gen);
+    (*v)((int)i / nodes, i%nodes) = 0;
   }
 }
 
@@ -309,9 +311,15 @@ void Network::backpropagate()
     counter++;
   }
   for (int i = 0; i < length-1; i++) {
-    if (reg_type == 2) *layers[length-2-i].weights -= (learning_rate * deltas[i]) + ((lambda/batch_size) * (*layers[length-2-i].weights));
-    else if (reg_type == 1) *layers[length-2-i].weights -= (learning_rate * deltas[i]) + ((lambda/(2*batch_size)) * l1_deriv(*layers[length-2-i].weights));
+    *layers[length-2-i].weights -= (0.9 * *layers[length-2-i].v) + (learning_rate * deltas[i]);
+
+    if (reg_type == 2) *layers[length-2-i].weights -= ((lambda/batch_size) * (*layers[length-2-i].weights));
+    else if (reg_type == 1) *layers[length-2-i].weights -= ((lambda/(2*batch_size)) * l1_deriv(*layers[length-2-i].weights));
+
     *layers[length-1-i].bias -= bias_lr * gradients[i];
+    //std::cout << *layers[length-2-i].v << "\n\n" << *layers[length-2-i].weights << "\n\n" << deltas[i] << "\n\n\n\n";
+    *layers[length-2-i].v = deltas[i];
+    
     if (strcmp(layers[length-2-i].activation_str, "prelu") == 0) {
       float sum = 0;
       for (int j = 0; j < layers[length-2-i].contents->rows(); j++) {
@@ -457,10 +465,10 @@ void Network::train()
     cost_sum += cost();
     acc_sum += accuracy();
     batches++;
-    // if (i > batch_size * 10) {
-    //   list_net();
-    //   exit(1);
-    // }
+    if (i > batch_size * 10) {
+      list_net();
+      exit(1);
+    }
     //    layers[10000000].alpha = 2;
   }
   epoch_acc = 1.0/((float) instances/batch_size) * acc_sum;
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -18,6 +18,7 @@
 class Layer {
 public:
   Eigen::MatrixXf* contents;
+  Eigen::MatrixXf* v;
   Eigen::MatrixXf* weights;
   Eigen::MatrixXf* bias;
   Eigen::MatrixXf* dZ;