jacobian

commit 3b19cbf9256b7c0c609abaf8a7de6ebdc710bc70
parent ff548d86aade4b7003e28ebfda13e7b24c9bab5a
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sat, 11 Jul 2020 10:23:59 -0700

Experiments with regularization + momentum

Diffstat:
M
src/bpnn.cpp
 | 
11
++++++++---
M
src/bpnn.hpp
 | 
1
+

2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -26,12 +26,12 @@ Layer::Layer(int batch_sz, int nodes)
       (*bias)(j, i) = 0;
     }
   }
-  dZ = new Eigen::MatrixXf (batch_sz, nodes);
 }
 
 void Layer::init_weights(Layer next)
 {
   weights = new Eigen::MatrixXf (contents->cols(), next.contents->cols());
+  v = new Eigen::MatrixXf (contents->cols(), next.contents->cols());
   int nodes = weights->cols();
   int n = contents->cols() + next.contents->cols();
   std::normal_distribution<float> d(0,sqrt(1.0/n));
@@ -40,6 +40,9 @@ void Layer::init_weights(Layer next)
     std::mt19937 gen(rd()); 
     (*weights)((int)i / nodes, i%nodes) = d(gen);
   }
+  for (int i = 0; i < (weights->rows()*weights->cols()); i++) {
+    (*v)((int)i / nodes, i%nodes) = 0;
+  }
 }
 
 Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, float l, float ratio)
@@ -163,7 +166,7 @@ float Network::cost()
     //    std::cout << *layers[i].weights << "\n\n" << (layers[i].weights->cwiseProduct(*layers[i].weights)).sum() << "\n\n\n";
     reg += (layers[i].weights->cwiseProduct(*layers[i].weights)).sum();
   }
-  return ((1.0/batch_size) * sum) + (lambda*reg);
+  return ((1.0/batch_size) * sum) + (1/2*lambda*reg);
 }
 
 float Network::accuracy()
@@ -189,7 +192,9 @@ void Network::backpropagate()
     counter++;
   }
   for (int i = 0; i < length-1; i++) {
-    *layers[length-2-i].weights -= (learning_rate * deltas[i]) + ((lambda/batch_size) * (*layers[length-2-i].weights * 2));
+    //    *layers[length-2-i].weights -= (learning_rate * deltas[i]) + ((lambda/batch_size) * (*layers[length-2-i].weights));
+    *layers[length-2-i].v = (0.9 * *layers[length-2-i].v) - ((learning_rate * deltas[i]));
+    *layers[length-2-i].weights += *layers[length-2-i].v;
     *layers[length-1-i].bias -= bias_lr * gradients[i];
   }
 }
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -18,6 +18,7 @@ class Layer {
 public:
   Eigen::MatrixXf* contents;
   Eigen::MatrixXf* weights;
+  Eigen::MatrixXf* v;
   Eigen::MatrixXf* bias;
   Eigen::MatrixXf* dZ;
   std::vector<Eigen::MatrixXf> prev_updates;