jacobian

commit 328c5d37aefdac5f6d0b724d32283ae0e4420175
parent 861f8eaab3805620755b46e00587b57838968e51
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sat,  1 Aug 2020 16:32:48 -0700

Working on gradient check

Diffstat:
M
example.cpp
 | 
1
+
M
src/bpnn.cpp
 | 
40
++++++++++++++++++++++++++++++++++++++++
M
src/bpnn.hpp
 | 
7
++++---

3 files changed, 45 insertions(+), 3 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -20,6 +20,7 @@ double bench(int batch_sz)
   net.add_layer(2, "linear");
   net.init_optimizer("demon", 0.9, 50);
   net.initialize();
+  net.grad_check();
   std::vector<float> vals;
   for (int i = 0; i < 50; i++) {
     net.train();
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -344,6 +344,46 @@ Eigen::MatrixXf l1_deriv(Eigen::MatrixXf m)
   return r;
 }
 
+void Network::numerical_grad(int i, float epsilon)
+{
+  Eigen::MatrixXf gradient (layers[i].weights->rows(), layers[i].weights->cols());
+  for (int i = 0; i < layers[i].weights->rows(); i++) {
+    for (int j = 0; j < layers[i].weights->cols(); j++) {
+      float current_cost = cost();
+      std::vector<Layer> backup = layers;
+      (*layers[i].contents)(i,j) += epsilon;
+      feedforward();
+      float end_cost = cost();
+      gradient(i,j) = end_cost / current_cost;
+      layers = backup;
+      batches = 0;
+    }
+  }
+}
+
+void Network::grad_check()                      \
+{
+  std::vector<Eigen::MatrixXf> gradients;
+  std::vector<Eigen::MatrixXf> deltas;
+  Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
+  for (int i = 0; i < error.rows(); i++) {
+    for (int j = 0; j < error.cols(); j++) {
+      float truth;
+      if (j==(*labels)(i,0)) truth = 1;
+      else truth = 0;
+      error(i,j) = (*layers[length-1].contents)(i,j) - truth;
+      checknan(error(i,j), "gradient of final layer");
+    }
+  }
+  gradients.push_back(error);
+  for (int i = length-2; i >= 1; i--) {
+    gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
+    deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
+    counter++;
+  }
+  std::cout << deltas[1] << "\n\n" << numerical_grad(1, 0.00001);
+}
+
 void Network::backpropagate()
 {
   std::vector<Eigen::MatrixXf> gradients;
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -36,12 +36,13 @@ public:
 };
 
 class Network {
-public:
   FILE* data;
   FILE* test_data;
   int instances;
   int test_instances;
-
+  void numerical_grad(int i, float epsilon);
+  void update_layer(float* vals, int datalen, int index);
+public:
   std::vector<Layer> layers;
   int length = 0;
 
@@ -69,7 +70,7 @@ public:
   void init_decay(char* type, float a_0, float k);
   void init_optimizer(char* name, ...);
   void initialize();
-  void update_layer(float* vals, int datalen, int index);
+  void grad_check();
   void set_activation(int index, std::function<float(float)> custom, std::function<float(float)> custom_deriv);
   
   void feedforward();