jacobian

commit e8ba5c59b0ecf8653c1663dc415447525bbe1231
parent 6ec82c19469932fd5edd96eab98888708331ed70
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Fri, 10 Jul 2020 15:31:32 -0700

Broken and bad numerical gradient

Diffstat:
M
bpnn.cpp
 | 
48
++++++++++++++++++++++++++++++++++++------------
M
example.cpp
 | 
3
+--

2 files changed, 37 insertions(+), 14 deletions(-)
diff --git a/bpnn.cpp b/bpnn.cpp
@@ -293,18 +293,42 @@ float Network::test(char* path)
   return 0;
 }
 
-// void Network::begin()
-// {
-//   Network sim = *this;
-//   for (int i = 0; i < sim.layers.size()-1; i++) {
-//     //    sim.layers[i].weights
-//     for (int j = 0; i < sim.layers[i].weights.rows(); i++) {
-//       for (int k = 0; i < sim.layers[i].weights.cols(); i++) {
-//       }
-//     }
-//   }
-//   //printf("Beginning train on %i instances for %i epochs...\n", instances, total_epochs);
-// }
+void Network::begin()
+{
+  float epsilon = 0.0001;
+  Network copy = *this;
+  std::vector<Eigen::MatrixXf> approx_gradients;
+  for (int i = 0; i < copy.layers.size()-1; i++) {
+    Eigen::MatrixXf current_approx = *copy.layers[i].weights;
+    for (int j = 0; i < copy.layers[i].weights->rows(); i++) {
+      for (int k = 0; i < copy.layers[i].weights->cols(); i++) {
+        Network sim1 = copy;
+        (*sim1.layers[i].contents)(j,k) += epsilon;
+        sim1.feedforward();
+        Network sim2 = copy;
+        (*sim2.layers[i].contents)(j,k) -= epsilon;
+        sim2.feedforward();
+        current_approx(j,k) = (sim1.cost() - sim2.cost())/(2*epsilon);
+      }
+    }
+    approx_gradients.push_back(current_approx);
+  }
+  for (Eigen::MatrixXf i : approx_gradients) {
+    std::cout << i << "\n\n";
+  }
+  // std::vector<Eigen::MatrixXf> gradients;
+  // std::vector<Eigen::MatrixXf> deltas;
+  // Eigen::MatrixXf error = ((*layers[length-1].contents) - (*labels));
+  // gradients.push_back(error.cwiseProduct(*layers[length-1].dZ));
+  // deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
+  // int counter = 1;
+  // for (int i = length-2; i >= 1; i--) {
+  //   gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
+  //   deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
+  //   counter++;
+  // }
+  //printf("Beginning train on %i instances for %i epochs...\n", instances, total_epochs);
+}
 
 void Network::train()
 {
diff --git a/example.cpp b/example.cpp
@@ -11,12 +11,11 @@ double bench(int batch_sz)
   net.add_layer(5, "relu");
   net.add_layer(1, "resig");
   net.initialize();
-  //  net.begin();
   for (int i = 0; i < 50; i++) {
     net.train();
   }
   auto end = std::chrono::high_resolution_clock::now();
-  net.list_net();
+  //;  net.list_net();
   return std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count() / pow(10,9);
 }