jacobian

commit ec6fcee421f1dd8af832160737daee59b41216ea
parent ca08263f2ff0cba7e76a66ebe99305a6e3a114b8
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Fri, 10 Jul 2020 18:03:35 -0700

Working on regularization + sanity checks

Diffstat:
M
bpnn.cpp
 | 
66
++++++++++++++++++++++++++++++++++++++++++++----------------------
M
bpnn.hpp
 | 
2
+-
M
example.cpp
 | 
5
+++--

3 files changed, 48 insertions(+), 25 deletions(-)
diff --git a/bpnn.cpp b/bpnn.cpp
@@ -153,10 +153,14 @@ void Network::list_net()
 float Network::cost()
 {
   float sum = 0;
+  float reg = 0; // Regularization term
   for (int i = 0; i < layers[length-1].contents->rows(); i++) {
     sum += ((*labels)(i, 0) - (*layers[length-1].contents)(i, 0)) * ((*labels)(i, 0) - (*layers[length-1].contents)(i, 0));
   }
-  return (1.0/batch_size) * sum;
+  for (int i = 0; i < layers.size()-1; i++) {
+    reg += (layers[i].contents->cwiseProduct(*layers[i].contents)).sum();
+  }
+  return ((1.0/batch_size) * sum) + (lambda*reg);
 }
 
 float Network::accuracy()
@@ -293,29 +297,46 @@ float Network::test(char* path)
   return 0;
 }
 
-void Network::begin()
+void Network::checks()
 {
-  float epsilon = 0.0001;
-  Network copy = *this;
-  std::vector<Eigen::MatrixXf> approx_gradients;
-  for (int i = 0; i < copy.layers.size()-1; i++) {
-    Eigen::MatrixXf current_approx = *copy.layers[i].weights;
-    for (int j = 0; i < copy.layers[i].weights->rows(); i++) {
-      for (int k = 0; i < copy.layers[i].weights->cols(); i++) {
-        Network sim1 = copy;
-        (*sim1.layers[i].contents)(j,k) += epsilon;
-        sim1.feedforward();
-        Network sim2 = copy;
-        (*sim2.layers[i].contents)(j,k) -= epsilon;
-        sim2.feedforward();
-        current_approx(j,k) = (sim1.cost() - sim2.cost())/(2*epsilon);
-      }
-    }
-    approx_gradients.push_back(current_approx);
-  }
-  for (Eigen::MatrixXf i : approx_gradients) {
-    std::cout << i << "\n\n";
+  //fclose(data);
+  int sanity_passed = 0;
+  std::cout << "Beginning sanity checks.\n\n";
+  // Check if regularization strength increases loss (as it should).
+  std::cout << "Regularization sanity check...";
+  Network copy1 = *this;
+  Network copy2 = *this;
+  copy1.lambda += 1;
+  copy1.next_batch();
+  copy1.feedforward();
+  copy2.next_batch();
+  copy2.feedforward();
+  if (copy1.cost() > copy2.cost()) {
+    std::cout << " \u001b[32mSucceeded!\n\u001b[37m";
+    sanity_passed++;
   }
+  else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+  // float epsilon = 0.0001;
+  // Network copy = *this;
+  // std::vector<Eigen::MatrixXf> approx_gradients;
+  // for (int i = 0; i < copy.layers.size()-1; i++) {
+  //   Eigen::MatrixXf current_approx = *copy.layers[i].weights;
+  //   for (int j = 0; i < copy.layers[i].weights->rows(); i++) {
+  //     for (int k = 0; i < copy.layers[i].weights->cols(); i++) {
+  //       Network sim1 = copy;
+  //       (*sim1.layers[i].contents)(j,k) += epsilon;
+  //       sim1.feedforward();
+  //       Network sim2 = copy;
+  //       (*sim2.layers[i].contents)(j,k) -= epsilon;
+  //       sim2.feedforward();
+  //       current_approx(j,k) = (sim1.cost() - sim2.cost())/(2*epsilon);
+  //     }
+  //   }
+  //   approx_gradients.push_back(current_approx);
+  // }
+  // for (Eigen::MatrixXf i : approx_gradients) {
+  //   std::cout << i << "\n\n";
+  // }
   // std::vector<Eigen::MatrixXf> gradients;
   // std::vector<Eigen::MatrixXf> deltas;
   // Eigen::MatrixXf error = ((*layers[length-1].contents) - (*labels));
@@ -332,6 +353,7 @@ void Network::begin()
 
 void Network::train()
 {
+  rewind(data);
   float cost_sum = 0;
   float acc_sum = 0;
   for (int i = 0; i <= instances-batch_size; i+=batch_size) {
diff --git a/bpnn.hpp b/bpnn.hpp
@@ -67,7 +67,7 @@ public:
   int next_batch();
   float test(char* path);
   void train();
-  void begin();
+  void checks();
 
   float get_acc();
   float get_cost();
diff --git a/example.cpp b/example.cpp
@@ -6,12 +6,13 @@
 double bench(int batch_sz)
 {
   auto start = std::chrono::high_resolution_clock::now();
-  Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 0.0, 0.9);
+  Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 0.1, 0.9);
   net.add_layer(4, "linear");
   net.add_layer(5, "relu");
   net.add_layer(1, "resig");
   net.initialize();
-  for (int i = 0; i < 50; i++) {
+  net.checks();
+  for (int i = 0; i < 1; i++) {
     net.train();
   }
   auto end = std::chrono::high_resolution_clock::now();