jacobian

commit 0a1ea1545936fcfe1516550bb10a161ce8ac19cc
parent eeccc528d0ba582a703025f1452ce938669e77ff
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sat, 11 Jul 2020 11:47:07 -0700

Attempts to fix checks result in more bugs

Diffstat:
M
example.cpp
 | 
4
++--
M
src/bpnn.cpp
 | 
5
+----
M
src/bpnn.hpp
 | 
2
+-
M
src/checks.cpp
 | 
18
++++++++++--------

4 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -10,9 +10,9 @@ double bench(int batch_sz)
   net.add_layer(4, "linear");
   net.add_layer(5, "relu");
   net.add_layer(1, "resig");
+  net.init_decay("step", 0, 2);
   net.initialize();
-  //net.list_net();
-  //  net.checks();
+  checks(net);
   for (int i = 0; i < 50; i++) {
     net.train();
   }
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -73,7 +73,7 @@ void Network::init_decay(char* type, float a_0, float k)
   }
   if (strcmp(type, "frac") == 0) {
     decay = [a_0, k](float lr, float t) -> float {
-      return a_0/(1-k*t);
+      return a_0/(1+(k*t));
     };
   }
 }
@@ -346,9 +346,7 @@ void Network::train()
   printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n", epochs, epoch_cost, epoch_acc, val_cost, val_acc);
   batches=1;
   rewind(data);
-  std::cout << learning_rate << "\nTHEN\n";
   learning_rate = decay(learning_rate, epochs);
-  std::cout << learning_rate << "\n\n\n";
   epochs++;
 }
 
@@ -356,4 +354,3 @@ float Network::get_acc() {return epoch_acc;}
 float Network::get_val_acc() {return val_acc;}
 float Network::get_cost() {return epoch_cost;}
 float Network::get_val_cost() {return val_cost;}
-
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -72,7 +72,6 @@ public:
   int next_batch();
   float test(char* path);
   void train();
-  void checks();
 
   float get_acc();
   float get_cost();
@@ -80,6 +79,7 @@ public:
   float get_val_cost();
 };
 
+void checks(Network net);
 void demo(int total_epochs);
 int prep_file(char* path, char* out_path);
 int split_file(char* path, int lines, float ratio);
diff --git a/src/checks.cpp b/src/checks.cpp
@@ -1,4 +1,4 @@
-void Network::checks()
+void checks(Network net)
 {
   int sanity_passed = 0;
   std::cout << "\u001b[4m\u001b[1mSANITY CHECKS:\u001b[0m\n";
@@ -7,8 +7,8 @@ void Network::checks()
 
   //  list_net();
   
-  Network copy1 = *this;
-  Network copy2 = *this;
+  Network copy1 = net;
+  Network copy2 = net;
   copy1.lambda += 1;
   copy1.next_batch();
   copy1.feedforward();
@@ -24,7 +24,7 @@ void Network::checks()
   
   // Check if zero cost is achievable on a batch
   std::cout << "Zero-cost sanity check...";
-  copy1 = *this;
+  copy1 = net;
   copy1.lambda = 0;
   copy1.next_batch();
   float finalcost;
@@ -45,7 +45,7 @@ void Network::checks()
   //  list_net();
   
   std::cout << "Gradient floating-point sanity check...";
-  copy1 = *this;
+  copy1 = net;
   copy1.next_batch();
   copy1.feedforward();
   std::vector<Eigen::MatrixXf> gradients;
@@ -79,7 +79,7 @@ void Network::checks()
   //  list_net();
   
   std::cout << "Expected loss sanity check...";
-  copy1 = *this;
+  copy1 = net;
   copy1.next_batch();
   copy1.feedforward();
   if (copy1.cost() <= 1) {
@@ -91,8 +91,8 @@ void Network::checks()
   //  list_net();
   
   std::cout << "Layer updates sanity check...";
-  copy1 = *this;
-  copy2 = *this;
+  copy1 = net;
+  copy2 = net;
   copy1.next_batch();
   copy1.feedforward();
   copy2.next_batch();
@@ -123,6 +123,8 @@ void Network::checks()
   else std::cout << " \u001b[31mFailed.\n\u001b[37m";
   
   std::cout << "\u001b[1m\nPassed " << sanity_passed << "/5" <<" sanity checks.\u001b[0m\n\n\n";
+
+  net.list_net();
   
   // float epsilon = 0.0001;
   // Network copy = *this;