jacobian

commit d8a3779bb8af0e1ee7fabde24e176716a8c2203a
parent b97d29c5f353ec5879bde79adf23bde370a4bf39
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sun, 12 Jul 2020 12:22:52 -0700

Training no longer errors/nans but is very broken

Diffstat:
M
example.cpp
 | 
23
++++++++++++-----------
M
src/bpnn.cpp
 | 
36
++++++++++++++++++++++--------------

2 files changed, 34 insertions(+), 25 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -6,23 +6,24 @@
 double bench(int batch_sz)
 {
   auto start = std::chrono::high_resolution_clock::now();
-  Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 0, 0.9);
+  Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 10, 0.9);
   net.add_layer(4, "linear");
   net.add_layer(5, "relu");
-  net.add_layer(2, "sigmoid");
+  net.add_layer(2, "linear");
   net.init_decay("step", 1, 2);
   net.initialize();
   // checks(net);
-  for (int i = 0; i < 5; i++) {
-    net.next_batch();
-    net.feedforward();
-    net.list_net();
-    net.backpropagate();
-    std::cout << net.cost() << " " << net.accuracy() << "\n";
-  }
-  // for (int i = 0; i < 1; i++) {
-  //   net.train();
+  // for (int i = 0; i < 10; i++) {
+  //   net.next_batch();
+  //   net.feedforward();
+  //   net.list_net();
+  //   net.backpropagate();
+  //   std::cout << net.cost() << " " << net.accuracy() << "\n";
   // }
+  for (int i = 0; i < 50; i++) {
+    net.train();
+    //    net.list_net();
+  }
   auto end = std::chrono::high_resolution_clock::now();
   //net.list_net();
   return std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count() / pow(10,9);
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -162,16 +162,23 @@ void Network::feedforward()
       }
     }
   }
-  std::cout << (*layers[length-1].contents) << "\n";
+  //std::cout << (*layers[length-1].contents) << "\n";
   for (int i = 0; i < layers[length-1].contents->rows(); i++) {
     float sum = 0;
+    Eigen::MatrixXf m = layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols());
+    Eigen::MatrixXf::Index maxRow, maxCol;
+    float max = m.maxCoeff(&maxRow, &maxCol);
+    //    std::cout << m << "(before with max "<< max <<")\n";
+    m = (m.array() - max).matrix();
+    //    std::cout << m << "(after with max "<< max <<")\n";
     for (int j = 0; j < layers[length-1].contents->cols(); j++) {
-      sum += exp((*layers[length-1].contents)(i,j));
+      sum += exp(m(0,j));
     }
     for (int j = 0; j < layers[length-1].contents->cols(); j++) {
-      std::cout << "(e^" << (*layers[length-1].contents)(i,j) << ")/" << sum << " -> " << exp((*layers[length-1].contents)(i,j)) << "/" << sum << " -> " << exp((*layers[length-1].contents)(i,j))/sum << "\n";
-      (*layers[length-1].contents)(i,j) = exp((*layers[length-1].contents)(i,j))/sum;
+      //      std::cout << "(e^" << m(0,j) << ")/" << sum << " -> " << exp(m(0,j)) << "/" << sum << " -> " << exp((m(0,j)))/sum << "\n";
+      m(0,j) = exp(m(0,j))/sum;
     }
+    layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols()) = m;
   }
 }
 
@@ -194,6 +201,7 @@ float Network::cost()
       float truth;
       if (j==(*labels)(i,0)) truth = 1;
       else truth = 0;
+      if ((*layers[length-1].contents)(i,j) == 0) (*layers[length-1].contents)(i,j) += 0.00001;
       //      std::cout << truth << " VS " << (*layers[length-1].contents)(i,j) << " SO " << truth * log((*layers[length-1].contents)(i,j)) << "\n";
       tempsum += truth * log((*layers[length-1].contents)(i,j));
     }
@@ -229,14 +237,14 @@ void Network::backpropagate()
   std::vector<Eigen::MatrixXf> gradients;
   std::vector<Eigen::MatrixXf> deltas;
   Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
-  std::cout << (*layers[length-1].contents) << "\n\n\n";
+  // std::cout << (*layers[length-1].contents) << "\n\n\n";
   for (int i = 0; i < error.rows(); i++) {
     for (int j = 0; j < error.cols(); j++) {
       float truth;
       if (j==(*labels)(i,0)) truth = 1;
       else truth = 0;
       error(i,j) = truth - (*layers[length-1].contents)(i,j);
-      std::cout << truth << "[as label is "<< (*labels)(i,0) <<"] - " << (*layers[length-1].contents)(i,j) << "[aka index " << i << " " << j << "] = " << error(i,j) << "\n";
+      // std::cout << truth << "[as label is "<< (*labels)(i,0) <<"] - " << (*layers[length-1].contents)(i,j) << "[aka index " << i << " " << j << "] = " << error(i,j) << "\n";
     }
   }
   //  std::cout << error << "\n\n";
@@ -250,14 +258,14 @@ void Network::backpropagate()
     deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
     counter++;
   }
-  std::cout << "-------\nGRADS INCOMING" << "\n\n";
-  for (Eigen::MatrixXf i : gradients) {
-    std::cout << i << "\n\n";
-  }
-  std::cout << "-------\nDELTAS INCOMING" << "\n\n";
-  for (Eigen::MatrixXf i : deltas) {
-    std::cout << i << "\n\n";
-  }
+  // std::cout << "-------\nGRADS INCOMING" << "\n\n";
+  // for (Eigen::MatrixXf i : gradients) {
+  //   std::cout << i << "\n\n";
+  // }
+  // std::cout << "-------\nDELTAS INCOMING" << "\n\n";
+  // for (Eigen::MatrixXf i : deltas) {
+  //   std::cout << i << "\n\n";
+  // }
   for (int i = 0; i < length-1; i++) {
     *layers[length-2-i].weights -= (learning_rate * deltas[i]) + ((lambda/batch_size) * (*layers[length-2-i].weights));
     //*layers[length-2-i].v = (0.9 * *layers[length-2-i].v) - ((learning_rate * deltas[i]));