jacobian

commit b97d29c5f353ec5879bde79adf23bde370a4bf39
parent b07e463e3f022e29f28f867c794bcd2b39dd64bd
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sun, 12 Jul 2020 11:47:13 -0700

Linear output seems to annihilate softmax (and network)

Diffstat:
M
example.cpp
 | 
9
+++++----
M
src/bpnn.cpp
 | 
20
++++++++++++--------

2 files changed, 17 insertions(+), 12 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -9,13 +9,14 @@ double bench(int batch_sz)
   Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 0, 0.9);
   net.add_layer(4, "linear");
   net.add_layer(5, "relu");
-  net.add_layer(2, "linear");
-  net.init_decay("exp", 1, 10);
+  net.add_layer(2, "sigmoid");
+  net.init_decay("step", 1, 2);
   net.initialize();
-  //  checks(net);
-  for (int i = 0; i < 100; i++) {
+  // checks(net);
+  for (int i = 0; i < 5; i++) {
     net.next_batch();
     net.feedforward();
+    net.list_net();
     net.backpropagate();
     std::cout << net.cost() << " " << net.accuracy() << "\n";
   }
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -162,12 +162,14 @@ void Network::feedforward()
       }
     }
   }
+  std::cout << (*layers[length-1].contents) << "\n";
   for (int i = 0; i < layers[length-1].contents->rows(); i++) {
     float sum = 0;
     for (int j = 0; j < layers[length-1].contents->cols(); j++) {
       sum += exp((*layers[length-1].contents)(i,j));
     }
     for (int j = 0; j < layers[length-1].contents->cols(); j++) {
+      std::cout << "(e^" << (*layers[length-1].contents)(i,j) << ")/" << sum << " -> " << exp((*layers[length-1].contents)(i,j)) << "/" << sum << " -> " << exp((*layers[length-1].contents)(i,j))/sum << "\n";
       (*layers[length-1].contents)(i,j) = exp((*layers[length-1].contents)(i,j))/sum;
     }
   }
@@ -227,12 +229,14 @@ void Network::backpropagate()
   std::vector<Eigen::MatrixXf> gradients;
   std::vector<Eigen::MatrixXf> deltas;
   Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
+  std::cout << (*layers[length-1].contents) << "\n\n\n";
   for (int i = 0; i < error.rows(); i++) {
     for (int j = 0; j < error.cols(); j++) {
       float truth;
       if (j==(*labels)(i,0)) truth = 1;
       else truth = 0;
       error(i,j) = truth - (*layers[length-1].contents)(i,j);
+      std::cout << truth << "[as label is "<< (*labels)(i,0) <<"] - " << (*layers[length-1].contents)(i,j) << "[aka index " << i << " " << j << "] = " << error(i,j) << "\n";
     }
   }
   //  std::cout << error << "\n\n";
@@ -246,14 +250,14 @@ void Network::backpropagate()
     deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
     counter++;
   }
-  // std::cout << "-------\nGRADS INCOMING" << "\n\n";
-  // for (Eigen::MatrixXf i : gradients) {
-  //   std::cout << i << "\n\n";
-  // }
-  // std::cout << "-------\nDELTAS INCOMING" << "\n\n";
-  // for (Eigen::MatrixXf i : deltas) {
-  //   std::cout << i << "\n\n";
-  // }
+  std::cout << "-------\nGRADS INCOMING" << "\n\n";
+  for (Eigen::MatrixXf i : gradients) {
+    std::cout << i << "\n\n";
+  }
+  std::cout << "-------\nDELTAS INCOMING" << "\n\n";
+  for (Eigen::MatrixXf i : deltas) {
+    std::cout << i << "\n\n";
+  }
   for (int i = 0; i < length-1; i++) {
     *layers[length-2-i].weights -= (learning_rate * deltas[i]) + ((lambda/batch_size) * (*layers[length-2-i].weights));
     //*layers[length-2-i].v = (0.9 * *layers[length-2-i].v) - ((learning_rate * deltas[i]));