jacobian

commit 1c7b79033155be149f8c65442f7c8f6f2683991a
parent 45fb12c364f6d0a63b5925424696c3c099f94931
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sat, 11 Jul 2020 17:08:20 -0700

Working on backprop fixes

Diffstat:
M
example.cpp
 | 
2
+-
M
src/bpnn.cpp
 | 
28
++++++++++++++++++++--------

2 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -9,7 +9,7 @@ double bench(int batch_sz)
   Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 0, 0.9);
   net.add_layer(4, "linear");
   net.add_layer(5, "relu");
-  net.add_layer(2, "resig");
+  net.add_layer(2, "linear");
   net.init_decay("step", 0, 2);
   net.initialize();
   //  checks(net);
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -203,17 +203,29 @@ void Network::backpropagate()
   std::vector<Eigen::MatrixXf> deltas;
   Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
   for (int i = 0; i < layers[length-1].contents->rows(); i++) {
-    float loss_i = 0;
-    for (int j = 0; j < layers[length-1].contents->rows(); j++) {
-      if (j == i) continue;
-      float classloss = (*layers[length-1].contents)(i,j) - (*labels)(i,0) + DELTA;
-      if (classloss > 0) loss_i += classloss;
-      else loss_i+=0;
+    for (int j = 0; j < layers[length-1].contents->cols(); j++) {
+      if (j == (*labels)(i,0)) {
+        float sum = 0;
+        for (int k = 0; k < layers[length-1].contents->cols(); k++) {
+          if (k == j) continue;
+          float intermediate = (*layers[length-1].contents)(i,j) - (*labels)(i,0);
+          if (intermediate > 0) sum+=1;
+        }
+        if (sum == 0) error(i,j) = 0; // IEEE floats are weird
+        else error(i,j) = -sum;
+      }
+      else {
+        float classloss = (*layers[length-1].contents)(i,j) - (*labels)(i,0) + DELTA;
+        if (classloss > 0) error(i, j) = 1;
+        else error(i, j) = 0;
+      }
     }
-    error(i, 0) = loss_i;
   }
-  gradients.push_back(error.cwiseProduct(*layers[length-1].dZ));
+  std::cout << error << "\n\n";
+  gradients.push_back(error);
   deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
+  std::cout << deltas[0] << "\n\n";
+  //  gradients[523] += error;
   int counter = 1;
   for (int i = length-2; i >= 1; i--) {
     gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));