jacobian

commit 45fb12c364f6d0a63b5925424696c3c099f94931
parent 358af32eaab011b242c4912f5ed68df502b8ef12
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sat, 11 Jul 2020 15:37:57 -0700

Attempting switch to SVM loss

Diffstat:
M
example.cpp
 | 
4
++--
M
src/bpnn.cpp
 | 
25
++++++++++++++++++-------

2 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -9,10 +9,10 @@ double bench(int batch_sz)
   Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 0, 0.9);
   net.add_layer(4, "linear");
   net.add_layer(5, "relu");
-  net.add_layer(1, "resig");
+  net.add_layer(2, "resig");
   net.init_decay("step", 0, 2);
   net.initialize();
-  checks(net);
+  //  checks(net);
   for (int i = 0; i < 50; i++) {
     net.train();
   }
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -175,16 +175,16 @@ void Network::list_net()
 
 float Network::cost()
 {
-  float sum = 0;
+  // float sum = 0;
   float reg = 0; // Regularization term
-  for (int i = 0; i < layers[length-1].contents->rows(); i++) {
-    sum += ((*labels)(i, 0) - (*layers[length-1].contents)(i, 0)) * ((*labels)(i, 0) - (*layers[length-1].contents)(i, 0));
-  }
+  // for (int i = 0; i < layers[length-1].contents->rows(); i++) {
+  //   sum += ((*labels)(i, 0) - (*layers[length-1].contents)(i, 0)) * ((*labels)(i, 0) - (*layers[length-1].contents)(i, 0));
+  // }
   for (int i = 0; i < layers.size()-1; i++) {
-    //    std::cout << *layers[i].weights << "\n\n" << (layers[i].weights->cwiseProduct(*layers[i].weights)).sum() << "\n\n\n";
     reg += (layers[i].weights->cwiseProduct(*layers[i].weights)).sum();
   }
-  return ((1.0/batch_size) * sum) + (1/2*lambda*reg);
+  // return ((1.0/batch_size) * sum) + (1/2*lambda*reg);
+  return (1/2*lambda*reg);
 }
 
 float Network::accuracy()
@@ -196,11 +196,22 @@ float Network::accuracy()
   return (1.0/batch_size) * correct;
 }
 
+#define DELTA 10
 void Network::backpropagate()
 {
   std::vector<Eigen::MatrixXf> gradients;
   std::vector<Eigen::MatrixXf> deltas;
-  Eigen::MatrixXf error = ((*layers[length-1].contents) - (*labels));
+  Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
+  for (int i = 0; i < layers[length-1].contents->rows(); i++) {
+    float loss_i = 0;
+    for (int j = 0; j < layers[length-1].contents->rows(); j++) {
+      if (j == i) continue;
+      float classloss = (*layers[length-1].contents)(i,j) - (*labels)(i,0) + DELTA;
+      if (classloss > 0) loss_i += classloss;
+      else loss_i+=0;
+    }
+    error(i, 0) = loss_i;
+  }
   gradients.push_back(error.cwiseProduct(*layers[length-1].dZ));
   deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
   int counter = 1;