jacobian

commit 201523bdea24e44cf3ab75c0062abf99c7c35bd7
parent 17ba8677ae8a570aebca581d2847978a2ba23ab8
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Mon, 15 Jun 2020 11:54:50 -0700

Fixing backprop: problems remain

Diffstat:
M
test.cpp
 | 
65
+++++++++++++++++++++++++++++++++++++++++------------------------

1 file changed, 41 insertions(+), 24 deletions(-)
diff --git a/test.cpp b/test.cpp
@@ -185,34 +185,45 @@ void Network::backpropagate()
 
   std::vector<Eigen::MatrixXd> gradients;
   std::vector<Eigen::MatrixXd> errors;
-  Eigen::MatrixXd e = ((*layers[length-1].contents ) - (*labels))*100 * ((*layers[length-1].contents ) - (*labels))*100;
+  Eigen::MatrixXd e = ((*layers[length-1].contents ) - (*labels)) * ((*layers[length-1].contents ) - (*labels));
   Eigen::MatrixXd D (layers[length-1].contents->cols(), layers[length-1].contents->cols());
   for (int i = 0; i < layers[length-1].contents->cols(); i++) {
+    for (int j = 0; j < layers[length - 1].contents->rows(); j++) {
+      D(j, i) = 0;
+    }
+  }
+  for (int i = 0; i < layers[length-1].contents->cols(); i++) {
     D(i, i) = (*layers[length-1].contents)(0, i) * (1 - (*layers[length-1].contents)(0, i));
   }
   gradients.push_back(layers[length-2].contents->transpose() * (D * e));
+  // std::cout << gradients[0] << "\n\n";
   // std::cout << D << "\n\nTHEN\n\n" << layers[length-2].contents->transpose() << "\n\nNEXT\n\n" << e << "\n\nSO\n\n" << gradients[0] << "\n\n\n\n\n";
-  // int counter = 0;
-  // for (int i = length-2; i >= 0; i--) {
-  //   Eigen::MatrixXd D_l (layers[i].contents->cols(), layers[i].contents->cols());
-  //   for (int j = 0; i < layers[i].contents->cols(); j++) {
-  //     // std::cout << *layers[i].contents << "\n\nAKA\n\n" << (*layers[i].contents)(0,j) << "\n\nTIMES\n\n" << (1 - (*layers[i].contents)(0,j)) << "FOR " << j <<"\n\n\n\n\n";
-  //     // std::cout << j << "\n\n";
-  //     if (j >= 5) {
-  //       break;
-  //     }
-  //     D_l(j, j) = (*layers[i].contents)(0,j) * (1 - (*layers[i].contents)(0,j));
-  //   }
-  //   std::cout << D_l << "\n\nTHEN\n\n" << layers[i].weights->transpose() << "\n\nNEXT\n\n" << gradients[counter] << "\n\n\n\n\n\n";
-
-  //   Eigen::MatrixXd e_l = D_l * (layers[i].weights->transpose() * gradients[counter]);
-  //   std::cout << "\n\nSO\n\n" << e_l <<  "\n\n\n\n\n";
-  //   gradients.push_back(e_l);
-  //   counter++;
-  // }
+  int counter = 0;
+  for (int i = length-2; i >= 0; i--) {
+    Eigen::MatrixXd D_l (layers[i].contents->cols(), layers[i].contents->cols());
+    std::cout << i << " Aye!\n";
+    for (int i = 0; i < layers[i].contents->cols(); i++) {
+      for (int j = 0; j < layers[i].contents->rows(); j++) {
+        D_l(j, i) = 0;
+      }
+    }
+    for (int j = 0; j < layers[i].contents->cols(); j++) {
+      std::cout << j << "\n\n";
+
+      D_l(j, j) = (*layers[i].contents)(0,j) * (1 - (*layers[i].contents)(0,j));
+    }
+    std::cout << D_l << "\n\nTHEN\n\n" << layers[i].weights->transpose() << "\n\nNEXT\n\n" << gradients[counter] << "\n\n";
+
+    Eigen::MatrixXd e_l = D_l * ( gradients[counter] * layers[i].weights->transpose());
+    std::cout << "\n\nSO\n\n" << e_l <<  "\n\n\n\n\n";
+    gradients.push_back(e_l);
+    counter++;
+  }
   for (int i = 1; i < gradients.size(); i++) {
     Eigen::MatrixXd gradient = gradients[i];
-    *layers[length-1].weights -= learning_rate * 1.0/N * gradient;
+    printf("%i\n", length-1-i);
+    std::cout << *layers[length-1-i].weights << " \n\n and \n\n " << gradients[i] << "\n";
+    *layers[length-1-i].weights -= (gradients[i]);
   }
 }
 
@@ -305,20 +316,20 @@ void Network::test(char* path)
   std::cout << "TEST COST: " << 1.0/((float) linecount) * totalcost << "\n";
 }
 
-int main()
+void demo()
 {
   // std::cout << "\n\n\n";
   int linecount = prep_file("./data_banknote_authentication.txt");
-  Network net ("./shuffled.txt", 4, 2, 1, 5, 1, 0.02);
+  Network net ("./shuffled.txt", 4, 2, 1, 5, 1, 1);
   float epoch_cost = 1000;
   int epochs = 0;
   net.batches= 1;
-  while (epochs < 10) {
+  while (epochs < 1) {
     // int linecount = prep_file("./data_banknote_authentication.txt");
     float cost_sum = 0;
     for (int i = 0; i < linecount; i++) {
       net.feedforward();
-      // net.backpropagate();
+      net.backpropagate();
       cost_sum += net.cost();
       // std::cout << net.cost() << " as it is " << net.labels[0] << " vs " << *net.layers[net.length-1].contents << "\n";
       // net.list_net();
@@ -331,8 +342,14 @@ int main()
     net.batches=1;
     epoch_cost = 1.0/((float) linecount) * cost_sum;
     printf("EPOCH %i: Cost is %f for %i instances.\n", epochs, epoch_cost, linecount);
+    std::cout << *net.layers[net.length-2].weights << "\n\n";
     epochs++;
   }
   net.test("./test.txt");
   net.feedforward();
 }
+
+int main()
+{
+  demo();
+}