jacobian

commit a6b10cf43deeda18f31b71984bc892dec95d7cc9
parent 7871a4f8bbc5fc62674a2b6f1dacd3473d9c7e30
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Thu, 30 Jul 2020 14:50:27 -0700

Located PReLU bug

Diffstat:
M
example.cpp
 | 
2
+-
M
src/bpnn.cpp
 | 
17
++++++++++++-----

2 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -16,7 +16,7 @@ double bench(int batch_sz)
   auto start = std::chrono::high_resolution_clock::now();
   Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 0, 0.9);
   net.add_layer(4, "linear");
-  net.add_prelu_layer(5, 0.02);
+  net.add_prelu_layer(5, 0.01);
   net.add_layer(2, "linear");
   net.initialize();
   for (int i = 0; i < 50; i++) {
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -196,21 +196,21 @@ void Network::feedforward()
       (*layers[length-1].contents)(j,k) = layers[length-1].activation((*layers[length-1].contents)(j,k));
     }
   }
-  std::cout << "\nSOFTMAX INPUT\n" << *layers[length-1].contents << "\n\n";
+  //  std::cout << "\nSOFTMAX INPUT\n" << *layers[length-1].contents << "\n\n";
   for (int i = 0; i < layers[length-1].contents->rows(); i++) {
     float sum = 0;
     Eigen::MatrixXf m = layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols());
     Eigen::MatrixXf::Index maxRow, maxCol;
     float max = m.maxCoeff(&maxRow, &maxCol);
     m = (m.array() - max).matrix();
-    std::cout << "\nGETTING SUM\n";
+    // std::cout << "\nGETTING SUM\n";
     for (int j = 0; j < layers[length-1].contents->cols(); j++) {
       checknan(m(0,j), "input to final layer");
       sum += exp(m(0,j));
-      std::cout << "Adding " << exp(m(0,j)) << "(aka e^"<< m(0, j) << ")\n";
+      // std::cout << "Adding " << exp(m(0,j)) << "(aka e^"<< m(0, j) << ")\n";
       checknan(sum, "sum in Softmax operation");
     }
-    std::cout << "\nFINAL ACTIVATION\n";
+    //std::cout << "\nFINAL ACTIVATION\n";
     for (int j = 0; j < layers[length-1].contents->cols(); j++) {
       m(0,j) = exp(m(0,j))/sum;
       //      std::cout << "Calculating " << exp(m(0,j)) << "/" << sum << " to be " << (*layers[length-1].contents)(i,j) << "(aka " << test<<")\n";
@@ -302,11 +302,16 @@ void Network::backpropagate()
     *layers[length-1-i].bias -= bias_lr * gradients[i];
     if (strcmp(layers[length-2-i].activation_str, "prelu") == 0) {
       float sum = 0;
+      std::cout << "\n\nGRAD:\n\n" << gradients[i] << "\n\n\n";
       for (int j = 0; j < layers[length-2-i].contents->rows(); j++) {
         for (int k = 0; k < layers[length-2-i].contents->cols(); k++) {
-          if ((*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha <= 0) sum += gradients[i](j,k) * (*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha;
+          if ((*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha <= 0) {
+            sum += gradients[i](j,k) * (*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha;
+            std::cout << "SUM += " << gradients[i](j,k) << " * " << (*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha << "\n";
+          }
         }
       }
+      std::cout << "SUM: " << sum << "\n";
       layers[length-2-i].alpha += learning_rate * sum;
       float a = layers[length-2-i].alpha;
       layers[length-2-i].activation = [a](float x) -> float
@@ -442,10 +447,12 @@ void Network::train()
     cost_sum += cost();
     acc_sum += accuracy();
     batches++;
+    list_net();
     // if (i > batch_size * 10) {
     //   list_net();
     //   exit(1);
     // }
+    layers[10000000].alpha = 2;
   }
   epoch_acc = 1.0/((float) instances/batch_size) * acc_sum;
   epoch_cost = 1.0/((float) instances/batch_size) * cost_sum;