jacobian

commit 9ec0da046648786d63e1454eda49cd32c7a37e3b
parent dd3201505150cd9e846965468b8adba1a2b079c8
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sun, 29 Nov 2020 10:38:38 -0800

Modularized backpropagation to a degree

Diffstat:
M
src/bpnn.cpp
 | 
28
+++-------------------------
M
src/bpnn.hpp
 | 
2
+-
M
src/cnn.cpp
 | 
31
++++++-------------------------

3 files changed, 10 insertions(+), 51 deletions(-)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -274,7 +274,7 @@ Eigen::MatrixXf l1_deriv(Eigen::MatrixXf m)
     return r;
 }
 
-void Network::backpropagate()
+Eigen::MatrixXf Network::backpropagate()
 {
     std::vector<Eigen::MatrixXf> gradients;
     std::vector<Eigen::MatrixXf> deltas;
@@ -303,31 +303,9 @@ void Network::backpropagate()
         update(deltas, i);
         if (reg_type == L2) *layers[length-2-i].weights -= ((lambda/batch_size) * (*layers[length-2-i].weights));
         else if (reg_type == L1) *layers[length-2-i].weights -= ((lambda/(2*batch_size)) * l1_deriv(*layers[length-2-i].weights));
-        *layers[length-1-i].bias -= bias_lr * gradients[i];
-        if (strcmp(layers[length-2-i].activation_str, "prelu") == 0) {
-            float sum = 0;
-            for (int j = 0; j < layers[length-2-i].contents->rows(); j++) {
-                for (int k = 0; k < layers[length-2-i].contents->cols(); k++) {
-                    if ((*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha <= 0) {
-                        //  TODO: Review questionable code | -t quality -m Choice of using index i+1 here is sketchy.
-                        sum += gradients[i+1](j,k) * (*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha;
-                    }
-                }
-            }
-            layers[length-2-i].alpha += learning_rate * sum;
-            float a = layers[length-2-i].alpha;
-            layers[length-2-i].activation = [a](float x) -> float
-            {
-                if (x > 0) return x;
-                else return a * x;
-            };
-            layers[length-2-i].activation_deriv = [a](float x) -> float
-            {
-                if (x > 0) return 1;
-                else return a;
-            };
-        }
+        *layers[length-1-i].bias -= bias_lr * gradients[i];        
     }
+    return gradients.back();
 }
 
 #include "data.cpp"
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -89,7 +89,7 @@ public:
     void list_net();
     float cost();
     float accuracy();
-    void backpropagate();
+    Eigen::MatrixXf backpropagate();
     int next_batch(int fd);
     float validate(char* path);
     void train();
diff --git a/src/cnn.cpp b/src/cnn.cpp
@@ -245,32 +245,11 @@ void ConvNet::list_net()
 
 void ConvNet::backpropagate()
 {
+    list_net();
+    char a;
+    std::cin >> a;
     std::vector<Eigen::MatrixXf> gradients;
-    std::vector<Eigen::MatrixXf> deltas;
-    Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
-    for (int i = 0; i < error.rows(); i++) {
-        for (int j = 0; j < error.cols(); j++) {
-            float truth;
-            if (j==(*labels)(i,0)) truth = 1;
-            else truth = 0;
-            error(i,j) = (*layers[length-1].contents)(i,j) - truth;
-            checknan(error(i,j), "gradient of final layer");
-        }
-    }
-    gradients.push_back(error);
-    deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
-    int counter = 1;
-    for (int i = length-2; i >= 1; i--) {
-        gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
-        deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
-        counter++;
-    }
-    for (int i = 0; i < length-1; i++) {
-        update(deltas, i);
-        if (reg_type == 2) *layers[length-2-i].weights -= ((lambda/batch_size) * (*layers[length-2-i].weights));
-        else if (reg_type == 1) *layers[length-2-i].weights -= ((lambda/(2*batch_size)) * l1_deriv(*layers[length-2-i].weights));
-        *layers[length-1-i].bias -= bias_lr * gradients[i];
-    }
+    gradients.push_back(Network::backpropagate());    
     Eigen::Map<Eigen::MatrixXf> reshaped(gradients[gradients.size()-1].data(),
                            conv_layers.back().output->rows(),
                            conv_layers.back().output->cols());
@@ -300,6 +279,8 @@ void ConvNet::backpropagate()
         }
         gradients.push_back(final_grad.cwiseProduct(*conv_layers[layer].dZ));
     }
+    list_net();
+    assert(2<1);
 }
 
 void ConvNet::train()