jacobian

commit ddadd852615a13047d6788ce992afac2b2af5c68
parent 6e347b335b4ffd2c958d9dc2f611daf36fc8b97d
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sun,  6 Sep 2020 21:54:54 -0700

Split softmax into its own function + more RNN setup

Diffstat:
M
src/bpnn.cpp
 | 
45
+++++++++++++++++++++++++--------------------
M
src/bpnn.hpp
 | 
1
+
M
src/rnn.cpp
 | 
31
++++++++++++++++++++++++++++---

3 files changed, 54 insertions(+), 23 deletions(-)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -159,26 +159,8 @@ void Network::set_activation(int index, std::function<float(float)> custom, std:
     layers[index].activation_deriv = custom_deriv;
 }
 
-void Network::feedforward()
+void Network::softmax()
 {
-    for (int i = 0; i < length-1; i++) {
-        for (int j = 0; j < layers[i].contents->rows(); j++) {
-            if (strcmp(layers[i].activation_str, "linear") == 0) break;
-            for (int k = 0; k < layers[i].contents->cols(); k++) {
-                (*layers[i].dZ)(j,k) = layers[i].activation_deriv((*layers[i].contents)(j,k));
-                (*layers[i].contents)(j,k) = layers[i].activation((*layers[i].contents)(j,k));
-            }
-        }
-        *layers[i+1].contents = (*layers[i].contents) * (*layers[i].weights);
-        *layers[i+1].contents += *layers[i+1].bias;
-    }
-    for (int j = 0; j < layers[length-1].contents->rows(); j++) {
-        if (strcmp(layers[length-1].activation_str, "linear") == 0) break;
-        for (int k = 0; k < layers[length-1].contents->cols(); k++) {
-            (*layers[length-1].dZ)(j,k) = layers[length-1].activation_deriv((*layers[length-1].contents)(j,k));
-            (*layers[length-1].contents)(j,k) = layers[length-1].activation((*layers[length-1].contents)(j,k));
-        }
-    }
     for (int i = 0; i < layers[length-1].contents->rows(); i++) {
         Eigen::MatrixXf m = layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols());
         Eigen::MatrixXf::Index maxRow, maxCol;
@@ -198,8 +180,31 @@ void Network::feedforward()
             checknan(m(0,j), "output of Softmax operation");
         }
 #endif
-        layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols()) = m;
     }
+    layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols()) = m;
+}
+
+void Network::feedforward()
+{
+    for (int i = 0; i < length-1; i++) {
+        for (int j = 0; j < layers[i].contents->rows(); j++) {
+            if (strcmp(layers[i].activation_str, "linear") == 0) break;
+            for (int k = 0; k < layers[i].contents->cols(); k++) {
+                (*layers[i].dZ)(j,k) = layers[i].activation_deriv((*layers[i].contents)(j,k));
+                (*layers[i].contents)(j,k) = layers[i].activation((*layers[i].contents)(j,k));
+            }
+        }
+        *layers[i+1].contents = (*layers[i].contents) * (*layers[i].weights);
+        *layers[i+1].contents += *layers[i+1].bias;
+    }
+    for (int j = 0; j < layers[length-1].contents->rows(); j++) {
+        if (strcmp(layers[length-1].activation_str, "linear") == 0) break;
+        for (int k = 0; k < layers[length-1].contents->cols(); k++) {
+            (*layers[length-1].dZ)(j,k) = layers[length-1].activation_deriv((*layers[length-1].contents)(j,k));
+            (*layers[length-1].contents)(j,k) = layers[length-1].activation((*layers[length-1].contents)(j,k));
+        }
+    }
+    softmax();
 }
 
 void Network::list_net()
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -81,6 +81,7 @@ public:
     void set_activation(int index, std::function<float(float)> custom, std::function<float(float)> custom_deriv);
   
     void feedforward();
+    void softmax();
     void list_net();
 
     bool early_stop;
diff --git a/src/rnn.cpp b/src/rnn.cpp
@@ -2,6 +2,7 @@
 
 class RecurrentLayer : public Layer {
 public:
+    Eigen::MatrixXf* s;
     Eigen::MatrixXf* rec_weights;
     void init_weights(RecurrentLayer next);
     RecurrentLayer(int rows, int columns, float a=0);
@@ -10,12 +11,16 @@ public:
 RecurrentLayer::RecurrentLayer(int rows, int columns, float a)
     :Layer(rows, columns, a)
 {
+    s = new Eigen::MatrixXf(contents->rows(), contents->cols());
     rec_weights = new Eigen::MatrixXf(contents->cols(), contents->cols());
-    for (int i = 0; i < (rec_weights->rows()*rec_weights->cols()); i++) {
+    for (int i = 0; i < (rec_weights->cols()*rec_weights->cols()); i++) {
         std::random_device rd;
-        std::mt19937 gen(rd()); 
+        std::mt19937 gen(rd());
         (*rec_weights)(static_cast<int>(i / columns), i%columns) = d(gen);
     }
+    for (int i = 0; i < rows*columns; i++) {
+        (*s)(static_cast<int>(i / nodes),i%nodes) = 0;
+    }
 }
 
 void init_weights(RecurrentLayer next)
@@ -48,4 +53,24 @@ RNN::RNN(char* path, int batch_sz, float learn_rate, float bias_rate, Regulariza
     :Network(path, batch_sz, learn_rate, bias_rate, regularization, l, ratio, early_exit, cutoff)
 {}
 
-
+void RNN::feedforward()
+{
+    for (int i = 0; i < length-1; i++) {
+        for (int j = 0; j < layers[i].contents->rows(); j++) {
+            if (strcmp(layers[i].activation_str, "linear") == 0) break;
+            for (int k = 0; k < layers[i].contents->cols(); k++) {
+                (*layers[i].dZ)(j,k) = layers[i].activation_deriv((*layers[i].contents)(j,k));
+                (*layers[i].contents)(j,k) = layers[i].activation((*layers[i].contents)(j,k));
+            }
+        }
+        *layers[i+1].contents = ((*layers[i].s) * (*layers[i].rec_weights)) + ((*layers[i].contents) * (*layers[i].weights));
+        *layers[i+1].contents += *layers[i+1].bias;
+    }
+    for (int j = 0; j < layers[length-1].contents->rows(); j++) {
+        if (strcmp(layers[length-1].activation_str, "linear") == 0) break;
+        for (int k = 0; k < layers[length-1].contents->cols(); k++) {
+            (*layers[length-1].dZ)(j,k) = layers[length-1].activation_deriv((*layers[length-1].contents)(j,k));
+            (*layers[length-1].contents)(j,k) = layers[length-1].activation((*layers[length-1].contents)(j,k));
+        }
+    }
+}