jacobian

commit f7a00efc9a1b55d5c2c46082eba1f35003ba57ac
parent 1685a5fb0b1d12d9a27d5a506c2a5d587a89d23d
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Wed, 29 Jul 2020 19:19:39 -0700

PReLU now compiles

Diffstat:
M
src/bpnn.cpp
 | 
32
+++++++++++++++++++-------------
M
src/bpnn.hpp
 | 
6
++++--
M
src/utils.cpp
 | 
4
++--

3 files changed, 25 insertions(+), 17 deletions(-)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -18,7 +18,7 @@
 
 #include "checks.cpp"
 
-Layer::Layer(int batch_sz, int nodes, int a=0)
+Layer::Layer(int batch_sz, int nodes, float a)
   :alpha(a)
 {
   contents = new Eigen::MatrixXf (batch_sz, nodes);
@@ -39,7 +39,6 @@ Layer::Layer(int batch_sz, int nodes, int a=0)
 void Layer::init_weights(Layer next)
 {
   weights = new Eigen::MatrixXf (contents->cols(), next.contents->cols());
-  v = new Eigen::MatrixXf (contents->cols(), next.contents->cols());
   int nodes = weights->cols();
   int n = contents->cols() + next.contents->cols();
   std::normal_distribution<float> d(0,sqrt(1.0/n));
@@ -48,9 +47,6 @@ void Layer::init_weights(Layer next)
     std::mt19937 gen(rd()); 
     (*weights)((int)i / nodes, i%nodes) = d(gen);
   }
-  for (int i = 0; i < (weights->rows()*weights->cols()); i++) {
-    (*v)((int)i / nodes, i%nodes) = 0;
-  }
 }
 
 Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, float l, float ratio)
@@ -89,7 +85,7 @@ void Network::add_prelu_layer(int nodes, float a)
 {
   length++;
   layers.emplace_back(batch_size, nodes, a);
-  strcpy(layers[length-1].activation_str, name);
+  strcpy(layers[length-1].activation_str, "prelu");
   layers[length-1].activation = [a](float x) -> float
   {
     if (x > 0) return x;
@@ -149,12 +145,12 @@ void Network::add_layer(int nodes, char* name)
     layers[length-1].activation_deriv = bipolar_sigmoid_deriv;
   }
   else if (strcmp(name, "tanh") == 0) {
-    layers[length-1].activation = tanh();
-    layers[length-1].activation_deriv = [](float x) -> float {1.0/cosh(x)};
+    layers[length-1].activation = [](float x) -> float {return tanh(x);};
+    layers[length-1].activation_deriv = [](float x) -> float {return 1.0/cosh(x);};
   }
   else if (strcmp(name, "hard_tanh") == 0) {
-    layers[length-1].activation = hard_tanh();
-    layers[length-1].activation_deriv = hard_tanh_deriv();
+    layers[length-1].activation = hard_tanh;
+    layers[length-1].activation_deriv = hard_tanh_deriv;
   }
   else if (strcmp(name, "resig") == 0) {
     layers[length-1].activation = rectifier(sigmoid);
@@ -304,13 +300,23 @@ void Network::backpropagate()
     *layers[length-1-i].bias -= bias_lr * gradients[i];
     if (strcmp(layers[length-2-i].activation_str, "prelu") == 0) {
       float sum = 0;
-      for (int i = 0; i < layers[length-2-i].rows(); i++) {
-        for (int j = 0; j < layers[length-2-i].cols(); j++) {
+      for (int i = 0; i < layers[length-2-i].contents->rows(); i++) {
+        for (int j = 0; j < layers[length-2-i].contents->cols(); j++) {
           if ((*layers[length-2-i].contents)(i,j)/layers[length-2-i].alpha <= 0) sum += gradients[i](i,j) * (*layers[length-2-i].contents)(i,j)/layers[length-2-i].alpha;
         }
       }
       layers[length-2-i].alpha += learning_rate * sum;
-      layers[length-2-i].activation = [layers[length-2-i].alpha]()
+      float a = layers[length-2-i].alpha;
+      layers[length-2-i].activation = [a](float x) -> float
+      {
+        if (x > 0) return x;
+        else return a * x;
+      };
+      layers[length-2-i].activation_deriv = [a](float x) -> float
+      {
+        if (x > 0) return 1;
+        else return a;
+      };
     }
   }
 }
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -10,6 +10,7 @@
 #include <iostream>
 #include <string>
 #include <cstdio>
+#include <cmath>
 #include <fstream>
 #include <random>
 #include <algorithm>
@@ -27,8 +28,8 @@ public:
   // PReLU layers shouldn't be Layers but inherit from them! Fix me!!
   float alpha;
   
-  Layer(int rows, int columns);
-  Layer(float* vals, int rows, int columns, int a=0);
+  Layer(int rows, int columns, float a=0);
+  Layer(float* vals, int rows, int columns);
   void init_weights(Layer next);
 };
 
@@ -60,6 +61,7 @@ public:
 
   Network(char* path, int batch_sz, float learn_rate, float bias_rate, float l, float ratio);
   void add_layer(int nodes, char* activation);
+  void add_prelu_layer(int nodes, float a);
   void init_decay(char* type, float a_0, float k);
   void initialize();
   void update_layer(float* vals, int datalen, int index);
diff --git a/src/utils.cpp b/src/utils.cpp
@@ -55,10 +55,10 @@ float bipolar(float x)
 }
 float bipolar_deriv(float x) {return 0;}
 
-float bipolar_sigmoid(float x) {return (1-exp(-x))/(1+exp(-x))}
+float bipolar_sigmoid(float x) {return (1-exp(-x))/(1+exp(-x));}
 float bipolar_sigmoid_deriv(float x) {return (2*exp(x))/(pow(exp(x)+1,2));}
 
-float hard_tanh(float x) {return max(-1, min(1,x))}
+float hard_tanh(float x) {return fmax(-1, fmin(1,x));}
 float hard_tanh_deriv(float x)
 {
   if (-1 < x && x < 1) return 1;