jacobian

commit b9f5ac760eae0eeb1b311f45d8e18dc26f6b7d67
parent 23bfc8c623e178121b92e404cf1292e87d53d3e7
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Wed, 29 Jul 2020 18:27:27 -0700

Beginnings of PReLU

Diffstat:
M
src/bpnn.cpp
 | 
20
+++++++++++++++++++-
M
src/bpnn.hpp
 | 
5
+++--

2 files changed, 22 insertions(+), 3 deletions(-)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -18,7 +18,8 @@
 
 #include "checks.cpp"
 
-Layer::Layer(int batch_sz, int nodes)
+Layer::Layer(int batch_sz, int nodes, int a=0)
+  :alpha(a)
 {
   contents = new Eigen::MatrixXf (batch_sz, nodes);
   dZ = new Eigen::MatrixXf (batch_sz, nodes);
@@ -84,6 +85,23 @@ void Network::init_decay(char* type, float a_0, float k)
   }
 }
 
+void Network::add_prelu_layer(int nodes, float a)
+{
+  length++;
+  layers.emplace_back(batch_size, nodes, a);
+  strcpy(layers[length-1].activation_str, name);
+  layers[length-1].activation = [a](float x) -> float
+  {
+    if (x > 0) return x;
+    else return a * x;
+  };
+  layers[length-1].activation_deriv = [a](float x) -> float
+  {
+    if (x > 0) return 1;
+    else return a;
+  };
+}
+
 // Gross code inbound
 void Network::add_layer(int nodes, char* name)
 {
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -18,16 +18,17 @@ class Layer {
 public:
   Eigen::MatrixXf* contents;
   Eigen::MatrixXf* weights;
-  Eigen::MatrixXf* v;
   Eigen::MatrixXf* bias;
   Eigen::MatrixXf* dZ;
   std::vector<Eigen::MatrixXf> prev_updates;
   std::function<float(float)> activation;
   std::function<float(float)> activation_deriv;
   char activation_str[1024];
+  // PReLU layers shouldn't be Layers but inherit from them! Fix me!!
+  float alpha;
   
   Layer(int rows, int columns);
-  Layer(float* vals, int rows, int columns);
+  Layer(float* vals, int rows, int columns, int a=0);
   void init_weights(Layer next);
 };