jacobian

commit a933982a1e0c88e5adc55251552ec1ffeb012c7d
parent a3a0497ed5158e6b2830b8f1e3cc95cfbf2b30a6
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Thu, 25 Jun 2020 18:35:23 -0700

Rectifier works, custom activations essentially allowed

Diffstat:
M
bpnn.cpp
 | 
24
++++++++++++------------
M
bpnn.hpp
 | 
6
+++---
M
example.cpp
 | 
10
++++++++++
M
utils.cpp
 | 
25
+++++++------------------
M
utils.hpp
 | 
7
+++----

5 files changed, 35 insertions(+), 37 deletions(-)
diff --git a/bpnn.cpp b/bpnn.cpp
@@ -63,20 +63,20 @@ void Network::initialize()
 void Network::set_activation(int index, char* name)
 {
   if (strcmp(name, "sigmoid") == 0) {
-    layers[index].activation = &sigmoid;
-    layers[index].activation_deriv = &sigmoid_deriv;
+    layers[index].activation = sigmoid;
+    layers[index].activation_deriv = sigmoid_deriv;
   }
   else if (strcmp(name, "linear") == 0) {
-    layers[index].activation = &linear;
-    layers[index].activation_deriv = &linear_deriv;
+    layers[index].activation = linear;
+    layers[index].activation_deriv = linear_deriv;
   }
   else if (strcmp(name, "relu") == 0) {
-    layers[index].activation = &(rectifier(linear));
-    layers[index].activation_deriv = &(rectifier(linear_deriv));
+    layers[index].activation = rectifier(linear);
+    layers[index].activation_deriv = rectifier(linear_deriv);
   }
   else if (strcmp(name, "resig") == 0) {
-    layers[index].activation = &resig;
-    layers[index].activation_deriv = &resig_deriv;
+    layers[index].activation = rectifier(sigmoid);
+    layers[index].activation_deriv = rectifier(sigmoid_deriv);
   }
   else {
     std::cout << "Warning! Incorrect activation specified. Exiting...\n";
@@ -88,8 +88,8 @@ void Network::feedforward()
 {
   for (int j = 0; j < layers[0].contents->rows(); j++) {
     for (int k = 0; k < layers[0].contents->cols(); k++) {
-      (*layers[0].dZ)(j,k) = (*layers[0].activation_deriv)((*layers[0].contents)(j,k));
-      (*layers[0].contents)(j,k) = (*layers[0].activation)((*layers[0].contents)(j,k));
+      (*layers[0].dZ)(j,k) = layers[0].activation_deriv((*layers[0].contents)(j,k));
+      (*layers[0].contents)(j,k) = layers[0].activation((*layers[0].contents)(j,k));
     }
   }
   for (int i = 0; i < length-1; i++) {
@@ -99,8 +99,8 @@ void Network::feedforward()
   for (int i = 1; i < length; i++) {
     for (int j = 0; j < layers[i].contents->rows(); j++) {
       for (int k = 0; k < layers[i].contents->cols(); k++) {
-        (*layers[i].dZ)(j,k) = (*layers[i].activation_deriv)((*layers[i].contents)(j,k));
-        (*layers[i].contents)(j,k) = (*layers[i].activation)((*layers[i].contents)(j,k));
+        (*layers[i].dZ)(j,k) = layers[i].activation_deriv((*layers[i].contents)(j,k));
+        (*layers[i].contents)(j,k) = layers[i].activation((*layers[i].contents)(j,k));
       }
     }
   }
diff --git a/bpnn.hpp b/bpnn.hpp
@@ -22,9 +22,9 @@ public:
   Eigen::MatrixXd* weights;
   Eigen::MatrixXd* bias;
   Eigen::MatrixXd* dZ;
-  double (*activation)(double);
-  double (*activation_deriv)(double);
-
+  std::function<double(double)> activation;
+  std::function<double(double)> activation_deriv;
+  
   Layer(int rows, int columns);
   Layer(float* vals, int rows, int columns);
   void init_weights(Layer next);
diff --git a/example.cpp b/example.cpp
@@ -1,6 +1,16 @@
 #include "bpnn.hpp"
 #include "utils.hpp"
 
+double lecun_tanh(double x)
+{
+  return 1.7159 * tanh((2.0/3) * x);
+}
+
+double lecun_tanh_deriv(double x)
+{
+  return 1.14393 * pow(sech(2.0/3 * x),2);
+}
+
 int main()
 {
   Network net ("./data_banknote_authentication.txt", 10, 0.01, 0.001);
diff --git a/utils.cpp b/utils.cpp
@@ -4,49 +4,38 @@
 #include <ctime>
 #include <cmath>
 #include <cstdio>
-#include <functional>
 #include <fcntl.h>
 #include <unistd.h>
 #include <sys/stat.h>
 
-inline double sigmoid(double x)
+double sigmoid(double x)
 {
   return 1.0/(1+exp(-x));
 }
 
-inline double sigmoid_deriv(double x)
+double sigmoid_deriv(double x)
 {
   return 1.0/(1+exp(-x)) * (1 - 1.0/(1+exp(x)));
 }
 
-inline double linear(double x)
+double linear(double x)
 {
   return x;
 }
 
-inline double linear_deriv(double x)
+double linear_deriv(double x)
 {
   return 1;
 }
 
-std::function<double(double)> rectifier(double (*activation)(double), x)
+std::function<double(double)> rectifier(double (*activation)(double))
 {
-  auto rectified = [](double x) -> double
+  auto rectified = [activation](double x) -> double
   { 
     if (x > 0) return (*activation)(x);
     else return 0; 
   };
-  return rectified
-}
-
-double rectifier_deriv(double (*activation_deriv)(double), x)
-{
-  auto rectified = [](double x) -> double
-  { 
-    if (x > 0) return (*activation_deriv)(x);
-    else return 0; 
-  };
-  return rectified
+  return rectified;
 }
 
 static uintmax_t wc(char const *fname)
diff --git a/utils.hpp b/utils.hpp
@@ -1,13 +1,12 @@
 #ifndef UTILS_H
 #define UTILS_H
 
+#include <functional>
+
 double sigmoid(double x);
 double sigmoid_deriv(double x);
-double resig(double x);
-double resig_deriv(double x);
-double relu(double x);
-double relu_deriv(double x);
 double linear(double x);
 double linear_deriv(double x);
+std::function<double(double)> rectifier(double (*activation)(double));
 
 #endif /* MODULE_H */