jacobian

commit 1efffd4ab4ee1ad898852d0409bceeff28eb029e
parent 99e8e1dfc3da45d728e0bec83fc6212b3708d47c
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Mon, 18 Jan 2021 13:32:09 -0800

Initial optimizations of activation functions

Diffstat:
M
src/bpnn.hpp
 | 
2
+-
M
src/utils.cpp
 | 
44
++++++++++++++++++++++++++++++++------------

2 files changed, 33 insertions(+), 13 deletions(-)
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -16,7 +16,7 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <unistd.h>
-// #include <gsl/gsl_assert>
+#include <gsl/gsl_assert>
 #include <lz4.h>
 
 #define BUFFER_SIZE 600*1024
diff --git a/src/utils.cpp b/src/utils.cpp
@@ -18,27 +18,47 @@
 #include <sys/stat.h>
 #include <eigen3/Eigen/Dense>
 
+inline float sgn(float val) {
+    return (0.0f < val) - (val < 0.0f);
+}
+
+double fexp(double val)
+{  
+    long tmp = static_cast<long>(1512775 * val + 1072632447) << 32;
+    return *reinterpret_cast<double*>(&tmp);
+}
+
+float ftanh(float val)
+{
+    return sgn(val) * (1 - 2/(fexp(2*abs(val))+1));
+}
+
+float fcosh(float val)
+{
+    return (fexp(val) + fexp(-val)) * 0.5;
+}
+
 // A bunch of hardcoded activation functions. Avoids much of the slowness of custom functions.
 // Although the std::function makes it not the fastest way, the functionality is worth it.
 // Yes, these functions may be a frustrating to read but they're just equations and I want to conserve space.
 
-float sigmoid(float x) {return 1.0/(1+exp(-x));}
-float sigmoid_deriv(float x) {return 1.0/(1+exp(-x)) * (1 - 1.0/(1+exp(-x)));}
+float sigmoid(float x) {return 1.0/(1+fexp(-x));}
+float sigmoid_deriv(float x) {return 1.0/(1+fexp(-x)) * (1 - 1.0/(1+fexp(-x)));}
 
 float linear(float x) {return x;}
 float linear_deriv(float x) {return 1;}
 
-float lecun_tanh(float x) {return 1.7159 * tanh((2.0/3) * x);}
-float lecun_tanh_deriv(float x) {return 1.14393 * pow(1.0/cosh(2.0/3 * x),2);}
+float lecun_tanh(float x) {return 1.7159 * ftanh((2.0/3) * x);}
+float lecun_tanh_deriv(float x) {return 1.14393 * pow(1.0/fcosh(2.0/3 * x),2);}
 
-float inverse_logit(float x) {return (exp(x)/(exp(x)+1));}
-float inverse_logit_deriv(float x) {return (exp(x)/pow(exp(x)+1, 2));}
+float inverse_logit(float x) {return (fexp(x)/(fexp(x)+1));}
+float inverse_logit_deriv(float x) {return (fexp(x)/pow(fexp(x)+1, 2));}
 
-float softplus(float x) {return log(1+exp(x));}
-float softplus_deriv(float x) {return exp(x)/(exp(x)+1);}
+float softplus(float x) {return log(1+fexp(x));}
+float softplus_deriv(float x) {return fexp(x)/(fexp(x)+1);}
 
-float cloglog(float x) {return 1-exp(-exp(x));}
-float cloglog_deriv(float x) {return exp(x-exp(x));}
+float cloglog(float x) {return 1-fexp(-fexp(x));}
+float cloglog_deriv(float x) {return fexp(x-fexp(x));}
 
 float step(float x)
 {
@@ -55,8 +75,8 @@ float bipolar(float x)
 }
 float bipolar_deriv(float x) {return 0;}
 
-float bipolar_sigmoid(float x) {return (1-exp(-x))/(1+exp(-x));}
-float bipolar_sigmoid_deriv(float x) {return (2*exp(x))/(pow(exp(x)+1,2));}
+float bipolar_sigmoid(float x) {return (1-fexp(-x))/(1+fexp(-x));}
+float bipolar_sigmoid_deriv(float x) {return (2*fexp(x))/(pow(fexp(x)+1,2));}
 
 float hard_tanh(float x) {return fmax(-1, fmin(1,x));}
 float hard_tanh_deriv(float x)