jacobian

commit 04e7ca9bdc0c176e2a42682fb3feb339450eaa3e
parent c3bc72f8b711f017c2651d5db3b7da789f089f92
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Wed, 29 Jul 2020 17:28:27 -0700

More activations

Diffstat:
M
src/utils.cpp
 | 
23
+++++++++++++++++++++++
M
src/utils.hpp
 | 
6
++++++

2 files changed, 29 insertions(+), 0 deletions(-)
diff --git a/src/utils.cpp b/src/utils.cpp
@@ -50,10 +50,33 @@ float step_deriv(float x) {return 0;}
 float bipolar(float x)
 {
   if (x > 0) return 1;
+  else if (x == 0) return 0;
   else return -1;
 }
 float bipolar_deriv(float x) {return 0;}
 
+float bipolar_sigmoid(float x) {return (1-exp(-x))/(1+exp(-x))}
+float bipolar_sigmoid_deriv(float x) {return (2*exp(x))/(pow(exp(x)+1,2));}
+
+float hard_tanh(float x) {return max(-1, min(1,x))}
+float hard_tanh_deriv(float x)
+{
+  if (-1 < x && x < 1) return 1;
+  else return 0;
+}
+
+float leaky_relu(float x)
+{
+  if (x > 0) return x;
+  else return 0.01 * x;
+}
+
+float leaky_relu_deriv(float x)
+{
+  if (x > 0) return 1;
+  else return 0.01;
+}
+
 std::function<float(float)> rectifier(float (*activation)(float))
 {
   auto rectified = [activation](float x) -> float
diff --git a/src/utils.hpp b/src/utils.hpp
@@ -30,6 +30,12 @@ float softplus(float x);
 float softplus_deriv(float x);
 float inverse_logit(float x);
 float inverse_logit_deriv(float x);
+float hard_tanh(float x);
+float hard_tanh_deriv(float x);
+float bipolar_sigmoid(float x);
+float bipolar_sigmoid_deriv(float x);
+float leaky_relu(float x);
+float leaky_relu_deriv(float x);
 std::function<float(float)> rectifier(float (*activation)(float));
 
 Eigen::MatrixXf strassen_mul(Eigen::MatrixXf a, Eigen::MatrixXf b);