jacobian

commit 4a0d6bfe1f77972297a700ea680e4bb64aa6d353
parent 787493f9d876e941eec624369e7ea61cd037160d
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Fri, 26 Jun 2020 20:08:36 -0700

Found activation bugs, fixed some + added new activation

Diffstat:
M
bpnn.cpp
 | 
8
++++----
M
utils.cpp
 | 
2
+-
M
utils.hpp
 | 
4
++++

3 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/bpnn.cpp b/bpnn.cpp
@@ -62,9 +62,9 @@ void Network::add_layer(int nodes, char* name)
     layers[length-1].activation = step;
     layers[length-1].activation_deriv = step_deriv;
   }
-  else if (strcmp(name, "bipolar") == 0) {
-    layers[length-1].activation = bipolar;
-    layers[length-1].activation_deriv = bipolar_deriv;
+  else if (strcmp(name, "tanh") == 0) {
+    layers[length-1].activation = mytanh;
+    layers[length-1].activation_deriv = tanh_deriv;
   }
   else if (strcmp(name, "lecun_tanh") == 0) {
     layers[length-1].activation = lecun_tanh;
@@ -185,7 +185,7 @@ void Network::backpropagate()
   }
   for (int i = 0; i < length-1; i++) {
     Eigen::MatrixXd gradient = gradients[i];
-    *layers[length-2-i].weights -= 0.5 * *layers[length-2-i].prev_update + learning_rate * deltas[i];
+    *layers[length-2-i].weights -= 0.9 * *layers[length-2-i].prev_update + learning_rate * deltas[i];
     *layers[length-1-i].bias -= bias_lr * gradients[i];
   }
 }
diff --git a/utils.cpp b/utils.cpp
@@ -20,7 +20,7 @@ double linear_deriv(double x) {return 1;}
 double lecun_tanh(double x) {return 1.7159 * tanh((2.0/3) * x);}
 double lecun_tanh_deriv(double x) {return 1.14393 * pow(1.0/cosh(2.0/3 * x),2);}
 
-double tanh(double x) {return tanh(x);}
+double mytanh(double x) {return tanh(x);}
 double tanh_deriv(double x) {return pow(1.0/cosh(x),2);}
 
 double inverse_logit(double x) {return (exp(x)/(exp(x)+1));}
diff --git a/utils.hpp b/utils.hpp
@@ -3,6 +3,7 @@
 
 #include <functional>
 
+// A zoo of activation functions.
 double sigmoid(double x);
 double sigmoid_deriv(double x);
 double linear(double x);
@@ -11,6 +12,8 @@ double step(double x);
 double step_deriv(double x);
 double bipolar(double x);
 double bipolar_deriv(double x);
+double mytanh(double x);
+double tanh_deriv(double x);
 double lecun_tanh(double x);
 double lecun_tanh_deriv(double x);
 double cloglog(double x);
@@ -21,5 +24,6 @@ double inverse_logit(double x);
 double inverse_logit_deriv(double x);
 std::function<double(double)> rectifier(double (*activation)(double));
 
+// Experimental fast file reading functions
 uintmax_t wc(char const *fname);
 #endif /* MODULE_H */