commit 96ab9f361a59f9226fd4e3e7cd5cd240f13cce37
parent ab211064394b589cf87f7762180739673cb9770d
Author: David Freifeld <freifeld.david@gmail.com>
Date: Thu, 25 Jun 2020 20:51:09 -0700
Added plenty of activation functions
Diffstat:
5 files changed, 49 insertions(+), 35 deletions(-)
diff --git a/bpnn.cpp b/bpnn.cpp
@@ -57,6 +57,22 @@ void Network::add_layer(int nodes, char* name)
layers[length-1].activation = linear;
layers[length-1].activation_deriv = linear_deriv;
}
+ if (strcmp(name, "step") == 0) {
+ layers[length-1].activation = step;
+ layers[length-1].activation_deriv = step_deriv;
+ }
+ if (strcmp(name, "bipolar") == 0) {
+ layers[length-1].activation = bipolar;
+ layers[length-1].activation_deriv = bipolar_deriv;
+ }
+ if (strcmp(name, "lecun_tanh") == 0) {
+ layers[length-1].activation = lecun_tanh;
+ layers[length-1].activation_deriv = lecun_tanh_deriv;
+ }
+ if (strcmp(name, "logit") == 0) {
+ layers[length-1].activation = logit;
+ layers[length-1].activation_deriv = logit_deriv;
+ }
else if (strcmp(name, "relu") == 0) {
layers[length-1].activation = rectifier(linear);
layers[length-1].activation_deriv = rectifier(linear_deriv);
@@ -300,7 +316,7 @@ void Network::train(int total_epochs)
double epochtime = (double) std::chrono::duration_cast<std::chrono::nanoseconds>(ep_end-ep_begin).count() / pow(10,9);
printf("Epoch %i/%i - time %f - cost %f - acc %f\n", epochs+1, total_epochs, epochtime, epoch_cost, epoch_accuracy);
printf("Avg time spent across %i batches: %lf on next batch, %lf on feedforward, %lf on backprop, %lf on cost, %lf on acc.\n", batches, times[0]/batches, times[1]/batches, times[2]/batches, times[3]/batches, times[4]/batches);
- printf("Time spent across epoch: %lf on next batch, %lf on feedforward, %lf on backprop, %lf on cost, %lf on acc.\n", times[0], times[1], times[2], times[3], times[4], epochtime-times[0]-times[1]-times[2]-times[3]-times[4]);
+ printf("Time spent across epoch: %lf on next batch, %lf on feedforward, %lf on backprop, %lf on cost, %lf on acc.\n\n", times[0], times[1], times[2], times[3], times[4], epochtime-times[0]-times[1]-times[2]-times[3]-times[4]);
batches=1;
epochs++;
rewind(data);
diff --git a/example.cpp b/example.cpp
@@ -1,16 +1,6 @@
#include "bpnn.hpp"
#include "utils.hpp"
-double lecun_tanh(double x)
-{
- return 1.7159 * tanh((2.0/3) * x);
-}
-
-double lecun_tanh_deriv(double x)
-{
- return 1.14393 * pow(1.0/cosh(2.0/3 * x),2);
-}
-
int main()
{
Network net ("./data_banknote_authentication.txt", 10, 0.01, 0.001);
diff --git a/example.py b/example.py
@@ -2,12 +2,6 @@ import mrbpnn
import numpy
import time
-def lecun_tanh(x):
- return 1.7159 * numpy.tanh((2.0/3) * x)
-
-def lecun_tanh_deriv(x):
- return 1.14393 * (1.0/numpy.cosh(2.0/3 * x))**2
-
init = time.time()
net = mrbpnn.Network("./data_banknote_authentication.txt", 10, 0.0155, 0.0155);
net.add_layer(4, "linear");
diff --git a/utils.cpp b/utils.cpp
@@ -8,31 +8,37 @@
#include <unistd.h>
#include <sys/stat.h>
-double sigmoid(double x)
-{
- return 1.0/(1+exp(-x));
-}
+// A bunch of hardcoded activation functions. Avoids much of the slowness of custom functions.
+// Although the std::function makes it not the fastest way, the functionality is worth it.
+// Yes, these functions may be a frustrating to read but they're just equations and I want to conserve space.
+double sigmoid(double x) {return 1.0/(1+exp(-x));}
+double sigmoid_deriv(double x) {return 1.0/(1+exp(-x)) * (1 - 1.0/(1+exp(x)));}
-double sigmoid_deriv(double x)
-{
- return 1.0/(1+exp(-x)) * (1 - 1.0/(1+exp(x)));
-}
+double linear(double x) {return x;}
+double linear_deriv(double x) {return 1;}
-double linear(double x)
+double lecun_tanh(double x) {return 1.7159 * tanh((2.0/3) * x);}
+double lecun_tanh_deriv(double x) {return 1.14393 * pow(1.0/cosh(2.0/3 * x),2);}
+
+double tanh(double x) {return tanh(x);} // For sake of symmetry.
+double tanh_deriv(double x) {return pow(1.0/cosh(x),2);}
+
+double logit(double x) {return log(x/(1-x));}
+double logit_deriv(double x) {return 1/(pow(x,2)-x);}
+
+double step(double x)
{
- return x;
+ if (x > 0) return 1;
+ else return 0;
}
+double step_deriv(double x) {return 0;}
-double linear_deriv(double x)
+double bipolar(double x)
{
- return 1;
+ if (x > 0) return 1;
+ else return -1;
}
-
-//double lecun_tanh(double x) {return 1.7159 * tanh((2.0/3) * x);}
-//double lecun_tanh_deriv(double x)
-//{
-// return 1.14393 * pow(1.0/cosh(2.0/3 * x),2);
-//}
+double bipolar_deriv(double x) {return 0;}
std::function<double(double)> rectifier(double (*activation)(double))
{
diff --git a/utils.hpp b/utils.hpp
@@ -7,6 +7,14 @@ double sigmoid(double x);
double sigmoid_deriv(double x);
double linear(double x);
double linear_deriv(double x);
+double step(double x);
+double step_deriv(double x);
+double bipolar(double x);
+double bipolar_deriv(double x);
+double logit(double x);
+double logit_deriv(double x);
+double lecun_tanh(double x);
+double lecun_tanh_deriv(double x);
std::function<double(double)> rectifier(double (*activation)(double));
#endif /* MODULE_H */
