commit a3a0497ed5158e6b2830b8f1e3cc95cfbf2b30a6
parent 20c4f7901d459a689c9c4201b3bb3fb83ff61c5d
Author: David Freifeld <freifeld.david@gmail.com>
Date: Thu, 25 Jun 2020 18:08:45 -0700
Trying to add functional-style rectifier
Diffstat:
3 files changed, 24 insertions(+), 44 deletions(-)
diff --git a/bpnn.cpp b/bpnn.cpp
@@ -3,23 +3,6 @@
#include <ctime>
#include <random>
-Layer::Layer(float* vals, int batch_sz, int nodes)
-{
- contents = new Eigen::MatrixXd (batch_sz, nodes);
- dZ = new Eigen::MatrixXd (batch_sz, nodes);
- int datalen = batch_sz*nodes;
- for (int i = 0; i < datalen; i++) {
- (*contents)((int)i / nodes,i%nodes) = vals[i];
- (*dZ)((int)i / nodes,i%nodes) = 0;
- }
- bias = new Eigen::MatrixXd (batch_sz, nodes);
- for (int i = 0; i < nodes; i++) {
- for (int j = 0; j < batch_sz; j++) {
- (*bias)(j, i) = 0.001;
- }
- }
-}
-
Layer::Layer(int batch_sz, int nodes)
{
contents = new Eigen::MatrixXd (batch_sz, nodes);
@@ -32,7 +15,7 @@ Layer::Layer(int batch_sz, int nodes)
bias = new Eigen::MatrixXd (batch_sz, nodes);
for (int i = 0; i < nodes; i++) {
for (int j = 0; j < batch_sz; j++) {
- (*bias)(j, i) = 0.001;
+ (*bias)(j, i) = 0;
}
}
dZ = new Eigen::MatrixXd (batch_sz, nodes);
@@ -88,8 +71,8 @@ void Network::set_activation(int index, char* name)
layers[index].activation_deriv = &linear_deriv;
}
else if (strcmp(name, "relu") == 0) {
- layers[index].activation = &relu;
- layers[index].activation_deriv = &relu_deriv;
+ layers[index].activation = &(rectifier(linear));
+ layers[index].activation_deriv = &(rectifier(linear_deriv));
}
else if (strcmp(name, "resig") == 0) {
layers[index].activation = &resig;
diff --git a/example.cpp b/example.cpp
@@ -3,14 +3,14 @@
int main()
{
- Network net ("./data_banknote_authentication.txt", 10, 0.0001, 0.000001);
+ Network net ("./data_banknote_authentication.txt", 10, 0.01, 0.001);
net.add_layer(4, "linear");
//net.add_layer(3, "resig");
net.add_layer(5, "sigmoid");
net.add_layer(1, "resig");
net.initialize();
net.list_net();
- net.train(500);
+ net.train(50);
// net.list_net();
//char line[1024];
//net.stream->getline(line, 1024);
diff --git a/utils.cpp b/utils.cpp
@@ -4,52 +4,49 @@
#include <ctime>
#include <cmath>
#include <cstdio>
+#include <functional>
#include <fcntl.h>
#include <unistd.h>
#include <sys/stat.h>
-double sigmoid(double x)
+inline double sigmoid(double x)
{
return 1.0/(1+exp(-x));
}
-double sigmoid_deriv(double x)
+inline double sigmoid_deriv(double x)
{
return 1.0/(1+exp(-x)) * (1 - 1.0/(1+exp(x)));
}
-double resig(double x)
-{
- if (x > 0) return 1.0/(1+exp(-x));
- else return 0;
-}
-
-double resig_deriv(double x)
-{
- if (x > 0) return 1.0/(1+exp(-x)) * (1 - 1.0/(1+exp(x)));
- else return 0;
-}
-
-double linear(double x)
+inline double linear(double x)
{
return x;
}
-double linear_deriv(double x)
+inline double linear_deriv(double x)
{
return 1;
}
-double relu(double x)
+std::function<double(double)> rectifier(double (*activation)(double), x)
{
- if (x > 0) return x;
- else return 0;
+ auto rectified = [](double x) -> double
+ {
+ if (x > 0) return (*activation)(x);
+ else return 0;
+ };
+ return rectified
}
-double relu_deriv(double x)
+double rectifier_deriv(double (*activation_deriv)(double), x)
{
- if (x > 0) return 1;
- else return 0;
+ auto rectified = [](double x) -> double
+ {
+ if (x > 0) return (*activation_deriv)(x);
+ else return 0;
+ };
+ return rectified
}
static uintmax_t wc(char const *fname)
