commit 0be5798cdcde98abeed8ca818d312cfcde51e410
parent 52c389b017298f07a363362afd73ecf54be816af
Author: David Freifeld <freifeld.david@gmail.com>
Date: Sun, 28 Mar 2021 18:17:21 -0700
Removed activation name property, fixed optimizers
Diffstat:
4 files changed, 77 insertions(+), 87 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -20,11 +20,12 @@ double bench(int batch_sz, int epochs)
{
auto start = std::chrono::high_resolution_clock::now();
Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, L2, 0, 0.9);
- net.add_layer(4, "linear", linear, linear_deriv);
- net.add_layer(5, "lecun", lecun_tanh, lecun_tanh_deriv);
- net.add_layer(2, "linear", linear, linear_deriv);
+ net.add_layer(4, linear, linear_deriv);
+ net.add_layer(5, lecun_tanh, lecun_tanh_deriv);
+ net.add_layer(2, linear, linear_deriv);
+ // net.init_optimizer(optimizers::demon(0.1, 50));
net.initialize();
- for (int i = 0; i < epochs; i++) {
+ for (int i = 0; i < epochs; i++) {
net.train();
}
auto end = std::chrono::high_resolution_clock::now();
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -56,11 +56,11 @@ Network::Network(const char* path, int batch_sz, float learn_rate, float bias_ra
data = open(TRAIN_BIN_PATH, O_RDONLY | O_NONBLOCK);
val_data = open(VAL_BIN_PATH, O_RDONLY | O_NONBLOCK);
instances = total_instances - val_instances;
- decay = [this]() -> void {};
- update = [this](std::vector<Eigen::MatrixXf> deltas, int i) {
- *layers[length-2-i].weights -= (learning_rate * deltas[i]);
+ decay = []() -> void {};
+ update = [](Layer& layer, Eigen::MatrixXf delta, float learning_rate) {
+ *layer.weights -= (learning_rate * delta);
};
- // File descriptors are nonnegative integers and open() returns -1 on failure.
+ // File descriptors are nonnegative integers and open() returns -1 on failure.
Ensures(batch_size < instances && data > 0 && val_data > 0);
}
@@ -70,12 +70,11 @@ Network::~Network()
close(val_data);
}
-void Network::add_layer(int nodes, const char* name, std::function<float(float)> activation, std::function<float(float)> activation_deriv)
+void Network::add_layer(int nodes, std::function<float(float)> activation, std::function<float(float)> activation_deriv)
{
Expects(nodes > 0);
length++;
layers.emplace_back(batch_size, nodes);
- strcpy(layers[length-1].activation_str, name);
layers[length-1].activation = activation;
layers[length-1].activation_deriv = activation_deriv;
}
@@ -116,7 +115,6 @@ void Network::feedforward()
{
for (int i = 0; i < length-1; i++) {
for (int j = 0; j < layers[i].contents->rows(); j++) {
- if (strcmp(layers[i].activation_str, "linear") == 0) break;
for (int k = 0; k < layers[i].contents->cols(); k++) {
(*layers[i].dZ)(j,k) = layers[i].activation_deriv((*layers[i].contents)(j,k));
(*layers[i].contents)(j,k) = layers[i].activation((*layers[i].contents)(j,k));
@@ -126,7 +124,6 @@ void Network::feedforward()
*layers[i+1].contents += *layers[i+1].bias;
}
for (int j = 0; j < layers[length-1].contents->rows(); j++) {
- if (strcmp(layers[length-1].activation_str, "linear") == 0) break;
for (int k = 0; k < layers[length-1].contents->cols(); k++) {
(*layers[length-1].dZ)(j,k) = layers[length-1].activation_deriv((*layers[length-1].contents)(j,k));
(*layers[length-1].contents)(j,k) = layers[length-1].activation((*layers[length-1].contents)(j,k));
@@ -138,12 +135,19 @@ void Network::feedforward()
void Network::list_net()
{
Expects(length > 1);
- std::cout << "-----------------------\nINPUT LAYER (LAYER 0)\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[0].activation_str << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[0].contents << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << *layers[0].weights << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[0].bias << "\n\n\n";
+ std::cout << "-----------------------\nINPUT LAYER (LAYER 0)\n"
+ << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[0].contents
+ << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << *layers[0].weights
+ << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[0].bias << "\n\n\n";
for (int i = 1; i < length-1; i++) {
- std::cout << "-----------------------\nLAYER " << i << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[i].activation_str;
- std::cout << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[i].contents << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[i].bias << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << *layers[i].weights << "\n\n\n";
+ std::cout << "-----------------------\nLAYER " << i
+ << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[i].contents
+ << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[i].bias
+ << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << *layers[i].weights << "\n\n\n";
}
- std::cout << "-----------------------\nOUTPUT LAYER (LAYER " << length-1 << ")\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[length-1].activation_str <<"\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[length-1].contents << "\n\n\u001b[31BIASES:\x1B[0;37m\n" << *layers[length-1].bias << "\n\n\n";
+ std::cout << "-----------------------\nOUTPUT LAYER (LAYER " << length-1
+ <<"\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[length-1].contents
+ << "\n\n\u001b[31BIASES:\x1B[0;37m\n" << *layers[length-1].bias << "\n\n\n";
}
float Network::cost()
@@ -213,8 +217,8 @@ Eigen::MatrixXf Network::backpropagate()
gradients.push_back(error);
deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
int counter = 1;
- for (int i = length-2; i >= 1; i--) {
- // TODO: Add nesterov momentum | -p B -t conundrum -t coding -m Without causing segmentation faults.
+ for (int i = length-2; i >= 1; i--) {
+ // TODO: Add nesterov momentum | -p B -t conundrum -t coding -m Without causing segmentation faults.
// (*layers[i].weights-((learning_rate * *layers[i].weights) + (0.9 * *layers[i].v))).transpose()
//grad_calc(gradients, counter, i)
gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
@@ -222,7 +226,7 @@ Eigen::MatrixXf Network::backpropagate()
counter++;
}
for (int i = 0; i < length-1; i++) {
- update(deltas, i);
+ update(layers[length-2-i], deltas[i], learning_rate);
if (reg_type == L2) *layers[length-2-i].weights -= ((lambda/batch_size) * (*layers[length-2-i].weights));
else if (reg_type == L1) *layers[length-2-i].weights -= ((lambda/(2*batch_size)) * l1_deriv(*layers[length-2-i].weights));
*layers[length-1-i].bias -= bias_lr * gradients[i];
@@ -249,6 +253,8 @@ void Network::validate(const char* path)
Ensures(lseek(val_data, 0, SEEK_CUR) == 0);
}
+#include "optimizers.cpp"
+
void Network::train()
{
float cost_sum = 0;
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -3,8 +3,6 @@
#include <eigen3/Eigen/Dense>
-//#include "../../mapreduce/mapreduce.h"
-
#include <vector>
#include <iostream>
#include <string>
@@ -16,7 +14,6 @@
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
-// #include <gsl/gsl_assert>
#include <lz4.h>
#define BUFFER_SIZE 600*1024
@@ -34,7 +31,7 @@ public:
std::function<float(float)> activation;
std::function<float(float)> activation_deriv;
char activation_str[32];
-
+
Layer(int rows, int columns);
Layer(float* vals, int rows, int columns);
void operator=(const Layer& that);
@@ -52,8 +49,8 @@ protected:
float val_cost;
std::function<void(void)> decay;
std::function<void(std::vector<Eigen::MatrixXf>, int, int)> grad_calc;
- std::function<void(std::vector<Eigen::MatrixXf>, int)> update;
- void next_batch(int fd);
+ std::function<void(Layer&, Eigen::MatrixXf, float)> update;
+ void next_batch(int fd);
public:
int data;
int val_data;
@@ -72,20 +69,21 @@ public:
int epochs = 0;
int batches = 0;
Eigen::MatrixXf* labels;
-
+
Network(const char* path, int batch_sz, float learn_rate,
float bias_rate, Regularization regularization,
float l, float ratio, bool early_exit=true, float cutoff=0);
~Network();
- void add_layer(int nodes, const char* name, std::function<float(float)> activation, std::function<float(float)> activation_deriv);
+ void add_layer(int nodes, std::function<float(float)> activation, std::function<float(float)> activation_deriv);
void initialize();
+ void init_optimizer(std::function<void(Layer&, Eigen::MatrixXf, float)> f);
void set_activation(int index, std::function<float(float)> custom, std::function<float(float)> custom_deriv);
void feedforward();
void softmax();
void list_net();
float cost();
float accuracy();
- Eigen::MatrixXf backpropagate();
+ Eigen::MatrixXf backpropagate();
void validate(const char* path);
void train();
float get_acc() {return epoch_acc;}
@@ -101,6 +99,13 @@ void prep(const char* rname, const char* wname);
void compress(const char* rname, const char* wname);
Eigen::MatrixXf l1_deriv(Eigen::MatrixXf m);
+namespace optimizers {
+std::function<void(Layer&, Eigen::MatrixXf, float)> momentum(float beta);
+std::function<void(Layer&, Eigen::MatrixXf, float)> demon(float beta_init, int max_ep);
+std::function<void(Layer&, Eigen::MatrixXf, float)> adam(float beta1, float beta2, float epsilon);
+std::function<void(Layer&, Eigen::MatrixXf, float)> adamax(float beta1, float beta2, float epsilon);
+}
+
#define MAXLINE 1024
#if (!RECKLESS)
diff --git a/src/optimizers.cpp b/src/optimizers.cpp
@@ -5,68 +5,46 @@
// Created by David Freifeld
//
-void Network::init_optimizer(const char* name, ...)
-{
- va_list args;
- va_start(args, name);
- if (strcmp(name, "momentum") == 0) {
- float beta = va_arg(args, double);
- va_end(args);
- update = [this, beta](std::vector<Eigen::MatrixXf> deltas, int i) {
- *layers[length-2-i].weights -= (beta * *layers[length-2-i].m) + (learning_rate * deltas[i]);
- *layers[length-2-i].m = (learning_rate * deltas[i]);
- };
- }
- // TODO: Remove redundant code | -t quality -m Attempt split into functions to remove reundant code
- if (strcmp(name, "nesterov") == 0) {
- float beta = va_arg(args, double);
- va_end(args);
- grad_calc = [this](std::vector<Eigen::MatrixXf> gradients, int i, int counter) -> void {
- gradients.push_back((gradients[counter-1] * (*layers[i].weights-((learning_rate * *layers[i].weights) + (0.9 * *layers[i].v))).transpose()).cwiseProduct(*layers[i].dZ));
- };
- update = [this, beta](std::vector<Eigen::MatrixXf> deltas, int i) {
- *layers[length-2-i].weights -= (beta * *layers[length-2-i].m) + (learning_rate * deltas[i]);
- *layers[length-2-i].m = (learning_rate * deltas[i]);
+
+std::function<void(Layer&, Eigen::MatrixXf, float)> optimizers::momentum(float beta) {
+ return [beta](Layer& layer, Eigen::MatrixXf delta, float learning_rate) {
+ *layer.weights -= (beta * *layer.m) + (learning_rate * delta);
+ *layer.m = (learning_rate * delta);
};
- }
- else if (strcmp(name, "demon") == 0) {
- float beta_init = va_arg(args, double);
- float max_ep = va_arg(args, int);
- float beta = beta_init;
- int prev_epoch = -1;
- update = [this, max_ep, prev_epoch, beta_init, beta](std::vector<Eigen::MatrixXf> deltas, int i) mutable {
- if (epochs > prev_epoch) {
+}
+
+std::function<void(Layer&, Eigen::MatrixXf, float)> optimizers::demon(float beta, int max_ep) {
+ float beta_init = beta;
+ float prev_epoch = -1;
+ float epochs = 0;
+ return [max_ep, epochs, beta_init, beta](Layer& layer, Eigen::MatrixXf delta, float learning_rate) mutable {
beta = beta_init * (1-(epochs/max_ep)) / ((beta_init * (1-(epochs/max_ep))) + (1-beta_init));
- prev_epoch = epochs;
- }
- *layers[length-2-i].weights -= (beta * *layers[length-2-i].m) + (learning_rate * deltas[i]);
- *layers[length-2-i].m = (learning_rate * deltas[i]);
+ *layer.weights -= (beta * *layer.m) + (learning_rate * delta);
+ *layer.m = (learning_rate * delta);
+ epochs++;
};
- }
- else if (strcmp(name, "adam") == 0) {
- float beta1 = va_arg(args, double);
- float beta2 = va_arg(args, double);
- float epsilon = va_arg(args, double);
- // TODO: Add bias correction to adam | -t coding -m (requires figuring out measuring t)
- // TODO: Investigate cwiseProduct in code | -t quality -m cwiseProduct here is sketchy, look into me
- update = [this, beta1, beta2, epsilon](std::vector<Eigen::MatrixXf> deltas, int i) {
- *layers[length-2-i].m = (beta1 * *layers[length-2-i].m) + ((1-beta1)*deltas[i]);
- *layers[length-2-i].v = (beta2 * *layers[length-2-i].v) + (1-beta2)*(deltas[i].cwiseProduct(deltas[i]));
- *layers[length-2-i].weights -= learning_rate * ((layers[length-2-i].v->cwiseSqrt()).array()+epsilon).pow(-1).cwiseProduct(layers[length-2-i].m->array()).matrix();
+}
+
+std::function<void(Layer&, Eigen::MatrixXf, float)> optimizers::adam(float beta1, float beta2, float epsilon) {
+ return [beta1, beta2, epsilon](Layer& layer, Eigen::MatrixXf delta, float learning_rate) {
+ *layer.m = (beta1 * *layer.m) + ((1-beta1)*delta);
+ *layer.v = (beta2 * *layer.v) + (1-beta2)*(delta.cwiseProduct(delta));
+ *layer.weights -= learning_rate *
+ ((layer.v->cwiseSqrt()).array()+epsilon).pow(-1).cwiseProduct(layer.m->array()).matrix();
};
- }
- else if (strcmp(name, "adamax") == 0) {
- float beta1 = va_arg(args, double);
- float beta2 = va_arg(args, double);
- float epsilon = va_arg(args, double);
- // TODO: Add bias correction for adamax | -t coding -m (requires figuring out measuring t)
- update = [this, beta1, beta2, epsilon](std::vector<Eigen::MatrixXf> deltas, int i) {
- *layers[length-2-i].m = (beta1 * *layers[length-2-i].m) + ((1-beta1)*deltas[i]);
- // TODO: Fix Adamax calculations | -p C -t quality -m Use of .sum() here is incredibly questionable. Do this correctly.
- if ((beta2 * *layers[length-2-i].v).sum() > deltas[i].array().abs().sum()) *layers[length-2-i].v = (beta2 * *layers[length-2-i].v);
- else *layers[length-2-i].v = deltas[i].array().abs().matrix();
- *layers[length-2-i].weights -= learning_rate * (layers[length-2-i].v->array().pow(-1).cwiseProduct(layers[length-2-i].m->array())).matrix();
+}
+
+std::function<void(Layer&, Eigen::MatrixXf, float)> optimizers::adamax(float beta1, float beta2, float epsilon) {
+ return [beta1, beta2, epsilon](Layer& layer, Eigen::MatrixXf delta, float learning_rate) {
+ *layer.m = (beta1 * *layer.m) + ((1-beta1)*delta);
+ if ((beta2 * *layer.v).sum() > delta.array().abs().sum()) *layer.v = (beta2 * *layer.v);
+ else *layer.v = delta.array().abs().matrix();
+ *layer.weights -= learning_rate *
+ (layer.v->array().pow(-1).cwiseProduct(layer.m->array())).matrix();
};
- }
- va_end(args);
+}
+
+void Network::init_optimizer(std::function<void(Layer&, Eigen::MatrixXf, float)> f)
+{
+ update = f;
}
