commit e74103c3940e4fa789cfd3ec3d966bad0afb168c
parent 23b633acde15d0c7f697fe95c0fdbbe4bc5b5ff5
Author: David Freifeld <freifeld.david@gmail.com>
Date: Mon, 10 Aug 2020 18:45:40 -0700
Working on proper copy constructors to allow for safer checks
Also fixed some indentation.
Diffstat:
| M | example.cpp | | | 1 | + |
| M | src/bpnn.cpp | | | 24 | ++++++++++++++++++------ |
| M | src/bpnn.hpp | | | 163 | +++++++++++++++++++++++++++++++++++++++---------------------------------------- |
| M | src/checks.cpp | | | 188 | ++++++++++++++++++++++++++++++++++++++++---------------------------------------- |
4 files changed, 194 insertions(+), 182 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -23,6 +23,7 @@ double bench(int batch_sz, int epochs)
net.add_layer(5, "lecun_tanh");
net.add_layer(2, "linear");
net.initialize();
+ checks(net);
for (int i = 0; i < epochs; i++) {
net.train();
}
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -20,7 +20,7 @@
#define cwise_product(a,b) (a).cwiseProduct(b)
#endif
-//#include "checks.cpp"
+#include "checks.cpp"
Layer::Layer(int batch_sz, int nodes, float a)
:alpha(a)
@@ -40,6 +40,18 @@ Layer::Layer(int batch_sz, int nodes, float a)
}
}
+Layer::Layer(const Layer& that)
+ :activation(that.activation), activation_deriv(that.activation_deriv), alpha(that.alpha)
+{
+ //strcpy(activation_str, that.activation_str);
+ *contents = *that.contents;
+ *v = *that.v;
+ *m = *that.m;
+ *weights = *that.weights;
+ *bias = *that.bias;
+ *dZ = *that.dZ;
+}
+
void Layer::init_weights(Layer next)
{
v = new Eigen::MatrixXf (contents->cols(), next.contents->cols());
@@ -75,6 +87,11 @@ Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, in
};
}
+// Network::Network(const Network& that)
+// :name(that.name), age(that.age)
+// {
+// }
+
void Network::init_decay(char* type, ...)
{
va_list args;
@@ -548,8 +565,3 @@ void Network::train()
decay();
epochs++;
}
-
-float Network::get_acc() {return epoch_acc;}
-float Network::get_val_acc() {return val_acc;}
-float Network::get_cost() {return epoch_cost;}
-float Network::get_val_cost() {return val_cost;}
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -17,83 +17,84 @@
class Layer {
public:
- Eigen::MatrixXf* contents;
- Eigen::MatrixXf* v;
- Eigen::MatrixXf* m;
- Eigen::MatrixXf* weights;
- Eigen::MatrixXf* bias;
- Eigen::MatrixXf* dZ;
- std::vector<Eigen::MatrixXf> prev_updates;
- std::function<float(float)> activation;
- std::function<float(float)> activation_deriv;
- char activation_str[1024];
- // PReLU layers shouldn't be Layers but inherit from them! Fix me!!
- float alpha;
+ Eigen::MatrixXf* contents;
+ Eigen::MatrixXf* v;
+ Eigen::MatrixXf* m;
+ Eigen::MatrixXf* weights;
+ Eigen::MatrixXf* bias;
+ Eigen::MatrixXf* dZ;
+ std::function<float(float)> activation;
+ std::function<float(float)> activation_deriv;
+ char activation_str[1024];
+ // PReLU layers shouldn't be Layers but inherit from them! Fix me!!
+ float alpha;
- Layer(int rows, int columns, float a=0);
- Layer(float* vals, int rows, int columns);
- void init_weights(Layer next);
+ Layer(int rows, int columns, float a=0);
+ Layer(float* vals, int rows, int columns);
+ Layer(const Layer& that);
+ void init_weights(Layer next);
};
class Network {
public:
- FILE* data;
- FILE* val_data;
- FILE* test_data;
- int instances;
- int val_instances;
- int test_instances;
- Eigen::MatrixXf numerical_grad(int i, float epsilon);
- void update_layer(float* vals, int datalen, int index);
+ FILE* data;
+ FILE* val_data;
+ FILE* test_data;
+ int instances;
+ int val_instances;
+ int test_instances;
+ Eigen::MatrixXf numerical_grad(int i, float epsilon);
+ void update_layer(float* vals, int datalen, int index);
- std::vector<Layer> layers;
- int length = 0;
+ std::vector<Layer> layers;
+ int length = 0;
- float epoch_acc;
- float epoch_cost;
- float val_acc;
- float val_cost;
+ float epoch_acc;
+ float epoch_cost;
+ float val_acc;
+ float val_cost;
- float learning_rate;
- float bias_lr;
- float lambda;
- int reg_type;
- int batch_size;
-
- int epochs = 0;
- int batches = 0;
- Eigen::MatrixXf* labels;
-
- std::function<void(void)> decay;
- std::function<void(std::vector<Eigen::MatrixXf>, int, int)> grad_calc;
- std::function<void(std::vector<Eigen::MatrixXf>, int)> update;
-
- Network(char* path, int batch_sz, float learn_rate, float bias_rate, int regularization, float l, float ratio, bool early_exit=true, float cutoff=0);
- void add_layer(int nodes, char* activation);
- void add_prelu_layer(int nodes, float a);
- void init_decay(char* type, ...);
- void init_optimizer(char* name, ...);
- void initialize();
- void grad_check();
- void set_activation(int index, std::function<float(float)> custom, std::function<float(float)> custom_deriv);
+ float learning_rate;
+ float bias_lr;
+ float lambda;
+ int reg_type;
+ int batch_size;
+
+ int epochs = 0;
+ int batches = 0;
+ Eigen::MatrixXf* labels;
+
+ std::function<void(void)> decay;
+ std::function<void(std::vector<Eigen::MatrixXf>, int, int)> grad_calc;
+ std::function<void(std::vector<Eigen::MatrixXf>, int)> update;
+
+ Network(char* path, int batch_sz, float learn_rate, float bias_rate, int regularization, float l, float ratio, bool early_exit=true, float cutoff=0);
+ void add_layer(int nodes, char* activation);
+ void add_prelu_layer(int nodes, float a);
+ void init_decay(char* type, ...);
+ void init_optimizer(char* name, ...);
+ void initialize();
+ void grad_check();
+ void set_activation(int index, std::function<float(float)> custom, std::function<float(float)> custom_deriv);
- void feedforward();
- void list_net();
-
- bool early_stop;
- float threshold;
-
- float cost();
- float accuracy();
- void backpropagate();
- int next_batch();
- float validate(char* path);
- void train();
-
- float get_acc();
- float get_cost();
- float get_val_acc();
- float get_val_cost();
+ void feedforward();
+ void list_net();
+
+ bool early_stop;
+ float threshold;
+
+ float cost();
+ float accuracy();
+ void backpropagate();
+ int next_batch();
+ float validate(char* path);
+ void train();
+
+
+ float get_acc() {return epoch_acc;}
+ float get_val_acc() {return val_acc;}
+ float get_cost() {return epoch_cost;}
+ float get_val_cost() {return val_cost;}
};
void checks(Network net);
@@ -103,22 +104,20 @@ int split_file(char* path, int lines, float ratio);
struct ValueError : public std::exception
{
- const char* message;
- const char* location;
- ValueError(const char* msg, const char* loc)
- :message{msg}, location{loc}
- {
- }
- const char* what() const throw () {
- char* error;
- sprintf(error, "%s (thrown in %s).", message, location);
- const char* error_message = error;
- return error_message;
- }
+ const char* message;
+ const char* location;
+ ValueError(const char* msg, const char* loc)
+ :message{msg}, location{loc}
+ {
+ }
+ const char* what() const throw () {
+ char* error;
+ sprintf(error, "%s (thrown in %s).", message, location);
+ const char* error_message = error;
+ return error_message;
+ }
};
-Eigen::MatrixXf l1_deriv(Eigen::MatrixXf);
-
#define MAXLINE 1024
#define ZERO_THRESHOLD pow(10, -8) // for checks
diff --git a/src/checks.cpp b/src/checks.cpp
@@ -8,124 +8,124 @@
void checks(Network net)
{
- Network original = net;
int sanity_passed = 0;
std::cout << "\u001b[4m\u001b[1mSANITY CHECKS:\u001b[0m\n";
// Check if regularization strength increases loss (as it should).
std::cout << "Regularization sanity check...";
- // list_net();
-
- Network copy1 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- Network copy2 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ net.list_net();
+ Network copy1 = net;
+ Network copy2 = net;
copy1.lambda += 1;
copy1.next_batch();
copy1.feedforward();
+
copy2.next_batch();
copy2.feedforward();
+ net.list_net();
if (copy1.cost() > copy2.cost()) {
std::cout << " \u001b[32mPassed!\n\u001b[37m";
sanity_passed++;
}
else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- // net.list_net();
+ // // net.list_net();
- // Check if zero cost is achievable on a batch
- std::cout << "Zero-cost sanity check...";
- Network copy3 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- copy3.lambda = 0;
- copy3.next_batch();
- float finalcost;
- for (int i = 0; i < 10000; i++) {
- copy3.feedforward();
- copy3.backpropagate();
- finalcost = copy3.cost();
- if (finalcost <= ZERO_THRESHOLD) {
- break;
- }
- }
- if (finalcost <= ZERO_THRESHOLD) {
- std::cout << " \u001b[32mPassed!\n\u001b[37m";
- sanity_passed++;
- }
- else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ // // Check if zero cost is achievable on a batch
+ // std::cout << "Zero-cost sanity check...";
+ // Network copy3 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ // copy3.lambda = 0;
+ // copy3.next_batch();
+ // float finalcost;
+ // for (int i = 0; i < 10000; i++) {
+ // copy3.feedforward();
+ // copy3.backpropagate();
+ // finalcost = copy3.cost();
+ // if (finalcost <= ZERO_THRESHOLD) {
+ // break;
+ // }
+ // }
+ // if (finalcost <= ZERO_THRESHOLD) {
+ // std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ // sanity_passed++;
+ // }
+ // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- // list_net();
+ // // list_net();
- std::cout << "Gradient floating-point sanity check...";
- Network copy4 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- copy4.next_batch();
- copy4.feedforward();
- std::vector<Eigen::MatrixXf> gradients;
- std::vector<Eigen::MatrixXf> deltas;
- Eigen::MatrixXf error = ((*copy4.layers[copy4.length-1].contents) - (*copy1.labels));
- gradients.push_back(error.cwiseProduct(*copy4.layers[copy4.length-1].dZ));
- deltas.push_back((*copy4.layers[copy4.length-2].contents).transpose() * gradients[0]);
- int counter = 1;
- for (int i = copy4.length-2; i >= 1; i--) {
- gradients.push_back((gradients[counter-1] * copy4.layers[i].weights->transpose()).cwiseProduct(*copy4.layers[i].dZ));
- deltas.push_back(copy4.layers[i-1].contents->transpose() * gradients[counter]);
- counter++;
- }
- auto check_gradients = [](std::vector<Eigen::MatrixXf> vec) -> bool {
- for (Eigen::MatrixXf i : vec) {
- for (int j = 0; j < i.rows(); j++) {
- for (int k = 0; k < i.cols(); k++) {
- if (i(j,k) == -0 || i(j,k) == INFINITY || i(j,k) == NAN || i(j,k) == -INFINITY) {
- return true;
- }
- }
- }
- }
- return false;
- };
- if (check_gradients(gradients) == false && check_gradients(deltas) == false) {
- std::cout << " \u001b[32mPassed!\n\u001b[37m";
- sanity_passed++;
- }
- else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ // std::cout << "Gradient floating-point sanity check...";
+ // Network copy4 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ // copy4.next_batch();
+ // copy4.feedforward();
+ // std::vector<Eigen::MatrixXf> gradients;
+ // std::vector<Eigen::MatrixXf> deltas;
+ // Eigen::MatrixXf error = ((*copy4.layers[copy4.length-1].contents) - (*copy1.labels));
+ // gradients.push_back(error.cwiseProduct(*copy4.layers[copy4.length-1].dZ));
+ // deltas.push_back((*copy4.layers[copy4.length-2].contents).transpose() * gradients[0]);
+ // int counter = 1;
+ // for (int i = copy4.length-2; i >= 1; i--) {
+ // gradients.push_back((gradients[counter-1] * copy4.layers[i].weights->transpose()).cwiseProduct(*copy4.layers[i].dZ));
+ // deltas.push_back(copy4.layers[i-1].contents->transpose() * gradients[counter]);
+ // counter++;
+ // }
+ // auto check_gradients = [](std::vector<Eigen::MatrixXf> vec) -> bool {
+ // for (Eigen::MatrixXf i : vec) {
+ // for (int j = 0; j < i.rows(); j++) {
+ // for (int k = 0; k < i.cols(); k++) {
+ // if (i(j,k) == -0 || i(j,k) == INFINITY || i(j,k) == NAN || i(j,k) == -INFINITY) {
+ // return true;
+ // }
+ // }
+ // }
+ // }
+ // return false;
+ // };
+ // if (check_gradients(gradients) == false && check_gradients(deltas) == false) {
+ // std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ // sanity_passed++;
+ // }
+ // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- // list_net();
+ // // list_net();
- std::cout << "Expected loss sanity check...";
+ // std::cout << "Expected loss sanity check...";
- Network copy5 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- copy5.next_batch();
- copy5.feedforward();
- if (copy5.cost() <= 1) {
- std::cout << " \u001b[32mPassed!\n\u001b[37m";
- sanity_passed++;
- }
- else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ // Network copy5 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ // copy5.next_batch();
+ // copy5.feedforward();
+ // if (copy5.cost() <= 1) {
+ // std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ // sanity_passed++;
+ // }
+ // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- // list_net();
+ // // list_net();
- std::cout << "Layer updates sanity check...";
- Network copy6 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- Network copy7 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- //copy2.list_net();
- //copy1.list_net();
- int passed;
- for (int i = 0; i < copy1.layers.size()-1; i++) {
- if (*copy1.layers[i].weights == *copy2.layers[i].weights) {
- // std::cout << *copy2.layers[i].weights <<"uninitweight\n\n";
- // std::cout << *copy1.layers[i].weights << " "<<i<<"weight\n\n";
- passed = -1;
- }
- }
- for (int i = 1; i < copy1.layers.size(); i++) {
- if (*copy1.layers[i].bias == *copy2.layers[i].bias) {
- // std::cout << *copy2.layers[i].bias <<"uninitbias\n\n";
- // std::cout << *copy1.layers[i].bias <<" " << i << "bias\n\n";
- passed = -1;
- }
- }
- if (passed == 1) {
- std::cout << " \u001b[32mPassed!\n\u001b[37m";
- sanity_passed++;
- }
- else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ // std::cout << "Layer updates sanity check...";
+ // Network copy6 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ // Network copy7 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ // //copy2.list_net();
+ // //copy1.list_net();
+ // int passed;
+ // for (int i = 0; i < copy1.layers.size()-1; i++) {
+ // if (*copy1.layers[i].weights == *copy2.layers[i].weights) {
+ // // std::cout << *copy2.layers[i].weights <<"uninitweight\n\n";
+ // // std::cout << *copy1.layers[i].weights << " "<<i<<"weight\n\n";
+ // passed = -1;
+ // }
+ // }
+ // for (int i = 1; i < copy1.layers.size(); i++) {
+ // if (*copy1.layers[i].bias == *copy2.layers[i].bias) {
+ // // std::cout << *copy2.layers[i].bias <<"uninitbias\n\n";
+ // // std::cout << *copy1.layers[i].bias <<" " << i << "bias\n\n";
+ // passed = -1;
+ // }
+ // }
+ // if (passed == 1) {
+ // std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ // sanity_passed++;
+ // }
+ // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
// std::cout << "Side effects sanity check...";
