commit a74bbc575ff06ed02020a2bbef68c398d1ee2da3
parent e3b403e0fed7926fc886bfb1fb307ee8539cc875
Author: David Freifeld <freifeld.david@gmail.com>
Date: Tue, 11 Aug 2020 13:23:16 -0700
Working on checks some more
Diffstat:
4 files changed, 172 insertions(+), 179 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -23,7 +23,7 @@ double bench(int batch_sz, int epochs)
net.add_layer(5, "lecun_tanh");
net.add_layer(2, "linear");
net.initialize();
- checks(net);
+ checks();
for (int i = 0; i < epochs; i++) {
net.train();
}
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -40,38 +40,18 @@ Layer::Layer(int batch_sz, int nodes, float a)
}
}
-Layer::Layer(const Layer& that)
-{
- contents = new Eigen::MatrixXf (that.contents->rows(), that.contents->cols());
- dZ = new Eigen::MatrixXf (that.dZ->rows(), that.dZ->cols());
- bias = new Eigen::MatrixXf (that.bias->rows(), that.bias->cols());
- v = new Eigen::MatrixXf (that.v->rows(), that.v->cols());
- m = new Eigen::MatrixXf (that.m->rows(), that.m->cols());
- weights = new Eigen::MatrixXf (that.weights->rows(), that.weights->cols());
- activation = that.activation;
- activation_deriv = that.activation_deriv;
- alpha = that.alpha;
- strcpy(activation_str, that.activation_str);
- *contents = *that.contents;
- *v = *that.v;
- *m = *that.m;
- *weights = *that.weights;
- *bias = *that.bias;
- *dZ = *that.dZ;
-}
-
void Layer::operator=(const Layer& that)
{
activation = that.activation;
activation_deriv = that.activation_deriv;
alpha = that.alpha;
strcpy(activation_str, that.activation_str);
- *contents = *that.contents;
- *v = *that.v;
- *m = *that.m;
- *weights = *that.weights;
- *bias = *that.bias;
- *dZ = *that.dZ;
+ // *contents = *that.contents;
+ // *v = *that.v;
+ // *m = *that.m;
+ // *weights = *that.weights;
+ // *bias = *that.bias;
+ // *dZ = *that.dZ;
}
void Layer::init_weights(Layer next)
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -31,7 +31,6 @@ public:
Layer(int rows, int columns, float a=0);
Layer(float* vals, int rows, int columns);
- Layer(const Layer& that);
void operator=(const Layer& that);
void init_weights(Layer next);
};
@@ -98,7 +97,7 @@ public:
float get_val_cost() {return val_cost;}
};
-void checks(Network net);
+void checks();
void demo(int total_epochs);
int prep_file(char* path, char* out_path);
int split_file(char* path, int lines, float ratio);
diff --git a/src/checks.cpp b/src/checks.cpp
@@ -3,174 +3,188 @@
// Jacobian
//
// Created by David Freifeld
-// Copyright © 2020 David Freifeld. All rights reserved.
//
-void checks(Network net)
+Network explicit_copy(Network src)
{
- int sanity_passed = 0;
- std::cout << "\u001b[4m\u001b[1mSANITY CHECKS:\u001b[0m\n";
- // Check if regularization strength increases loss (as it should).
- std::cout << "Regularization sanity check...";
+ Network dst ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
+ dst.layers = src.layers;
+ assert(src.layers.size() == dst.layers.size());
+ for (int i = 0; i < src.layers.size(); i++) {
+ dst.layers[i] = src.layers[i];
+ }
+ return src;
+}
+
+void checks()
+{
+ Network net ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
+ net.add_layer(4, "linear");
+ net.add_layer(5, "lecun_tanh");
+ net.add_layer(2, "linear");
+ net.initialize();
+ int sanity_passed = 0;
+ std::cout << "\u001b[4m\u001b[1mSANITY CHECKS:\u001b[0m\n";
+ // Check if regularization strength increases loss (as it should).
+ std::cout << "Regularization sanity check...";
- net.list_net();
- Network copy1 = net;
- Network copy2 = net;
- copy1.lambda += 1;
- copy1.next_batch();
- copy1.feedforward();
+ net.list_net();
+ Network copy1 = explicit_copy(net);
+ Network copy2 = explicit_copy(net);
+ copy1.next_batch();
+ copy1.feedforward();
- copy2.next_batch();
- copy2.feedforward();
- net.list_net();
- if (copy1.cost() > copy2.cost()) {
- std::cout << " \u001b[32mPassed!\n\u001b[37m";
- sanity_passed++;
- }
- else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ copy2.next_batch();
+ copy2.feedforward();
+ net.list_net();
+ if (copy1.cost() > copy2.cost()) {
+ std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ sanity_passed++;
+ }
+ else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- // // net.list_net();
+ // // net.list_net();
- // // Check if zero cost is achievable on a batch
- // std::cout << "Zero-cost sanity check...";
- // Network copy3 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- // copy3.lambda = 0;
- // copy3.next_batch();
- // float finalcost;
- // for (int i = 0; i < 10000; i++) {
- // copy3.feedforward();
- // copy3.backpropagate();
- // finalcost = copy3.cost();
- // if (finalcost <= ZERO_THRESHOLD) {
- // break;
- // }
- // }
- // if (finalcost <= ZERO_THRESHOLD) {
- // std::cout << " \u001b[32mPassed!\n\u001b[37m";
- // sanity_passed++;
- // }
- // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ // // Check if zero cost is achievable on a batch
+ // std::cout << "Zero-cost sanity check...";
+ // Network copy3 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ // copy3.lambda = 0;
+ // copy3.next_batch();
+ // float finalcost;
+ // for (int i = 0; i < 10000; i++) {
+ // copy3.feedforward();
+ // copy3.backpropagate();
+ // finalcost = copy3.cost();
+ // if (finalcost <= ZERO_THRESHOLD) {
+ // break;
+ // }
+ // }
+ // if (finalcost <= ZERO_THRESHOLD) {
+ // std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ // sanity_passed++;
+ // }
+ // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- // // list_net();
+ // // list_net();
- // std::cout << "Gradient floating-point sanity check...";
- // Network copy4 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- // copy4.next_batch();
- // copy4.feedforward();
- // std::vector<Eigen::MatrixXf> gradients;
- // std::vector<Eigen::MatrixXf> deltas;
- // Eigen::MatrixXf error = ((*copy4.layers[copy4.length-1].contents) - (*copy1.labels));
- // gradients.push_back(error.cwiseProduct(*copy4.layers[copy4.length-1].dZ));
- // deltas.push_back((*copy4.layers[copy4.length-2].contents).transpose() * gradients[0]);
- // int counter = 1;
- // for (int i = copy4.length-2; i >= 1; i--) {
- // gradients.push_back((gradients[counter-1] * copy4.layers[i].weights->transpose()).cwiseProduct(*copy4.layers[i].dZ));
- // deltas.push_back(copy4.layers[i-1].contents->transpose() * gradients[counter]);
- // counter++;
- // }
- // auto check_gradients = [](std::vector<Eigen::MatrixXf> vec) -> bool {
- // for (Eigen::MatrixXf i : vec) {
- // for (int j = 0; j < i.rows(); j++) {
- // for (int k = 0; k < i.cols(); k++) {
- // if (i(j,k) == -0 || i(j,k) == INFINITY || i(j,k) == NAN || i(j,k) == -INFINITY) {
- // return true;
- // }
- // }
- // }
- // }
- // return false;
- // };
- // if (check_gradients(gradients) == false && check_gradients(deltas) == false) {
- // std::cout << " \u001b[32mPassed!\n\u001b[37m";
- // sanity_passed++;
- // }
- // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ // std::cout << "Gradient floating-point sanity check...";
+ // Network copy4 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ // copy4.next_batch();
+ // copy4.feedforward();
+ // std::vector<Eigen::MatrixXf> gradients;
+ // std::vector<Eigen::MatrixXf> deltas;
+ // Eigen::MatrixXf error = ((*copy4.layers[copy4.length-1].contents) - (*copy1.labels));
+ // gradients.push_back(error.cwiseProduct(*copy4.layers[copy4.length-1].dZ));
+ // deltas.push_back((*copy4.layers[copy4.length-2].contents).transpose() * gradients[0]);
+ // int counter = 1;
+ // for (int i = copy4.length-2; i >= 1; i--) {
+ // gradients.push_back((gradients[counter-1] * copy4.layers[i].weights->transpose()).cwiseProduct(*copy4.layers[i].dZ));
+ // deltas.push_back(copy4.layers[i-1].contents->transpose() * gradients[counter]);
+ // counter++;
+ // }
+ // auto check_gradients = [](std::vector<Eigen::MatrixXf> vec) -> bool {
+ // for (Eigen::MatrixXf i : vec) {
+ // for (int j = 0; j < i.rows(); j++) {
+ // for (int k = 0; k < i.cols(); k++) {
+ // if (i(j,k) == -0 || i(j,k) == INFINITY || i(j,k) == NAN || i(j,k) == -INFINITY) {
+ // return true;
+ // }
+ // }
+ // }
+ // }
+ // return false;
+ // };
+ // if (check_gradients(gradients) == false && check_gradients(deltas) == false) {
+ // std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ // sanity_passed++;
+ // }
+ // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- // // list_net();
+ // // list_net();
- // std::cout << "Expected loss sanity check...";
+ // std::cout << "Expected loss sanity check...";
- // Network copy5 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- // copy5.next_batch();
- // copy5.feedforward();
- // if (copy5.cost() <= 1) {
- // std::cout << " \u001b[32mPassed!\n\u001b[37m";
- // sanity_passed++;
- // }
- // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ // Network copy5 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ // copy5.next_batch();
+ // copy5.feedforward();
+ // if (copy5.cost() <= 1) {
+ // std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ // sanity_passed++;
+ // }
+ // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- // // list_net();
+ // // list_net();
- // std::cout << "Layer updates sanity check...";
- // Network copy6 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- // Network copy7 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- // //copy2.list_net();
- // //copy1.list_net();
- // int passed;
- // for (int i = 0; i < copy1.layers.size()-1; i++) {
- // if (*copy1.layers[i].weights == *copy2.layers[i].weights) {
- // // std::cout << *copy2.layers[i].weights <<"uninitweight\n\n";
- // // std::cout << *copy1.layers[i].weights << " "<<i<<"weight\n\n";
- // passed = -1;
- // }
- // }
- // for (int i = 1; i < copy1.layers.size(); i++) {
- // if (*copy1.layers[i].bias == *copy2.layers[i].bias) {
- // // std::cout << *copy2.layers[i].bias <<"uninitbias\n\n";
- // // std::cout << *copy1.layers[i].bias <<" " << i << "bias\n\n";
- // passed = -1;
- // }
- // }
- // if (passed == 1) {
- // std::cout << " \u001b[32mPassed!\n\u001b[37m";
- // sanity_passed++;
- // }
- // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ // std::cout << "Layer updates sanity check...";
+ // Network copy6 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ // Network copy7 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ // //copy2.list_net();
+ // //copy1.list_net();
+ // int passed;
+ // for (int i = 0; i < copy1.layers.size()-1; i++) {
+ // if (*copy1.layers[i].weights == *copy2.layers[i].weights) {
+ // // std::cout << *copy2.layers[i].weights <<"uninitweight\n\n";
+ // // std::cout << *copy1.layers[i].weights << " "<<i<<"weight\n\n";
+ // passed = -1;
+ // }
+ // }
+ // for (int i = 1; i < copy1.layers.size(); i++) {
+ // if (*copy1.layers[i].bias == *copy2.layers[i].bias) {
+ // // std::cout << *copy2.layers[i].bias <<"uninitbias\n\n";
+ // // std::cout << *copy1.layers[i].bias <<" " << i << "bias\n\n";
+ // passed = -1;
+ // }
+ // }
+ // if (passed == 1) {
+ // std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ // sanity_passed++;
+ // }
+ // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- // std::cout << "Side effects sanity check...";
+ // std::cout << "Side effects sanity check...";
- // if (net == original) {
- // std::cout << " \u001b[32mPassed!\n\u001b[37m";
- // sanity_passed++;
- // }
- // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ // if (net == original) {
+ // std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ // sanity_passed++;
+ // }
+ // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- std::cout << "\u001b[1m\nPassed " << sanity_passed << "/6" <<" sanity checks.\u001b[0m\n\n\n";
+ std::cout << "\u001b[1m\nPassed " << sanity_passed << "/6" <<" sanity checks.\u001b[0m\n\n\n";
- // net.list_net();
+ // net.list_net();
- // float epsilon = 0.0001;
- // Network copy = *this;
- // std::vector<Eigen::MatrixXf> approx_gradients;
- // for (int i = 0; i < copy.layers.size()-1; i++) {
- // Eigen::MatrixXf current_approx = *copy.layers[i].weights;
- // for (int j = 0; i < copy.layers[i].weights->rows(); i++) {
- // for (int k = 0; i < copy.layers[i].weights->cols(); i++) {
- // Network sim1 = copy;
- // (*sim1.layers[i].contents)(j,k) += epsilon;
- // sim1.feedforward();
- // Network sim2 = copy;
- // (*sim2.layers[i].contents)(j,k) -= epsilon;
- // sim2.feedforward();
- // current_approx(j,k) = (sim1.cost() - sim2.cost())/(2*epsilon);
- // }
- // }
- // approx_gradients.push_back(current_approx);
- // }
- // for (Eigen::MatrixXf i : approx_gradients) {
- // std::cout << i << "\n\n";
- // }
- // std::vector<Eigen::MatrixXf> gradients;
- // std::vector<Eigen::MatrixXf> deltas;
- // Eigen::MatrixXf error = ((*layers[length-1].contents) - (*labels));
- // gradients.push_back(error.cwiseProduct(*layers[length-1].dZ));
- // deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
- // int counter = 1;
- // for (int i = length-2; i >= 1; i--) {
- // gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
- // deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
- // counter++;
- // }
- //printf("Beginning train on %i instances for %i epochs...\n", instances, total_epochs);
+ // float epsilon = 0.0001;
+ // Network copy = *this;
+ // std::vector<Eigen::MatrixXf> approx_gradients;
+ // for (int i = 0; i < copy.layers.size()-1; i++) {
+ // Eigen::MatrixXf current_approx = *copy.layers[i].weights;
+ // for (int j = 0; i < copy.layers[i].weights->rows(); i++) {
+ // for (int k = 0; i < copy.layers[i].weights->cols(); i++) {
+ // Network sim1 = copy;
+ // (*sim1.layers[i].contents)(j,k) += epsilon;
+ // sim1.feedforward();
+ // Network sim2 = copy;
+ // (*sim2.layers[i].contents)(j,k) -= epsilon;
+ // sim2.feedforward();
+ // current_approx(j,k) = (sim1.cost() - sim2.cost())/(2*epsilon);
+ // }
+ // }
+ // approx_gradients.push_back(current_approx);
+ // }
+ // for (Eigen::MatrixXf i : approx_gradients) {
+ // std::cout << i << "\n\n";
+ // }
+ // std::vector<Eigen::MatrixXf> gradients;
+ // std::vector<Eigen::MatrixXf> deltas;
+ // Eigen::MatrixXf error = ((*layers[length-1].contents) - (*labels));
+ // gradients.push_back(error.cwiseProduct(*layers[length-1].dZ));
+ // deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
+ // int counter = 1;
+ // for (int i = length-2; i >= 1; i--) {
+ // gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
+ // deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
+ // counter++;
+ // }
+ //printf("Beginning train on %i instances for %i epochs...\n", instances, total_epochs);
}
