commit 0a30e1a91c485010104263335f5f206467068b9c
parent aed7570cf32653045dc53367f01a114333c20e61
Author: David Freifeld <freifeld.david@gmail.com>
Date: Tue, 11 Aug 2020 15:04:50 -0700
Basic ability to call checks
Diffstat:
5 files changed, 188 insertions(+), 175 deletions(-)
diff --git a/checks.cpp b/checks.cpp
@@ -0,0 +1,172 @@
+//
+// checks.cpp
+// Jacobian
+//
+// Created by David Freifeld
+//
+
+#include "./src/bpnn.hpp"
+
+#define ZERO_THRESHOLD pow(10, -5)
+
+Network default_net()
+{
+ Network net ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
+ net.add_layer(4, "linear");
+ net.add_layer(5, "lecun_tanh");
+ net.add_layer(2, "linear");
+ net.initialize();
+ return net;
+}
+
+Network explicit_copy(Network src)
+{
+ Network dst ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
+ dst = src;
+ for (int i = 0; i < src.layers.size(); i++) {
+ dst.layers[i] = src.layers[i];
+ }
+ return dst;
+}
+
+void regularization_check(int& sanity_passed, int& total_checks)
+{
+ Network net = default_net();
+ std::cout << "\u001b[4m\u001b[1mSANITY CHECKS:\u001b[0m\n";
+ // Check if regularization strength increases loss (as it should).
+ std::cout << "Regularization sanity check...";
+
+ net.list_net();
+ Network copy1 = explicit_copy(net);
+ Network copy2 = explicit_copy(net);
+ copy1.next_batch();
+ copy1.feedforward();
+
+ copy2.next_batch();
+ copy2.feedforward();
+ net.list_net();
+ if (copy1.cost() > copy2.cost()) {
+ std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ sanity_passed++;
+ }
+ else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ total_checks++;
+}
+
+void zero_check(int& sanity_passed, int& total_checks)
+{
+ Network net = default_net();
+ std::cout << "Zero-cost sanity check...";
+ net.next_batch();
+ float finalcost;
+ for (int i = 0; i < 100000; i++) {
+ net.feedforward();
+ net.backpropagate();
+ finalcost = net.cost();
+ if (finalcost <= ZERO_THRESHOLD) {
+ break;
+ }
+ }
+ if (finalcost <= ZERO_THRESHOLD) {
+ std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ sanity_passed++;
+ }
+ else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ total_checks++;
+}
+
+void floating_point_check(int& sanity_passed, int& total_checks)
+{
+ Network net = default_net();
+ std::cout << "Gradient floating-point sanity check...";
+ net.next_batch();
+ net.feedforward();
+ std::vector<Eigen::MatrixXf> gradients;
+ std::vector<Eigen::MatrixXf> deltas;
+ Eigen::MatrixXf error = ((*net.layers[net.length-1].contents) - (*net.labels));
+ gradients.push_back(error.cwiseProduct(*net.layers[net.length-1].dZ));
+ deltas.push_back((*net.layers[net.length-2].contents).transpose() * gradients[0]);
+ int counter = 1;
+ for (int i = net.length-2; i >= 1; i--) {
+ gradients.push_back((gradients[counter-1] * net.layers[i].weights->transpose()).cwiseProduct(*net.layers[i].dZ));
+ deltas.push_back(net.layers[i-1].contents->transpose() * gradients[counter]);
+ counter++;
+ }
+ auto check_gradients = [](std::vector<Eigen::MatrixXf> vec) -> bool {
+ for (Eigen::MatrixXf i : vec) {
+ for (int j = 0; j < i.rows(); j++) {
+ for (int k = 0; k < i.cols(); k++) {
+ if (i(j,k) == -0 || i(j,k) == INFINITY || i(j,k) == NAN || i(j,k) == -INFINITY) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ };
+ if (check_gradients(gradients) == false && check_gradients(deltas) == false) {
+ std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ sanity_passed++;
+ }
+ else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ total_checks++;
+}
+
+void expected_loss_check(int& sanity_passed, int& total_checks)
+{
+ Network net = default_net();
+ std::cout << "Expected loss sanity check...";
+ net.next_batch();
+ net.feedforward();
+ if (net.cost() <= 1) {
+ std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ sanity_passed++;
+ }
+ else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ total_checks++;
+}
+
+void update_check(int& sanity_passed, int& total_checks)
+{
+ // std::cout << "Layer updates sanity check...";
+ // Network copy6 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ // Network copy7 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ // //copy2.list_net();
+ // //copy1.list_net();
+ // int passed;
+ // for (int i = 0; i < copy1.layers.size()-1; i++) {
+ // if (*copy1.layers[i].weights == *copy2.layers[i].weights) {
+ // // std::cout << *copy2.layers[i].weights <<"uninitweight\n\n";
+ // // std::cout << *copy1.layers[i].weights << " "<<i<<"weight\n\n";
+ // passed = -1;
+ // }
+ // }
+ // for (int i = 1; i < copy1.layers.size(); i++) {
+ // if (*copy1.layers[i].bias == *copy2.layers[i].bias) {
+ // // std::cout << *copy2.layers[i].bias <<"uninitbias\n\n";
+ // // std::cout << *copy1.layers[i].bias <<" " << i << "bias\n\n";
+ // passed = -1;
+ // }
+ // }
+ // if (passed == 1) {
+ // std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ // sanity_passed++;
+ // }
+ // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+}
+
+void sanity_checks()
+{
+ int sanity_passed = 0;
+ int total_checks = 0;
+ zero_check(sanity_passed, total_checks);
+ std::cout << "\u001b[1m\nPassed " << sanity_passed << "/" << total_checks <<" sanity checks.\u001b[0m\n";
+ if ((float)sanity_passed/total_checks < 0.5) {
+ std::cout << "Majority of sanity checks failed. Exiting." << "\n";
+ exit(1);
+ }
+}
+
+void grad_checks()
+{
+}
diff --git a/example.cpp b/example.cpp
@@ -12,6 +12,7 @@
#include "./src/bpnn.hpp"
#include "./src/utils.hpp"
+#include "./checks.cpp"
#include "unistd.h"
#include <ctime>
@@ -23,7 +24,6 @@ double bench(int batch_sz, int epochs)
net.add_layer(5, "lecun_tanh");
net.add_layer(2, "linear");
net.initialize();
- checks();
for (int i = 0; i < epochs; i++) {
net.train();
}
@@ -33,9 +33,21 @@ double bench(int batch_sz, int epochs)
int main(int argc, char** argv)
{
- if (argc < 3) {
- std::cout << "Invalid command! Pass two integers - batch_size and epochs (in that order)." << "\n";
+ if (argc < 2) {
+ std::cout << "Invalid command! Either pass a special option or pass two integers - batch_size and epochs (in that order)." << "\n";
exit(1);
}
- std::cout << bench(strtol(argv[1], NULL, 10), strtol(argv[2], NULL, 10)) << "\n";
+ else if (strcmp(argv[1], "sanity-checks") == 0) {
+ sanity_checks();
+ }
+ else if (strcmp(argv[1], "grad-checks") == 0) {
+ grad_checks();
+ }
+ else if (argc < 3) {
+ std::cout << "Invalid command! Either pass a special option or pass two integers - batch_size and epochs (in that order)." << "\n";
+ exit(1);
+ }
+ else {
+ std::cout << bench(strtol(argv[1], NULL, 10), strtol(argv[2], NULL, 10)) << "\n";
+ }
}
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -20,8 +20,6 @@
#define cwise_product(a,b) (a).cwiseProduct(b)
#endif
-#include "checks.cpp"
-
Layer::Layer(int batch_sz, int nodes, float a)
:alpha(a)
{
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -97,8 +97,6 @@ public:
float get_val_cost() {return val_cost;}
};
-void checks();
-void demo(int total_epochs);
int prep_file(char* path, char* out_path);
int split_file(char* path, int lines, float ratio);
@@ -119,7 +117,6 @@ struct ValueError : public std::exception
};
#define MAXLINE 1024
-#define ZERO_THRESHOLD pow(10, -5) // for checks
#if (!RECKLESS)
#define checknan(x, loc) if(x==INFINITY || x==NAN || x == -INFINITY) throw ValueError("Detected NaN in operation", loc)
diff --git a/src/checks.cpp b/src/checks.cpp
@@ -1,166 +0,0 @@
-//
-// checks.cpp
-// Jacobian
-//
-// Created by David Freifeld
-//
-
-Network explicit_copy(Network src)
-{
- Network dst ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
- dst = src;
- for (int i = 0; i < src.layers.size(); i++) {
- dst.layers[i] = src.layers[i];
- }
- return dst;
-}
-
-void regularization_check(int& sanity_passed, int& total_checks)
-{
- Network net ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
- net.add_layer(4, "linear");
- net.add_layer(5, "lecun_tanh");
- net.add_layer(2, "linear");
- net.initialize();
- std::cout << "\u001b[4m\u001b[1mSANITY CHECKS:\u001b[0m\n";
- // Check if regularization strength increases loss (as it should).
- std::cout << "Regularization sanity check...";
-
- net.list_net();
- Network copy1 = explicit_copy(net);
- Network copy2 = explicit_copy(net);
- copy1.next_batch();
- copy1.feedforward();
-
- copy2.next_batch();
- copy2.feedforward();
- net.list_net();
- if (copy1.cost() > copy2.cost()) {
- std::cout << " \u001b[32mPassed!\n\u001b[37m";
- sanity_passed++;
- }
- else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- total_checks++;
-}
-
-void zero_check(int& sanity_passed, int& total_checks)
-{
- Network net ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
- net.add_layer(4, "linear");
- net.add_layer(5, "lecun_tanh");
- net.add_layer(2, "linear");
- net.initialize();
- std::cout << "Zero-cost sanity check...";
- net.next_batch();
- float finalcost;
- for (int i = 0; i < 100000; i++) {
- net.feedforward();
- net.backpropagate();
- finalcost = net.cost();
- if (finalcost <= ZERO_THRESHOLD) {
- break;
- }
- }
- if (finalcost <= ZERO_THRESHOLD) {
- std::cout << " \u001b[32mPassed!\n\u001b[37m";
- sanity_passed++;
- }
- else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- total_checks++;
-}
-
-void floating_point_check(int& sanity_passed, int& total_checks)
-{
- Network net ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
- net.add_layer(4, "linear");
- net.add_layer(5, "lecun_tanh");
- net.add_layer(2, "linear");
- net.initialize();
- std::cout << "Gradient floating-point sanity check...";
- net.next_batch();
- net.feedforward();
- std::vector<Eigen::MatrixXf> gradients;
- std::vector<Eigen::MatrixXf> deltas;
- Eigen::MatrixXf error = ((*net.layers[net.length-1].contents) - (*net.labels));
- gradients.push_back(error.cwiseProduct(*net.layers[net.length-1].dZ));
- deltas.push_back((*net.layers[net.length-2].contents).transpose() * gradients[0]);
- int counter = 1;
- for (int i = net.length-2; i >= 1; i--) {
- gradients.push_back((gradients[counter-1] * net.layers[i].weights->transpose()).cwiseProduct(*net.layers[i].dZ));
- deltas.push_back(net.layers[i-1].contents->transpose() * gradients[counter]);
- counter++;
- }
- auto check_gradients = [](std::vector<Eigen::MatrixXf> vec) -> bool {
- for (Eigen::MatrixXf i : vec) {
- for (int j = 0; j < i.rows(); j++) {
- for (int k = 0; k < i.cols(); k++) {
- if (i(j,k) == -0 || i(j,k) == INFINITY || i(j,k) == NAN || i(j,k) == -INFINITY) {
- return true;
- }
- }
- }
- }
- return false;
- };
- if (check_gradients(gradients) == false && check_gradients(deltas) == false) {
- std::cout << " \u001b[32mPassed!\n\u001b[37m";
- sanity_passed++;
- }
- else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- total_checks++;
-}
-
-void expected_loss_check(int& sanity_passed, int& total_checks)
-{
- Network net ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
- net.add_layer(4, "linear");
- net.add_layer(5, "lecun_tanh");
- net.add_layer(2, "linear");
- net.initialize();
- std::cout << "Expected loss sanity check...";
- net.next_batch();
- net.feedforward();
- if (net.cost() <= 1) {
- std::cout << " \u001b[32mPassed!\n\u001b[37m";
- sanity_passed++;
- }
- else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- total_checks++;
-}
-
-void update_check(int& sanity_passed, int& total_checks)
-{
- // std::cout << "Layer updates sanity check...";
- // Network copy6 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- // Network copy7 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
- // //copy2.list_net();
- // //copy1.list_net();
- // int passed;
- // for (int i = 0; i < copy1.layers.size()-1; i++) {
- // if (*copy1.layers[i].weights == *copy2.layers[i].weights) {
- // // std::cout << *copy2.layers[i].weights <<"uninitweight\n\n";
- // // std::cout << *copy1.layers[i].weights << " "<<i<<"weight\n\n";
- // passed = -1;
- // }
- // }
- // for (int i = 1; i < copy1.layers.size(); i++) {
- // if (*copy1.layers[i].bias == *copy2.layers[i].bias) {
- // // std::cout << *copy2.layers[i].bias <<"uninitbias\n\n";
- // // std::cout << *copy1.layers[i].bias <<" " << i << "bias\n\n";
- // passed = -1;
- // }
- // }
- // if (passed == 1) {
- // std::cout << " \u001b[32mPassed!\n\u001b[37m";
- // sanity_passed++;
- // }
- // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
-}
-
-void checks()
-{
- int sanity_passed = 0;
- int total_checks = 0;
- zero_check(sanity_passed, total_checks);
- std::cout << "\u001b[1m\nPassed " << sanity_passed << "/" << total_checks <<" sanity checks.\u001b[0m\n\n\n";
-}
