commit d90e46849e0008c944deec8d019f352344263a96
parent 0a30e1a91c485010104263335f5f206467068b9c
Author: David Freifeld <freifeld.david@gmail.com>
Date: Tue, 11 Aug 2020 15:57:14 -0700
Added basic checks + custom checks added to workflow
Diffstat:
6 files changed, 130 insertions(+), 51 deletions(-)
diff --git a/.github/workflows/macos-latest.yml b/.github/workflows/macos-latest.yml
@@ -27,7 +27,10 @@ jobs:
run: cmake . -DCXX=ON
- name: make-build
run: make
- - name: ctest
- run: ctest
-
+ - name: basic-checks
+ run: ./jacobian_cli basic-checks
+ - name: sanity-checks
+ run: ./jacobian_cli sanity-checks
+ - name: grad-checks
+ run: ./jacobian_cli grad-checks
diff --git a/.github/workflows/ubuntu-latest.yml b/.github/workflows/ubuntu-latest.yml
@@ -27,7 +27,10 @@ jobs:
run: cmake . -DCXX=ON
- name: make-build
run: make
- - name: ctest
- run: ctest
-
+ - name: basic-checks
+ run: ./jacobian_cli basic-checks
+ - name: sanity-checks
+ run: ./jacobian_cli sanity-checks
+ - name: grad-checks
+ run: ./jacobian_cli grad-checks
diff --git a/checks.cpp b/checks.cpp
@@ -7,18 +7,22 @@
#include "./src/bpnn.hpp"
-#define ZERO_THRESHOLD pow(10, -5)
+#define ZERO_THRESHOLD 5*pow(10, -5)
+// Simple example network to be used in each check.
Network default_net()
{
Network net ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
net.add_layer(4, "linear");
net.add_layer(5, "lecun_tanh");
net.add_layer(2, "linear");
+ net.init_optimizer("momentum", 0);
net.initialize();
+ net.silenced = true;
return net;
}
+// Proper cloning of networks for providing a reference point.
Network explicit_copy(Network src)
{
Network dst ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
@@ -29,12 +33,13 @@ Network explicit_copy(Network src)
return dst;
}
+// Regularization should increase the cost.
void regularization_check(int& sanity_passed, int& total_checks)
{
Network net = default_net();
std::cout << "\u001b[4m\u001b[1mSANITY CHECKS:\u001b[0m\n";
// Check if regularization strength increases loss (as it should).
- std::cout << "Regularization sanity check...";
+ std::cout << "Regularization check...";
net.list_net();
Network copy1 = explicit_copy(net);
@@ -53,13 +58,14 @@ void regularization_check(int& sanity_passed, int& total_checks)
total_checks++;
}
+// Given a small batch size and enough time, the network should be able to get its cost very close to zero.
void zero_check(int& sanity_passed, int& total_checks)
{
Network net = default_net();
- std::cout << "Zero-cost sanity check...";
+ std::cout << "Zero-cost check...";
net.next_batch();
float finalcost;
- for (int i = 0; i < 100000; i++) {
+ for (int i = 0; i < 10000; i++) {
net.feedforward();
net.backpropagate();
finalcost = net.cost();
@@ -75,54 +81,27 @@ void zero_check(int& sanity_passed, int& total_checks)
total_checks++;
}
+// There should be no weird floating point numbers in layer updates.
void floating_point_check(int& sanity_passed, int& total_checks)
{
Network net = default_net();
- std::cout << "Gradient floating-point sanity check...";
+ std::cout << "Update floating-point check...";
net.next_batch();
net.feedforward();
- std::vector<Eigen::MatrixXf> gradients;
- std::vector<Eigen::MatrixXf> deltas;
- Eigen::MatrixXf error = ((*net.layers[net.length-1].contents) - (*net.labels));
- gradients.push_back(error.cwiseProduct(*net.layers[net.length-1].dZ));
- deltas.push_back((*net.layers[net.length-2].contents).transpose() * gradients[0]);
- int counter = 1;
- for (int i = net.length-2; i >= 1; i--) {
- gradients.push_back((gradients[counter-1] * net.layers[i].weights->transpose()).cwiseProduct(*net.layers[i].dZ));
- deltas.push_back(net.layers[i-1].contents->transpose() * gradients[counter]);
- counter++;
- }
- auto check_gradients = [](std::vector<Eigen::MatrixXf> vec) -> bool {
- for (Eigen::MatrixXf i : vec) {
- for (int j = 0; j < i.rows(); j++) {
- for (int k = 0; k < i.cols(); k++) {
- if (i(j,k) == -0 || i(j,k) == INFINITY || i(j,k) == NAN || i(j,k) == -INFINITY) {
- return true;
+ net.backpropagate();
+ for (int i = 0; i < net.length-1; i++) {
+ for (int j = 0; j < net.layers[i].m->rows(); j++) {
+ for (int k = 0; k < net.layers[i].m->cols(); k++) {
+ if ((*net.layers[i].m)(j,k) == -0 || (*net.layers[i].m)(j,k) == INFINITY || (*net.layers[i].m)(j,k) == NAN || (*net.layers[i].m)(j,k) == -INFINITY) {
+ std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ total_checks++;
+ return;
+ }
}
- }
}
- }
- return false;
- };
- if (check_gradients(gradients) == false && check_gradients(deltas) == false) {
- std::cout << " \u001b[32mPassed!\n\u001b[37m";
- sanity_passed++;
- }
- else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- total_checks++;
-}
-
-void expected_loss_check(int& sanity_passed, int& total_checks)
-{
- Network net = default_net();
- std::cout << "Expected loss sanity check...";
- net.next_batch();
- net.feedforward();
- if (net.cost() <= 1) {
- std::cout << " \u001b[32mPassed!\n\u001b[37m";
- sanity_passed++;
}
- else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ sanity_passed++;
total_checks++;
}
@@ -160,6 +139,7 @@ void sanity_checks()
int sanity_passed = 0;
int total_checks = 0;
zero_check(sanity_passed, total_checks);
+ floating_point_check(sanity_passed, total_checks);
std::cout << "\u001b[1m\nPassed " << sanity_passed << "/" << total_checks <<" sanity checks.\u001b[0m\n";
if ((float)sanity_passed/total_checks < 0.5) {
std::cout << "Majority of sanity checks failed. Exiting." << "\n";
@@ -167,6 +147,95 @@ void sanity_checks()
}
}
+void run_check(int& basic_passed, int& total_checks)
+{
+ std::cout << "Default net check...";
+ try {
+ Network net = default_net();
+ for (int i = 0; i < 50; i++) {
+ net.train();
+ }
+ }
+ catch (...) {
+ std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ total_checks++;
+ return;
+ }
+ std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ basic_passed++;
+ total_checks++;
+}
+
+void optimizers_check(int& basic_passed, int& total_checks)
+{
+ std::cout << "Optimizers check...";
+ try {
+ std::string optimizers [5] = {"momentum", "demon", "adam", "adamax", "sgd"};
+ for (std::string optimizer : optimizers) {
+ Network net ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
+ net.add_layer(4, "linear");
+ net.add_layer(5, "lecun_tanh");
+ net.add_layer(2, "linear");
+ if (optimizer == "momentum") net.init_optimizer("momentum", 0.9);
+ if (optimizer == "momentum") net.init_optimizer("demon", 0.9, 50);
+ if (optimizer == "momentum") net.init_optimizer("adam", 0.999, 0.9, pow(10,-6));
+ if (optimizer == "momentum") net.init_optimizer("adamax", 0.999, 0.9, pow(10,-6));
+ if (optimizer == "momentum") net.init_optimizer("sgd");
+ net.initialize();
+ net.silenced=true;
+ for (int i = 0; i < 50; i++) {
+ net.train();
+ }
+ }
+ }
+ catch (...) {
+ std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ total_checks++;
+ return;
+ }
+ std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ basic_passed++;
+ total_checks++;
+}
+
+void prelu_check(int& basic_passed, int& total_checks)
+{
+ std::cout << "PReLU check...";
+ try {
+ Network net = default_net();
+ for (int i = 0; i < 50; i++) {
+ Network net ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
+ net.add_layer(4, "linear");
+ net.add_prelu_layer(5, 0.01);
+ net.add_layer(2, "linear");
+ net.initialize();
+ net.silenced=true;
+ }
+ }
+ catch (...) {
+ std::cout << " \u001b[31mFailed.\n\u001b[37m";
+ total_checks++;
+ return;
+ }
+ std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ basic_passed++;
+ total_checks++;
+}
+
+void basic_checks()
+{
+ int basic_passed = 0;
+ int total_checks = 0;
+ run_check(basic_passed, total_checks);
+ optimizers_check(basic_passed, total_checks);
+ prelu_check(basic_passed, total_checks);
+ std::cout << "\u001b[1m\nPassed " << basic_passed << "/" << total_checks <<" basic checks.\u001b[0m\n";
+ if ((float)basic_passed/total_checks < 0.5) {
+ std::cout << "Majority of basic checks failed. Exiting." << "\n";
+ exit(1);
+ }
+}
+
void grad_checks()
{
}
diff --git a/example.cpp b/example.cpp
@@ -37,6 +37,9 @@ int main(int argc, char** argv)
std::cout << "Invalid command! Either pass a special option or pass two integers - batch_size and epochs (in that order)." << "\n";
exit(1);
}
+ else if (strcmp(argv[1], "basic-checks") == 0) {
+ basic_checks();
+ }
else if (strcmp(argv[1], "sanity-checks") == 0) {
sanity_checks();
}
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -554,7 +554,7 @@ void Network::train()
epoch_acc = 1.0/((float) instances/batch_size) * acc_sum;
epoch_cost = 1.0/((float) instances/batch_size) * cost_sum;
validate(VAL_PATH);
- printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n", epochs, epoch_cost, epoch_acc, val_cost, val_acc);
+ if (silenced == false) printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n", epochs, epoch_cost, epoch_acc, val_cost, val_acc);
batches=1;
rewind(data);
decay();
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -60,6 +60,7 @@ public:
int reg_type;
int batch_size;
+ bool silenced = false;
int epochs = 0;
int batches = 0;
Eigen::MatrixXf* labels;
