commit 2781f8c7de154e2e102c1fa6c87e462113d12c9c
parent 35b202dde0c7e0c1b9291fcdda44537eec9eca24
Author: David Freifeld <freifeld.david@gmail.com>
Date: Mon, 13 Jul 2020 10:39:09 -0700
Working on more safety checks
Diffstat:
3 files changed, 27 insertions(+), 23 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -10,9 +10,9 @@ double bench(int batch_sz)
net.add_layer(4, "linear");
net.add_layer(5, "relu");
net.add_layer(2, "linear");
- // net.init_decay("step", 1, 2);
+ net.init_decay("step", 1, 2);
net.initialize();
- // checks(net);
+ // checks(net);
// for (int i = 0; i < 10; i++) {
// net.next_batch();
// net.feedforward();
@@ -20,12 +20,12 @@ double bench(int batch_sz)
// net.backpropagate();
// std::cout << net.cost() << " " << net.accuracy() << "\n";
// }
- for (int i = 0; i < 50; i++) {
+ for (int i = 0; i < 25; i++) {
net.train();
// net.list_net();
}
- std::cout << *net.layers[net.length-1].contents << "\n\n";
- std::cout << *net.labels << "\n";
+ // std::cout << *net.layers[net.length-1].contents << "\n\n";
+ // std::cout << *net.labels << "\n";
auto end = std::chrono::high_resolution_clock::now();
//net.list_net();
return std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count() / pow(10,9);
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -10,7 +10,13 @@
#define MAXLINE 1024
#define ZERO_THRESHOLD pow(10, -8) // for checks
+#if (!RECKLESS)
#define checknan(x, loc) if(x==INFINITY || x==NAN || x == -INFINITY) throw ValueError("Detected NaN in operation", loc)
+#else
+#define checknan(x, loc)
+#endif
+
+#include "checks.cpp"
struct ValueError : public std::exception
{
@@ -177,31 +183,20 @@ void Network::feedforward()
}
}
}
- //std::cout << (*layers[length-1].contents) << "\n";
for (int i = 0; i < layers[length-1].contents->rows(); i++) {
float sum = 0;
Eigen::MatrixXf m = layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols());
Eigen::MatrixXf::Index maxRow, maxCol;
float max = m.maxCoeff(&maxRow, &maxCol);
- // std::cout << m << "(before with max "<< max <<")\n";
m = (m.array() - max).matrix();
- // std::cout << m << "(after with max "<< max <<")\n";
for (int j = 0; j < layers[length-1].contents->cols(); j++) {
-#ifndef RECKLESS
checknan(m(0,j), "input to final layer");
-#endif
- // if(m(0,j)==INFINITY || m(0,j)==NAN || m(0,j)== -INFINITY) throw ValueError("Detected NaN or inf value in layer", "input to final layer");
sum += exp(m(0,j));
-#ifndef RECKLESS
checknan(sum, "sum in Softmax operation");
-#endif
}
for (int j = 0; j < layers[length-1].contents->cols(); j++) {
- // std::cout << "(e^" << m(0,j) << ")/" << sum << " -> " << exp(m(0,j)) << "/" << sum << " -> " << exp((m(0,j)))/sum << "\n";
m(0,j) = exp(m(0,j))/sum;
-#ifndef RECKLESS
checknan(m(0,j), "output of Softmax operation");
-#endif
}
layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols()) = m;
}
@@ -213,7 +208,7 @@ void Network::list_net()
for (int i = 1; i < length-1; i++) {
std::cout << "-----------------------\nLAYER " << i << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[i].activation_str << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[i].contents << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[i].bias << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << *layers[i].weights << "\n\n\n";
}
- std::cout << "-----------------------\nOUTPUT LAYER (LAYER " << length-1 << ")\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[length-1].activation_str <<"\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[length-1].contents << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[length-1].bias << "\n\n\n";
+ std::cout << "-----------------------\nOUTPUT LAYER (LAYER " << length-1 << ")\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[length-1].activation_str <<"\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[length-1].contents << "\n\n\u001b[31BIASES:\x1B[0;37m\n" << *layers[length-1].bias << "\n\n\n";
}
float Network::cost()
@@ -269,6 +264,7 @@ void Network::backpropagate()
if (j==(*labels)(i,0)) truth = 1;
else truth = 0;
error(i,j) = (*layers[length-1].contents)(i,j) - truth;
+ checknan(error(i,j), "gradient of final layer");
// std::cout << truth << "[as label is "<< (*labels)(i,0) <<"] - " << (*layers[length-1].contents)(i,j) << "[aka index " << i << " " << j << "] = " << error(i,j) << "\n";
}
}
@@ -405,15 +401,13 @@ float Network::test(char* path)
return 0;
}
-#include "checks.cpp"
-
void Network::train()
{
rewind(data);
float cost_sum = 0;
float acc_sum = 0;
for (int i = 0; i <= instances-batch_size; i+=batch_size) {
- [[unlikely]] if (i != instances-batch_size) { // Don't try to advance batch on final batch.
+ if (i != instances-batch_size) { // Don't try to advance batch on final batch.
next_batch();
}
feedforward();
diff --git a/src/checks.cpp b/src/checks.cpp
@@ -1,5 +1,6 @@
void checks(Network net)
{
+ Network original = net;
int sanity_passed = 0;
std::cout << "\u001b[4m\u001b[1mSANITY CHECKS:\u001b[0m\n";
// Check if regularization strength increases loss (as it should).
@@ -20,7 +21,7 @@ void checks(Network net)
}
else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- // list_net();
+ // net.list_net();
// Check if zero cost is achievable on a batch
std::cout << "Zero-cost sanity check...";
@@ -121,10 +122,18 @@ void checks(Network net)
sanity_passed++;
}
else std::cout << " \u001b[31mFailed.\n\u001b[37m";
+
+ // std::cout << "Side effects sanity check...";
- std::cout << "\u001b[1m\nPassed " << sanity_passed << "/5" <<" sanity checks.\u001b[0m\n\n\n";
+ // if (net == original) {
+ // std::cout << " \u001b[32mPassed!\n\u001b[37m";
+ // sanity_passed++;
+ // }
+ // else std::cout << " \u001b[31mFailed.\n\u001b[37m";
- net.list_net();
+ std::cout << "\u001b[1m\nPassed " << sanity_passed << "/6" <<" sanity checks.\u001b[0m\n\n\n";
+
+ // net.list_net();
// float epsilon = 0.0001;
// Network copy = *this;
@@ -159,4 +168,5 @@ void checks(Network net)
// counter++;
// }
//printf("Beginning train on %i instances for %i epochs...\n", instances, total_epochs);
+
}
