commit 88ebc20ce6a58ff68c714917c9506cfbdca0e84a
parent 2a1d87c5b8419ac159b1c63de6417539d1e1957f
Author: David Freifeld <freifeld.david@gmail.com>
Date: Tue, 14 Jul 2020 09:04:04 -0700
Backprop is unstable + largely broken
Diffstat:
4 files changed, 50 insertions(+), 33 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -8,9 +8,9 @@ double bench(int batch_sz)
auto start = std::chrono::high_resolution_clock::now();
Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 0, 0.9);
net.add_layer(4, "linear");
- net.add_layer(5, "relu");
+ net.add_layer(5, "lecun_tanh");
net.add_layer(2, "linear");
- net.init_decay("step", 1, 2);
+ // net.init_decay("step", 1, 2);
net.initialize();
// checks(net);
// for (int i = 0; i < 10; i++) {
@@ -20,7 +20,7 @@ double bench(int batch_sz)
// net.backpropagate();
// std::cout << net.cost() << " " << net.accuracy() << "\n";
// }
- for (int i = 0; i < 25; i++) {
+ for (int i = 0; i < 75; i++) {
net.train();
// net.list_net();
}
@@ -33,6 +33,7 @@ double bench(int batch_sz)
int main()
{
+ // sleep(30);
std::cout << bench(16) << "\n";
// bench(50);
// bench(50);
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -7,33 +7,8 @@
#define TEST_PATH "./test.txt"
#define TRAIN_PATH "./train.txt"
-#define MAXLINE 1024
-#define ZERO_THRESHOLD pow(10, -8) // for checks
-
-#if (!RECKLESS)
-#define checknan(x, loc) if(x==INFINITY || x==NAN || x == -INFINITY) throw ValueError("Detected NaN in operation", loc)
-#else
-#define checknan(x, loc)
-#endif
-
#include "checks.cpp"
-struct ValueError : public std::exception
-{
- const char* message;
- const char* location;
- ValueError(const char* msg, const char* loc)
- :message{msg}, location{loc}
- {
- }
- const char* what() const throw () {
- char* error;
- sprintf(error, "%s (thrown in %s).", message, location);
- const char* error_message = error;
- return error_message;
- }
-};
-
Layer::Layer(int batch_sz, int nodes)
{
contents = new Eigen::MatrixXf (batch_sz, nodes);
@@ -224,8 +199,10 @@ float Network::cost()
if ((*layers[length-1].contents)(i,j) == 0) (*layers[length-1].contents)(i,j) += 0.00001;
// std::cout << truth << " VS " << (*layers[length-1].contents)(i,j) << " SO " << truth * log((*layers[length-1].contents)(i,j)) << "\n";
tempsum += truth * log((*layers[length-1].contents)(i,j));
+ checknan(tempsum, "summation for row inside cost calculation");
}
sum-=tempsum;
+ checknan(tempsum, "total summation inside cost calculation");
}
for (int i = 0; i < layers.size()-1; i++) {
reg += (layers[i].weights->cwiseProduct(*layers[i].weights)).sum();
@@ -258,12 +235,15 @@ void Network::backpropagate()
std::vector<Eigen::MatrixXf> deltas;
Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
// std::cout << (*layers[length-1].contents) << "\n\n\n";
+ // std::cout << "OUTPUT:\n" << (*layers[length-1].contents) << "\n\n";
+ // std::cout << "WEIGHT\n" << (*layers[length-2].weights) << "\n\n";
+ // std::cout << "X:\n" << (*layers[length-2].contents) << "\n\n";
for (int i = 0; i < error.rows(); i++) {
for (int j = 0; j < error.cols(); j++) {
float truth;
if (j==(*labels)(i,0)) truth = 1;
else truth = 0;
- error(i,j) = (*layers[length-1].contents)(i,j) - truth;
+ error(i,j) = truth - (*layers[length-1].contents)(i,j);
checknan(error(i,j), "gradient of final layer");
// std::cout << truth << "[as label is "<< (*labels)(i,0) <<"] - " << (*layers[length-1].contents)(i,j) << "[aka index " << i << " " << j << "] = " << error(i,j) << "\n";
}
@@ -279,6 +259,8 @@ void Network::backpropagate()
deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
counter++;
}
+ // std::cout << "GRAD:\n"<< gradients[0] << "\n\n";
+ // std::cout << "DELTA:\n"<< deltas[0] << "\n\n";
// std::cout << "-------\nGRADS INCOMING" << "\n\n";
// for (Eigen::MatrixXf i : gradients) {
// std::cout << i << "\n\n";
@@ -293,6 +275,7 @@ void Network::backpropagate()
//*layers[length-2-i].weights += *layers[length-2-i].v;
*layers[length-1-i].bias -= bias_lr * gradients[i];
}
+ // std::cout << "NEW WEIGHT:\n" << (*layers[length-2].weights) << "\n\n\n\n";
}
void Network::update_layer(float* vals, int datalen, int index)
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -84,4 +84,29 @@ void demo(int total_epochs);
int prep_file(char* path, char* out_path);
int split_file(char* path, int lines, float ratio);
+struct ValueError : public std::exception
+{
+ const char* message;
+ const char* location;
+ ValueError(const char* msg, const char* loc)
+ :message{msg}, location{loc}
+ {
+ }
+ const char* what() const throw () {
+ char* error;
+ sprintf(error, "%s (thrown in %s).", message, location);
+ const char* error_message = error;
+ return error_message;
+ }
+};
+
+#define MAXLINE 1024
+#define ZERO_THRESHOLD pow(10, -8) // for checks
+
+#if (!RECKLESS)
+#define checknan(x, loc) if(x==INFINITY || x==NAN || x == -INFINITY) throw ValueError("Detected NaN in operation", loc)
+#else
+#define checknan(x, loc)
+#endif
+
#endif /* MODULE_H */
diff --git a/src/cnn.cpp b/src/cnn.cpp
@@ -3,6 +3,12 @@
#define LARGE_NUM 1000000 // Remove me.
+#if (!RECKLESS)
+#define checknan(x, loc) if(x==INFINITY || x==NAN || x == -INFINITY) throw ValueError("Detected NaN in operation", loc)
+#else
+#define checknan(x, loc)
+#endif
+
int ReverseInt (int i)
{
unsigned char ch1, ch2, ch3, ch4;
@@ -279,6 +285,8 @@ void ConvNet::backpropagate()
if (j==(*labels)(i,0)) truth = 1;
else truth = 0;
error(i,j) = (*layers[length-1].contents)(i,j) - truth;
+ // std::cout << error(i, j) << " " << (*layers[length-1].contents)(i,j) << " " << truth << "\n";
+ checknan(error(i, j), "gradient of final layer");
// std::cout << truth << "[as label is "<< (*labels)(i,0) <<"] - " << (*layers[length-1].contents)(i,j) << "[aka index " << i << " " << j << "] = " << error(i,j) << "\n";
}
}
@@ -294,6 +302,7 @@ void ConvNet::backpropagate()
}
gradients.push_back((gradients[gradients.size()-1] * layers[0].weights->transpose()).cwiseProduct(*layers[0].dZ));
for (int i = 0; i < length-1; i++) {
+ std::cout << learning_rate << " (LR) \n" << deltas[i] << "\n\n";
*layers[length-2-i].weights -= learning_rate * deltas[i];
*layers[length-1-i].bias -= bias_lr * gradients[i];
}
@@ -317,7 +326,7 @@ void ConvNet::train()
{
float cost_sum = 0;
float acc_sum = 0;
- for (int i = 0; i <= 10; i++) {
+ for (int i = 0; i <= 1; i++) {
if (i != instances-batch_size) { // Don't try to advance batch on final batch.
next_batch();
}
@@ -338,7 +347,7 @@ void ConvNet::train()
int main()
{
- ConvNet net ("../data_banknote_authentication.txt", 0.05, 0.01, 0, 0.9);
+ ConvNet net ("../data_banknote_authentication.txt", 0.05, 0.01, 5, 0.9);
Eigen::MatrixXf labels (1,1);
labels << 2;
net.set_label(labels);
@@ -350,9 +359,8 @@ int main()
net.init_decay("step", 1, 2);
net.initialize();
- for (int i = 0; i < 50; i++) {
+ for (int i = 0; i < 1; i++) {
net.train();
}
- net.list_net();
std::cout << *net.layers[net.length-1].contents << "\n";
}
