commit f6a070985611b40ed2d55e31cf142e3909396a86
parent 129f8e605e178d2dfa8d42c69e10ad94e5b50474
Author: David Freifeld <freifeld.david@gmail.com>
Date: Tue, 14 Jul 2020 16:48:28 -0700
CNN doesn't break if you don't breathe on it too hard
Diffstat:
4 files changed, 60 insertions(+), 18 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -18,7 +18,7 @@ double bench(int batch_sz)
net.add_layer(4, "linear");
net.add_layer(6, "lecun_tanh");
net.add_layer(2, "linear");
- net.init_decay("step", 1, 2);
+ // net.init_decay("step", 1, 2);
net.initialize();
// checks(net);
// for (int i = 0; i < 10; i++) {
@@ -28,7 +28,7 @@ double bench(int batch_sz)
// net.backpropagate();
// std::cout << net.cost() << " " << net.accuracy() << "\n";
// }
- for (int i = 0; i < 500; i++) {
+ for (int i = 0; i < 1; i++) {
net.train();
// net.list_net();
}
diff --git a/scripts/benchmark.py b/scripts/benchmark.py
@@ -1,3 +1,11 @@
+#
+# benchmark.py
+# Jacobian
+#
+# Created by David Freifeld
+# Copyright © 2020 David Freifeld. All rights reserved.
+#
+
import mrbpnn
import matplotlib.pyplot as plt
import numpy
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -166,23 +166,30 @@ void Network::feedforward()
}
}
}
+ // std::cout << "\nSOFTMAX INPUT\n" << *layers[length-1].contents << "\n\n";
for (int i = 0; i < layers[length-1].contents->rows(); i++) {
float sum = 0;
- Eigen::MatrixXf m = layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols());
- Eigen::MatrixXf::Index maxRow, maxCol;
- float max = m.maxCoeff(&maxRow, &maxCol);
- m = (m.array() - max).matrix();
+ Eigen::MatrixXf m = *layers[length-1].contents;//->block(i,0,1,layers[length-1].contents->cols());
+ // Eigen::MatrixXf::Index maxRow, maxCol;
+ // float max = m.maxCoeff(&maxRow, &maxCol);
+ // m = (m.array() - max).matrix();
+ // std::cout << "\nGETTING SUM\n";
for (int j = 0; j < layers[length-1].contents->cols(); j++) {
checknan(m(0,j), "input to final layer");
sum += exp(m(0,j));
+ // std::cout << "Adding " << exp(m(0,j)) << "(aka e^"<< m(0, j) << ")\n";
checknan(sum, "sum in Softmax operation");
}
+ // std::cout << "\nFINAL ACTIVATION\n";
for (int j = 0; j < layers[length-1].contents->cols(); j++) {
- m(0,j) = exp(m(0,j))/sum;
+ (*layers[length-1].contents)(i,j) = exp(m(0,j))/sum;
+ float test = exp(m(0,j))/sum;
+ // std::cout << "Calculating " << exp(m(0,j)) << "/" << sum << " to be " << (*layers[length-1].contents)(i,j) << "(aka " << test<<")\n";
checknan(m(0,j), "output of Softmax operation");
}
layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols()) = m;
}
+ // std::cout << "\n\n";
}
void Network::list_net()
@@ -242,15 +249,26 @@ void Network::backpropagate()
std::vector<Eigen::MatrixXf> gradients;
std::vector<Eigen::MatrixXf> deltas;
Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
+ std::cout << "\nTRUTH:\n";
for (int i = 0; i < error.rows(); i++) {
for (int j = 0; j < error.cols(); j++) {
float truth;
if (j==(*labels)(i,0)) truth = 1;
else truth = 0;
+ std::cout << truth << " ";
error(i,j) = (*layers[length-1].contents)(i,j) - truth;
checknan(error(i,j), "gradient of final layer");
+ // std::cout << truth << "[as label is "<< (*labels)(i,0) <<"] - " << (*layers[length-1].contents)(i,j) << "[aka index " << i << " " << j << "] = " << error(i,j) << "\n";
}
- }
+ std::cout << "\n";
+ }
+ std::cout << "\n\n";
+ std::cout << "\nLABELS:\n";
+ std::cout << *labels << "\n\n";
+ std::cout << "\nPREDICTION:\n";
+ std::cout << (*layers[length-1].contents) << "\n\n";
+ std::cout << "\nERR:\n";
+ std::cout << error << "\n\n";
gradients.push_back(error);
deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
int counter = 1;
diff --git a/src/cnn.cpp b/src/cnn.cpp
@@ -285,15 +285,26 @@ void ConvNet::backpropagate()
std::vector<Eigen::MatrixXf> gradients;
std::vector<Eigen::MatrixXf> deltas;
Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
+ // std::cout << "\nTRUTH:\n";
for (int i = 0; i < error.rows(); i++) {
for (int j = 0; j < error.cols(); j++) {
float truth;
if (j==(*labels)(i,0)) truth = 1;
else truth = 0;
+ //std::cout << truth << " ";
error(i,j) = (*layers[length-1].contents)(i,j) - truth;
checknan(error(i, j), "gradient of final layer");
}
+ // std::cout << "\n";
}
+ // std::cout << "\n\n";
+ // std::cout << "\nLABELS:\n";
+ // std::cout << *labels << "\n\n";
+ // std::cout << "\nPREDICTION:\n";
+ // std::cout << (*layers[length-1].contents) << "\n\n";
+ // std::cout << "\nERR:\n";
+ // std::cout << error << "\n\n";
+ // std::cout << "\n\n\n------------------\n\n\n";
gradients.push_back(error);
deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
int counter = 1;
@@ -328,19 +339,22 @@ void ConvNet::train()
{
float cost_sum = 0;
float acc_sum = 0;
- for (int i = 0; i <= 10; i++) {
+ for (int i = 0; i <= 100; i++) {
if (i != instances-batch_size) { // Don't try to advance batch on final batch.
next_batch();
}
process();
feedforward();
+ // if (batches == 0) list_net();
backpropagate();
cost_sum += cost();
acc_sum += accuracy();
batches++;
+ // if (batches == 1) (1);
+ // if (batches > 15) exit(1);
}
- epoch_acc = 1.0/(10) * acc_sum;
- epoch_cost = 1.0/(10) * cost_sum;
+ epoch_acc = 1.0/(100) * acc_sum;
+ epoch_cost = 1.0/(100) * cost_sum;
printf("Epoch %i complete - cost %f - acc %f\n", epochs, epoch_cost, epoch_acc);
batches=0;
learning_rate = decay(learning_rate, epochs);
@@ -353,16 +367,18 @@ int main()
Eigen::MatrixXf labels (1,1);
labels << 2;
net.set_label(labels);
- net.add_conv_layer(28,28,1,4,4,0);
+ net.add_conv_layer(28,28,1,14,14,0);
+ net.add_conv_layer(14,14,1,7,7,0);
//net.add_pool_layer(5,5,1,2,0);
- net.add_layer(625, "linear");
- net.add_layer(5, "relu");
- net.add_layer(10, "linear");
+ net.add_layer(49, "resig");
+ net.add_layer(5, "lecun_tanh");
+ net.add_layer(10, "resig");
// net.init_decay("step", 1, 2);
+ net.list_net();
net.initialize();
- for (int i = 0; i < 10; i++) {
- net.train();
- }
+ // for (int i = 0; i < 50; i++) {
+ // net.train();
+ // }
std::cout << *net.layers[net.length-1].contents << "\n";
}
