commit ad4e0624fca9fcf5c74a9b2677ff362933133375
parent 131f4d117056a32c8c6ad5c898361262a119cee0
Author: David Freifeld <freifeld.david@gmail.com>
Date: Sun, 29 Nov 2020 10:21:30 -0800
Continued struggles with exploding gradients
Diffstat:
| M | src/cnn.cpp | | | 231 | +++++++++++++++++++++++++++++++++++++++++-------------------------------------- |
| M | src/cnn.hpp | | | 71 | +++++++++++++++++++++++++++++++++++++---------------------------------- |
2 files changed, 156 insertions(+), 146 deletions(-)
diff --git a/src/cnn.cpp b/src/cnn.cpp
@@ -93,13 +93,15 @@ unsigned char* read_mnist_labels(std::string full_path, int number_of_labels) {
}
}
-ConvLayer::ConvLayer(int x, int y, int stride, int kern_x, int kern_y, int pad)
- :padding(pad), stride_len(stride)
+ConvLayer::ConvLayer(int x, int y, int stride, int kern_x, int kern_y, int pad, std::function<float(float)> activ, std::function<float(float)> activ_deriv)
+ :padding(pad), stride_len(stride), activation(activ), activation_deriv(activ_deriv)
{
pad*=2;
input = new Eigen::MatrixXf (x+pad,y+pad);
+ dZ = new Eigen::MatrixXf (x+pad,y+pad);
for (int i = 0; i < (x+pad)*(y+pad); i++) {
(*input)((int)i / (y+pad),i%(y+pad)) = 0;
+ (*dZ)((int)i / (y+pad),i%(y+pad)) = 0;
}
kernel = new Eigen::MatrixXf (kern_x, kern_y);
for (int i = 0; i < kern_x*kern_y; i++) {
@@ -114,126 +116,131 @@ ConvLayer::ConvLayer(int x, int y, int stride, int kern_x, int kern_y, int pad)
void ConvLayer::convolute()
{
- for (int i = 0; i < output->cols(); i+=stride_len) {
- for (int j = 0; j < output->rows(); j+=stride_len) {
- // std::cout << j << ","<< i<< " vs "<< output->rows() << "," << output->cols() << "\n";
- (*output)(j, i) = (*kernel * (input->block(j, i, kernel->rows(), kernel->cols()))).sum();
+ for (int i = 0; i < (*input).rows(); i++) {
+ for (int j = 0; j < (*input).cols(); j++) {
+ (*dZ)(i, j) = activation_deriv((*input)(i, j));
+ (*input)(i, j) = activation((*input)(i, j));
+ }
+ }
+ for (int i = 0; i < output->rows(); i+=stride_len) {
+ for (int j = 0; j < output->cols(); j+=stride_len) {
+ (*output)(i, j) = (*kernel * (input->block(i, j, kernel->rows(), kernel->cols()))).sum();
+ }
}
- }
- *output = (output->array() + bias).matrix();
+ *output = (output->array() + bias).matrix();
}
void ConvLayer::set_input(Eigen::MatrixXf* matrix)
{
- input->block(padding, padding, matrix->rows(), matrix->cols()) = *matrix;
+ input->block(padding, padding, matrix->rows(), matrix->cols()) = *matrix;
}
// Will eventually be different from ConvLayer
PoolingLayer::PoolingLayer(int x, int y, int stride, int kern_x, int kern_y, int pad)
:padding(pad), stride_len(stride)
{
- input = new Eigen::MatrixXf (x+pad,y+pad);
- for (int i = 0; i < (x+pad)*(y+pad); i++) {
- (*input)((int)i / (y+pad),i%(y+pad)) = 0;
- }
- kernel = new Eigen::MatrixXf (kern_x, kern_y);
- for (int i = 0; i < kern_x*kern_y; i++) {
- (*kernel)((int)i / kern_y,i%kern_y) = (float) rand()/RAND_MAX;
- }
- output = new Eigen::MatrixXf (x-kern_x+1, y-kern_y+1);
- for (int i = 0; i < (x-kern_x+1)*(y-kern_y+1); i++) {
- (*output)((int)i / (y-kern_y+1),i%(y-kern_y+1)) = (float) rand()/RAND_MAX;
- }
+ input = new Eigen::MatrixXf (x+pad,y+pad);
+ for (int i = 0; i < (x+pad)*(y+pad); i++) {
+ (*input)((int)i / (y+pad),i%(y+pad)) = 0;
+ }
+ kernel = new Eigen::MatrixXf (kern_x, kern_y);
+ for (int i = 0; i < kern_x*kern_y; i++) {
+ (*kernel)((int)i / kern_y,i%kern_y) = (float) rand()/RAND_MAX;
+ }
+ output = new Eigen::MatrixXf (x-kern_x+1, y-kern_y+1);
+ for (int i = 0; i < (x-kern_x+1)*(y-kern_y+1); i++) {
+ (*output)((int)i / (y-kern_y+1),i%(y-kern_y+1)) = (float) rand()/RAND_MAX;
+ }
};
void PoolingLayer::pool()
{
// It doesn't look like anything better than O(n^4) is doable for this as kernel needs to go through matrix and you need to index kernel. LOOK INTO ME!!
- float maxnum = -LARGE_NUM;
- for (int i = 0; i < input->cols() - kernel->cols(); i+=stride_len) {
- for (int j = 0; j < input->rows() - kernel->rows(); j+=stride_len) {
- for (int k = 0; k < kernel->cols(); k++) {
- for (int l = 0; l < kernel->rows(); l++) {
- if ((input->block(j, i, kernel->rows(), kernel->cols()))(l, k) > maxnum) {
- maxnum = (input->block(j, i, kernel->rows(), kernel->cols()))(l, k);
- }
+ float maxnum = -LARGE_NUM;
+ for (int i = 0; i < input->cols() - kernel->cols(); i+=stride_len) {
+ for (int j = 0; j < input->rows() - kernel->rows(); j+=stride_len) {
+ for (int k = 0; k < kernel->cols(); k++) {
+ for (int l = 0; l < kernel->rows(); l++) {
+ if ((input->block(j, i, kernel->rows(), kernel->cols()))(l, k) > maxnum) {
+ maxnum = (input->block(j, i, kernel->rows(), kernel->cols()))(l, k);
+ }
+ }
+ }
}
- }
}
- }
}
ConvNet::ConvNet(char* path, float learn_rate, float bias_rate, Regularization reg, float l, float ratio)
- : Network(path, 1, learn_rate, bias_rate, reg, l, ratio), preprocess_length{0}
+ :Network(path, 1, learn_rate, bias_rate, reg, l, ratio), preprocess_length{0}
{
- ReadMNIST(10000,784,data);
- data_labels = read_mnist_labels("./t10k-labels-idx1-ubyte",10000);
- labels = new Eigen::MatrixXf (1, 1);
+ ReadMNIST(10000,784,data);
+ data_labels = read_mnist_labels("./t10k-labels-idx1-ubyte",10000);
+ labels = new Eigen::MatrixXf (1, 1);
}
-void ConvNet::add_conv_layer(int x, int y, int stride, int kern_x, int kern_y, int pad)
+void ConvNet::add_conv_layer(int x, int y, int stride, int kern_x, int kern_y, int pad, std::function<float(float)> activ, std::function<float(float)> activ_deriv)
{
- preprocess_length+=1;
- conv_layers.emplace_back(x,y,stride,kern_x, kern_y,pad);
+ preprocess_length+=1;
+ conv_layers.emplace_back(x,y,stride,kern_x, kern_y,pad,activ,activ_deriv);
}
// May make this inaccessible to user code and just have it called from add_conv_layer as pooling is basically always paired with conv.
void ConvNet::add_pool_layer(int x, int y, int stride, int kern_x, int kern_y, int pad)
{
- pool_layers.emplace_back(x,y,stride,kern_x,kern_y,pad);
+ pool_layers.emplace_back(x,y,stride,kern_x,kern_y,pad);
}
void ConvNet::initialize()
{
- for (int i = 0; i < length-1; i++) {
- layers[i].init_weights(layers[i+1]);
- }
+ for (int i = 0; i < length-1; i++) {
+ layers[i].init_weights(layers[i+1]);
+ }
}
void ConvNet::next_batch()
{
- for (int i = 0; i < 784; i++) {
- (*conv_layers[0].input)(i/28, i%28) = data[batches][i];
- }
- (*labels)(0,0) = (float)(int)data_labels[batches];
+ for (int i = 0; i < 784; i++) {
+ (*conv_layers[0].input)(i/28, i%28) = data[batches][i];
+ }
+ (*labels)(0,0) = (float)(int)data_labels[batches];
}
void ConvNet::process()
{
- // Assumes pooling is immediately after any conv layer.
- for (int i = 0; i < preprocess_length-1; i++) {
- conv_layers[i].convolute();
- // pool_layers[i].input = conv_layers[i].output;
- // pool_layers[i].pool();
- conv_layers[i+1].input = conv_layers[i].output;
- }
- conv_layers[preprocess_length-1].convolute();
- //pool_layers[preprocess_length-1].input = conv_layers[preprocess_length-1].output;
- //pool_layers[preprocess_length-1].pool();
- // std::cout << "Output:\n" << *pool_layers[preprocess_length-1].output << "\n\n";
- Eigen::Map<Eigen::RowVectorXf> flattened (conv_layers[preprocess_length-1].output->data(), conv_layers[preprocess_length-1].output->size());
- // std::cout << "Flattened:\n" << flattened << "\n\n";
- for (int i = 0; i < flattened.cols(); i++) {
- (*layers[0].contents)(0, i) = flattened[i];
- }
+ // Assumes pooling is immediately after any conv layer.
+ for (int i = 0; i < preprocess_length-1; i++) {
+ conv_layers[i].convolute();
+ // pool_layers[i].input = conv_layers[i].output;
+ // pool_layers[i].pool();
+ conv_layers[i+1].input = conv_layers[i].output;
+ }
+ conv_layers[preprocess_length-1].convolute();
+ //pool_layers[preprocess_length-1].input = conv_layers[preprocess_length-1].output;
+ //pool_layers[preprocess_length-1].pool();
+ // std::cout << "Output:\n" << *pool_layers[preprocess_length-1].output << "\n\n";
+ Eigen::Map<Eigen::RowVectorXf> flattened (conv_layers[preprocess_length-1].output->data(), conv_layers[preprocess_length-1].output->size());
+ // std::cout << "Flattened:\n" << flattened << "\n\n";
+ for (int i = 0; i < flattened.cols(); i++) {
+ (*layers[0].contents)(0, i) = flattened[i];
+ }
}
void ConvNet::set_label(Eigen::MatrixXf newlabels)
{
- *labels = newlabels;
+ *labels = newlabels;
}
void ConvNet::list_net()
{
- for (int i = 0; i < preprocess_length; i++) {
- std::cout << "-----------------------\nCONVOLUTIONAL LAYER " << i << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nStride: " << conv_layers[i].stride_len << "\nPadding: " << conv_layers[i].padding << "\n\n\u001b[31mINPUT:\x1B[0;37m\n" << *conv_layers[i].input << "\n\n\u001b[31mKERNEL:\x1B[0;37m\n" << *conv_layers[i].kernel << "\n\n\u001b[31mOUTPUT:\x1B[0;37m\n" << *conv_layers[i].output << "\n\n\u001b[31mBIAS:\x1B[0;37m\n" << conv_layers[i].bias << "\n\n\n";
- //std::cout << "-----------------------\nPOOLING LAYER " << i << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nStride: " << pool_layers[i].stride_len << "\nPadding: " << conv_layers[i].padding << "\n\n\u001b[31mINPUT:\x1B[0;37m\n" << *pool_layers[i].input << "\n\n\u001b[31mKERNEL:\x1B[0;37m\n-" << *pool_layers[i].kernel << "\n\n\u001b[31mOUTPUT:\x1B[0;37m\n" << *pool_layers[i].output << "\n\n\n";
- }
- std::cout << "-----------------------\nINPUT LAYER (LAYER 0)\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[0].activation_str << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[0].contents << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << *layers[0].weights << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[0].bias << "\n\n\n";
- for (int i = 1; i < length-1; i++) {
- std::cout << "-----------------------\nLAYER " << i << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[i].activation_str << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[i].contents << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[i].bias << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << *layers[i].weights << "\n\n\n";
- }
- std::cout << "-----------------------\nOUTPUT LAYER (LAYER " << length-1 << ")\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[length-1].activation_str <<"\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[length-1].contents << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[length-1].bias << "\n\n\n";
+ for (int i = 0; i < preprocess_length; i++) {
+ std::cout << "-----------------------\nCONVOLUTIONAL LAYER " << i << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nStride: " << conv_layers[i].stride_len << "\nPadding: " << conv_layers[i].padding << "\n\n\u001b[31mINPUT:\x1B[0;37m\n" << *conv_layers[i].input << "\n\n\u001b[31mKERNEL:\x1B[0;37m\n" << *conv_layers[i].kernel << "\n\n\u001b[31mOUTPUT:\x1B[0;37m\n" << *conv_layers[i].output << "\n\n\u001b[31mBIAS:\x1B[0;37m\n" << conv_layers[i].bias << "\n\n\n";
+ //std::cout << "-----------------------\nPOOLING LAYER " << i << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nStride: " << pool_layers[i].stride_len << "\nPadding: " << conv_layers[i].padding << "\n\n\u001b[31mINPUT:\x1B[0;37m\n" << *pool_layers[i].input << "\n\n\u001b[31mKERNEL:\x1B[0;37m\n-" << *pool_layers[i].kernel << "\n\n\u001b[31mOUTPUT:\x1B[0;37m\n" << *pool_layers[i].output << "\n\n\n";
+ }
+ std::cout << "-----------------------\nINPUT LAYER (LAYER 0)\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[0].activation_str << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[0].contents << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << *layers[0].weights << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[0].bias << "\n\n\n";
+ for (int i = 1; i < length-1; i++) {
+ std::cout << "-----------------------\nLAYER " << i << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[i].activation_str << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[i].contents << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[i].bias << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << *layers[i].weights << "\n\n\n";
+ }
+ std::cout << "-----------------------\nOUTPUT LAYER (LAYER " << length-1 << ")\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[length-1].activation_str <<"\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[length-1].contents << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[length-1].bias << "\n\n\n";
}
void ConvNet::backpropagate()
@@ -265,8 +272,8 @@ void ConvNet::backpropagate()
*layers[length-1-i].bias -= bias_lr * gradients[i];
}
Eigen::Map<Eigen::MatrixXf> reshaped(gradients[gradients.size()-1].data(),
- conv_layers.back().output->rows(),
- conv_layers.back().output->cols());
+ conv_layers.back().output->rows(),
+ conv_layers.back().output->cols());
gradients[gradients.size()-1] = reshaped;
std::vector<Eigen::MatrixXf> conv_deltas;
for (int layer = conv_layers.size()-1; layer >= 0; layer--) {
@@ -286,58 +293,58 @@ void ConvNet::backpropagate()
Eigen::MatrixXf padded_grad = Eigen::MatrixXf::Zero(gradients.back().rows() + ((flipped_kernel.rows() - 1)*2), gradients.back().cols() + ((flipped_kernel.cols() - 1)*2));
padded_grad.block(flipped_kernel.rows() - 1, flipped_kernel.cols() - 1, gradients.back().rows(), gradients.back().cols()) = gradients.back();
Eigen::MatrixXf final_grad (padded_grad.rows() - flipped_kernel.rows() + 1, padded_grad.cols() - flipped_kernel.cols() + 1);
- std::cout << padded_grad.rows() << " " << flipped_kernel.rows() << "\n";
for (int i = 0; i < padded_grad.rows() - flipped_kernel.rows() + 1; i++) {
for (int j = 0; j < padded_grad.cols() - gradients.back().cols() + 1; j++) {
final_grad(i, j) = (flipped_kernel * padded_grad.block(i, j, flipped_kernel.rows(), flipped_kernel.cols())).sum();
}
}
- gradients.push_back(final_grad);
+ gradients.push_back(final_grad.cwiseProduct(*conv_layers[layer].dZ));
}
}
void ConvNet::train()
{
- float cost_sum = 0;
- float acc_sum = 0;
- for (int i = 0; i <= 100; i++) {
- if (i != instances-batch_size) { // Don't try to advance batch on final batch.
- next_batch();
+ float cost_sum = 0;
+ float acc_sum = 0;
+ for (int i = 0; i <= 100; i++) {
+ if (i != instances-batch_size) { // Don't try to advance batch on final batch.
+ next_batch();
+ }
+ process();
+ feedforward();
+ backpropagate();
+ cost_sum += cost();
+ acc_sum += accuracy();
+ batches++;
}
- process();
- feedforward();
- backpropagate();
- cost_sum += cost();
- acc_sum += accuracy();
- batches++;
- }
- epoch_acc = 1.0/(100) * acc_sum;
- epoch_cost = 1.0/(100) * cost_sum;
- printf("Epoch %i complete - cost %f - acc %f\n", epochs, epoch_cost, epoch_acc);
- batches=0;
- decay();
- epochs++;
+ list_net();
+ epoch_acc = 1.0/(100) * acc_sum;
+ epoch_cost = 1.0/(100) * cost_sum;
+ printf("Epoch %i complete - cost %f - acc %f\n", epochs, epoch_cost, epoch_acc);
+ batches=0;
+ decay();
+ epochs++;
}
int main()
{
- ConvNet net ("../data_banknote_authentication.txt", 0.05, 0.01, L2, 0, 0.9);
- Eigen::MatrixXf labels (1,1);
- net.add_conv_layer(28,28,1,9,9,0);
- // net.add_pool_layer(20,20,1,6,6,0);
- net.add_conv_layer(20,20,1,6,6,0);
- std::cout << net.conv_layers[net.conv_layers.size()-1].output->rows() << "\n";
- //net.add_pool_layer(10,10,1,2,2,0);
- net.add_layer(400, "sigmoid", sigmoid, sigmoid_deriv);
- net.add_layer(5, "lecun_tanh", lecun_tanh, lecun_tanh_deriv);
- net.add_layer(10, "resig", rectifier(sigmoid), rectifier(sigmoid_deriv));
- // net.list_net();
- // net.init_decay("step", 1, 2);
- net.initialize();
- //net.list_net();
+ ConvNet net ("../data_banknote_authentication.txt", 0.05, 0.01, L2, 0, 0.9);
+ Eigen::MatrixXf labels (1,1);
+ net.add_conv_layer(28, 28, 1, 9, 9, 0, lecun_tanh, lecun_tanh_deriv);
+ // net.add_pool_layer(20,20,1,6,6,0);
+ net.add_conv_layer(20, 20, 1, 6, 6, 0, lecun_tanh, lecun_tanh_deriv);
+ std::cout << net.conv_layers[net.conv_layers.size()-1].output->rows() << "\n";
+ //net.add_pool_layer(10,10,1,2,2,0);
+ net.add_layer(400, "sigmoid", sigmoid, sigmoid_deriv);
+ net.add_layer(5, "lecun_tanh", lecun_tanh, lecun_tanh_deriv);
+ net.add_layer(10, "resig", rectifier(sigmoid), rectifier(sigmoid_deriv));
+ // net.list_net();
+ // net.init_decay("step", 1, 2);
+ net.initialize();
+ //net.list_net();
- for (int i = 0; i < 1; i++) {
- net.train();
- }
- net.list_net();
+ for (int i = 0; i < 1; i++) {
+ net.train();
+ }
+ net.list_net();
}
diff --git a/src/cnn.hpp b/src/cnn.hpp
@@ -6,50 +6,53 @@
class ConvLayer
{
public:
- int stride_len;
- int padding;
- Eigen::MatrixXf* input;
- Eigen::MatrixXf* kernel;
- Eigen::MatrixXf* output;
- float bias;
+ int stride_len;
+ int padding;
+ Eigen::MatrixXf* input;
+ Eigen::MatrixXf* kernel;
+ Eigen::MatrixXf* output;
+ Eigen::MatrixXf* dZ;
+ std::function<float(float)> activation;
+ std::function<float(float)> activation_deriv;
+ float bias;
- ConvLayer(int x, int y, int stride, int kern_x, int kern_y, int pad);
- void convolute();
- void set_input(Eigen::MatrixXf* matrix);
+ ConvLayer(int x, int y, int stride, int kern_x, int kern_y, int pad, std::function<float(float)> activ, std::function<float(float)> activ_deriv);
+ void convolute();
+ void set_input(Eigen::MatrixXf* matrix);
};
class PoolingLayer
{
public:
- int stride_len;
- int padding;
- Eigen::MatrixXf* input;
- Eigen::MatrixXf* kernel;
- Eigen::MatrixXf* output;
-
- void pool();
- PoolingLayer(int x, int y, int stride, int kern_x, int kern_y, int pad);
+ int stride_len;
+ int padding;
+ Eigen::MatrixXf* input;
+ Eigen::MatrixXf* kernel;
+ Eigen::MatrixXf* output;
+
+ void pool();
+ PoolingLayer(int x, int y, int stride, int kern_x, int kern_y, int pad);
};
class ConvNet : public Network
{
public:
- int preprocess_length;
- std::vector<std::vector<double>> data;
- unsigned char* data_labels;
-
- std::vector<ConvLayer> conv_layers;
- std::vector<PoolingLayer> pool_layers;
-
- ConvNet(char* path, float learn_rate, float bias_rate, Regularization reg, float l, float ratio);
- void list_net();
- void process(); // Runs the convolutional and pooling layers.
- void next_batch();
- void backpropagate();
- void train();
- void add_conv_layer(int x, int y, int stride, int kern_x, int kern_y, int pad);
- void add_pool_layer(int x, int y, int stride, int kern_x, int kern_y, int pad);
- void set_label(Eigen::MatrixXf newlabels);
- void initialize();
+ int preprocess_length;
+ std::vector<std::vector<double>> data;
+ unsigned char* data_labels;
+
+ std::vector<ConvLayer> conv_layers;
+ std::vector<PoolingLayer> pool_layers;
+
+ ConvNet(char* path, float learn_rate, float bias_rate, Regularization reg, float l, float ratio);
+ void list_net();
+ void process(); // Runs the convolutional and pooling layers.
+ void next_batch();
+ void backpropagate();
+ void train();
+ void add_conv_layer(int x, int y, int stride, int kern_x, int kern_y, int pad, std::function<float(float)> activ, std::function<float(float)> activ_deriv);
+ void add_pool_layer(int x, int y, int stride, int kern_x, int kern_y, int pad);
+ void set_label(Eigen::MatrixXf newlabels);
+ void initialize();
};
#endif /* MODULE_H */
