commit 129f8e605e178d2dfa8d42c69e10ad94e5b50474
parent bf7232775545867fd456307a2ad98111052ec82d
Author: David Freifeld <freifeld.david@gmail.com>
Date: Tue, 14 Jul 2020 16:01:09 -0700
Backprop looks normal for BPNN
Diffstat:
5 files changed, 21 insertions(+), 26 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -16,9 +16,9 @@ double bench(int batch_sz)
auto start = std::chrono::high_resolution_clock::now();
Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 0, 0.9);
net.add_layer(4, "linear");
- net.add_layer(5, "lecun_tanh");
+ net.add_layer(6, "lecun_tanh");
net.add_layer(2, "linear");
- // net.init_decay("step", 1, 2);
+ net.init_decay("step", 1, 2);
net.initialize();
// checks(net);
// for (int i = 0; i < 10; i++) {
@@ -28,7 +28,7 @@ double bench(int batch_sz)
// net.backpropagate();
// std::cout << net.cost() << " " << net.accuracy() << "\n";
// }
- for (int i = 0; i < 75; i++) {
+ for (int i = 0; i < 500; i++) {
net.train();
// net.list_net();
}
diff --git a/scripts/sweep.yaml b/scripts/sweep.yaml
@@ -1,3 +1,11 @@
+#
+# sweep.yaml
+# Jacobian
+#
+# Created by David Freifeld
+# Copyright © 2020 David Freifeld. All rights reserved.
+#
+
program: example.py
method: bayes
metric:
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -3,7 +3,6 @@
// Jacobian
//
// Created by David Freifeld
-// Copyright © 2020 David Freifeld. All rights reserved.
//
#include "bpnn.hpp"
@@ -58,6 +57,7 @@ Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, fl
test_instances = split_file(SHUFFLED_PATH, total_instances, ratio);
instances = total_instances - test_instances;
data = fopen(TRAIN_PATH, "r");
+ test_data = fopen(path, "r");
decay = [](float lr, float t) -> float {
return lr;
};
@@ -242,25 +242,17 @@ void Network::backpropagate()
std::vector<Eigen::MatrixXf> gradients;
std::vector<Eigen::MatrixXf> deltas;
Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
- // std::cout << (*layers[length-1].contents) << "\n\n\n";
- // std::cout << "OUTPUT:\n" << (*layers[length-1].contents) << "\n\n";
- // std::cout << "WEIGHT\n" << (*layers[length-2].weights) << "\n\n";
- // std::cout << "X:\n" << (*layers[length-2].contents) << "\n\n";
for (int i = 0; i < error.rows(); i++) {
for (int j = 0; j < error.cols(); j++) {
float truth;
if (j==(*labels)(i,0)) truth = 1;
else truth = 0;
- error(i,j) = truth - (*layers[length-1].contents)(i,j);
+ error(i,j) = (*layers[length-1].contents)(i,j) - truth;
checknan(error(i,j), "gradient of final layer");
- // std::cout << truth << "[as label is "<< (*labels)(i,0) <<"] - " << (*layers[length-1].contents)(i,j) << "[aka index " << i << " " << j << "] = " << error(i,j) << "\n";
}
}
- // std::cout << error << "\n\n";
gradients.push_back(error);
deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
- // std::cout << deltas[0] << "\n\n";
- // gradients[523] += error;
int counter = 1;
for (int i = length-2; i >= 1; i--) {
gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
@@ -361,7 +353,6 @@ int split_file(char* path, int lines, float ratio)
float Network::test(char* path)
{
- FILE* test_data = fopen(path, "r");
float costsum = 0;
float accsum = 0;
for (int i = 0; i <= test_instances-batch_size; i+=batch_size) {
@@ -389,6 +380,7 @@ float Network::test(char* path)
}
val_acc = 1.0/((float) test_instances/batch_size) * accsum;
val_cost = 1.0/((float) test_instances/batch_size) * costsum;
+ rewind(test_data);
return 0;
}
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -34,6 +34,7 @@ public:
class Network {
public:
FILE* data;
+ FILE* test_data;
int instances;
int test_instances;
diff --git a/src/cnn.cpp b/src/cnn.cpp
@@ -3,7 +3,6 @@
// Jacobian
//
// Created by David Freifeld
-// Copyright © 2020 David Freifeld. All rights reserved.
//
#include "bpnn.hpp"
@@ -286,31 +285,26 @@ void ConvNet::backpropagate()
std::vector<Eigen::MatrixXf> gradients;
std::vector<Eigen::MatrixXf> deltas;
Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
- // std::cout << (*layers[length-1].contents) << "\n\n\n";
for (int i = 0; i < error.rows(); i++) {
for (int j = 0; j < error.cols(); j++) {
float truth;
if (j==(*labels)(i,0)) truth = 1;
else truth = 0;
error(i,j) = (*layers[length-1].contents)(i,j) - truth;
- // std::cout << error(i, j) << " " << (*layers[length-1].contents)(i,j) << " " << truth << "\n";
checknan(error(i, j), "gradient of final layer");
- // std::cout << truth << "[as label is "<< (*labels)(i,0) <<"] - " << (*layers[length-1].contents)(i,j) << "[aka index " << i << " " << j << "] = " << error(i,j) << "\n";
}
}
gradients.push_back(error);
deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
int counter = 1;
for (int i = length-2; i >= 1; i--) {
- //std::cout << "--GRAD---\n" << gradients[counter-1] << "\n\n" << layers[i].weights->transpose() << "\n\n" << *layers[i].dZ << "\n\n";
gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
- //std::cout << "---DELTA---\n" << gradients[counter] << "\n\n" << layers[i].weights->transpose() << "\n\n" << *layers[i].dZ << "\n\n";
deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
counter++;
}
gradients.push_back((gradients[gradients.size()-1] * layers[0].weights->transpose()).cwiseProduct(*layers[0].dZ));
for (int i = 0; i < length-1; i++) {
- std::cout << learning_rate << " (LR) \n" << deltas[i] << "\n\n";
+ // std::cout << learning_rate << " (LR) \n" << deltas[i] << "\n\n";
*layers[length-2-i].weights -= learning_rate * deltas[i];
*layers[length-1-i].bias -= bias_lr * gradients[i];
}
@@ -334,7 +328,7 @@ void ConvNet::train()
{
float cost_sum = 0;
float acc_sum = 0;
- for (int i = 0; i <= 1; i++) {
+ for (int i = 0; i <= 10; i++) {
if (i != instances-batch_size) { // Don't try to advance batch on final batch.
next_batch();
}
@@ -345,8 +339,8 @@ void ConvNet::train()
acc_sum += accuracy();
batches++;
}
- epoch_acc = 1.0/(10000) * acc_sum;
- epoch_cost = 1.0/(10000) * cost_sum;
+ epoch_acc = 1.0/(10) * acc_sum;
+ epoch_cost = 1.0/(10) * cost_sum;
printf("Epoch %i complete - cost %f - acc %f\n", epochs, epoch_cost, epoch_acc);
batches=0;
learning_rate = decay(learning_rate, epochs);
@@ -364,10 +358,10 @@ int main()
net.add_layer(625, "linear");
net.add_layer(5, "relu");
net.add_layer(10, "linear");
- net.init_decay("step", 1, 2);
+ // net.init_decay("step", 1, 2);
net.initialize();
- for (int i = 0; i < 1; i++) {
+ for (int i = 0; i < 10; i++) {
net.train();
}
std::cout << *net.layers[net.length-1].contents << "\n";
