commit 5e70fba38c3bce59a155aaea9dadc7809490832c
parent 6f482b7974dcba5042b6dfce7db6453223fda80e
Author: David Freifeld <freifeld.david@gmail.com>
Date: Sat, 5 Sep 2020 11:25:20 -0700
Minimal updates, better typecasting
Diffstat:
8 files changed, 142 insertions(+), 212 deletions(-)
diff --git a/checks.cpp b/checks.cpp
@@ -236,6 +236,49 @@ void basic_checks()
}
}
-void grad_checks()
-{
-}
+// Eigen::MatrixXf Network::numerical_grad(int i, float epsilon)
+// {
+// Eigen::MatrixXf gradient (layers[i].weights->rows(), layers[i].weights->cols());
+// for (int j = 0; j < layers[i].weights->rows(); j++) {
+// for (int k = 0; k < layers[i].weights->cols(); k++) {
+// float current_cost = cost();
+// std::vector<Layer> backup = layers;
+// (*layers[i].weights)(j,k) += epsilon;
+// feedforward();
+// float end_cost = cost();
+// gradient(j,k) = (end_cost - current_cost)/epsilon;
+// layers = backup;
+// batches = 0;
+// }
+// }
+// return gradient;
+// }
+
+// void Network::grad_check()
+// {
+// std::vector<Layer> backup = layers;
+// feedforward();
+// layers = backup;
+// batches = 0;
+// std::vector<Eigen::MatrixXf> gradients;
+// std::vector<Eigen::MatrixXf> deltas;
+// Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
+// for (int i = 0; i < error.rows(); i++) {
+// for (int j = 0; j < error.cols(); j++) {
+// float truth;
+// if (j==(*labels)(i,0)) truth = 1;
+// else truth = 0;
+// error(i,j) = (*layers[length-1].contents)(i,j) - truth;
+// checknan(error(i,j), "gradient of final layer");
+// }
+// }
+// int counter = 1;
+// gradients.push_back(error);
+// deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
+// for (int i = length-2; i >= 1; i--) {
+// gradients.push_back(cwise_product(gradients[counter-1] * layers[i].weights->transpose(),*layers[i].dZ));
+// std::cout << layers[i-1].contents->transpose() * gradients[counter];
+// deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
+// counter++;
+// }
+// }
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -7,6 +7,8 @@
#include "bpnn.hpp"
#include "utils.hpp"
+#include <atomic>
+#include <chrono>
#include <ctime>
#include <random>
@@ -27,14 +29,12 @@ Layer::Layer(int batch_sz, int nodes, float a)
dZ = new Eigen::MatrixXf (batch_sz, nodes);
int datalen = batch_sz*nodes;
for (int i = 0; i < datalen; i++) {
- (*contents)((int)i / nodes,i%nodes) = 0;
- (*dZ)((int)i / nodes,i%nodes) = 0;
+ (*contents)(static_cast<int>(i / nodes),i%nodes) = 0;
+ (*dZ)(static_cast<int>(i / nodes),i%nodes) = 0;
}
bias = new Eigen::MatrixXf (batch_sz, nodes);
for (int i = 0; i < nodes; i++) {
- for (int j = 0; j < batch_sz; j++) {
- (*bias)(j, i) = 0;
- }
+ for (int j = 0; j < batch_sz; j++) (*bias)(j, i) = 0;
}
}
@@ -64,8 +64,8 @@ void Layer::init_weights(Layer next)
std::random_device rd;
std::mt19937 gen(rd());
(*weights)((int)i / nodes, i%nodes) = d(gen);
- (*v)((int)i / nodes, i%nodes) = 0;
- (*m)((int)i / nodes, i%nodes) = 0;
+ (*v)(static_cast<int>(i / nodes), i%nodes) = 0;
+ (*m)(static_cast<int>(i / nodes), i%nodes) = 0;
}
}
@@ -79,9 +79,7 @@ Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, in
val_data = fopen(VAL_PATH, "r");
instances = total_instances - val_instances;
assert(batch_size > 0 || batch_size < instances);
- decay = [this]() -> void {
- learning_rate = learning_rate;
- };
+ decay = [this]() -> void {};
update = [this](std::vector<Eigen::MatrixXf> deltas, int i) {
*layers[length-2-i].weights -= (learning_rate * deltas[i]);
};
@@ -141,76 +139,19 @@ void Network::add_prelu_layer(int nodes, float a)
};
}
-// Gross code inbound
-void Network::add_layer(int nodes, char* name)
+void Network::add_layer(int nodes, std::function<float(float)> activation, std::function<float(float)> activation_deriv)
{
length++;
layers.emplace_back(batch_size, nodes);
strcpy(layers[length-1].activation_str, name);
- if (strcmp(name, "sigmoid") == 0) {
- layers[length-1].activation = sigmoid;
- layers[length-1].activation_deriv = sigmoid_deriv;
- }
- else if (strcmp(name, "linear") == 0) {
- layers[length-1].activation = linear;
- layers[length-1].activation_deriv = linear_deriv;
- }
- else if (strcmp(name, "step") == 0) {
- layers[length-1].activation = step;
- layers[length-1].activation_deriv = step_deriv;
- }
- else if (strcmp(name, "lecun_tanh") == 0) {
- layers[length-1].activation = lecun_tanh;
- layers[length-1].activation_deriv = lecun_tanh_deriv;
- }
- else if (strcmp(name, "inverse_logit") == 0) {
- layers[length-1].activation = inverse_logit;
- layers[length-1].activation_deriv = inverse_logit_deriv;
- }
- else if (strcmp(name, "cloglog") == 0) {
- layers[length-1].activation = cloglog;
- layers[length-1].activation_deriv = cloglog_deriv;
- }
- else if (strcmp(name, "softplus") == 0) {
- layers[length-1].activation = softplus;
- layers[length-1].activation_deriv = softplus_deriv;
- }
- else if (strcmp(name, "relu") == 0) {
- layers[length-1].activation = rectifier(linear);
- layers[length-1].activation_deriv = rectifier(linear_deriv);
- }
- else if (strcmp(name, "leaky_relu") == 0) {
- layers[length-1].activation = leaky_relu;
- layers[length-1].activation_deriv = leaky_relu_deriv;
- }
- else if (strcmp(name, "bipolar_sigmoid") == 0) {
- layers[length-1].activation = bipolar_sigmoid;
- layers[length-1].activation_deriv = bipolar_sigmoid_deriv;
- }
- else if (strcmp(name, "tanh") == 0) {
- layers[length-1].activation = [](float x) -> float {return tanh(x);};
- layers[length-1].activation_deriv = [](float x) -> float {return 1.0/cosh(x);};
- }
- else if (strcmp(name, "hard_tanh") == 0) {
- layers[length-1].activation = hard_tanh;
- layers[length-1].activation_deriv = hard_tanh_deriv;
- }
- else if (strcmp(name, "resig") == 0) {
- layers[length-1].activation = rectifier(sigmoid);
- layers[length-1].activation_deriv = rectifier(sigmoid_deriv);
- }
- else {
- std::cout << "Warning! Incorrect activation specified. Exiting.\n";
- exit(1);
- }
+ layers[length-1].activation = activation;
+ layers[length-1].activation_deriv = activation_deriv;
}
void Network::initialize()
{
labels = new Eigen::MatrixXf (batch_size,layers[length-1].contents->cols());
- for (int i = 0; i < length-1; i++) {
- layers[i].init_weights(layers[i+1]);
- }
+ for (int i = 0; i < length-1; i++) layers[i].init_weights(layers[i+1]);
}
void Network::set_activation(int index, std::function<float(float)> custom, std::function<float(float)> custom_deriv)
@@ -326,53 +267,6 @@ Eigen::MatrixXf l1_deriv(Eigen::MatrixXf m)
return r;
}
-Eigen::MatrixXf Network::numerical_grad(int i, float epsilon)
-{
- Eigen::MatrixXf gradient (layers[i].weights->rows(), layers[i].weights->cols());
- for (int j = 0; j < layers[i].weights->rows(); j++) {
- for (int k = 0; k < layers[i].weights->cols(); k++) {
- float current_cost = cost();
- std::vector<Layer> backup = layers;
- (*layers[i].weights)(j,k) += epsilon;
- feedforward();
- float end_cost = cost();
- gradient(j,k) = (end_cost - current_cost)/epsilon;
- layers = backup;
- batches = 0;
- }
- }
- return gradient;
-}
-
-void Network::grad_check() \
-{
- std::vector<Layer> backup = layers;
- feedforward();
- layers = backup;
- batches = 0;
- std::vector<Eigen::MatrixXf> gradients;
- std::vector<Eigen::MatrixXf> deltas;
- Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
- for (int i = 0; i < error.rows(); i++) {
- for (int j = 0; j < error.cols(); j++) {
- float truth;
- if (j==(*labels)(i,0)) truth = 1;
- else truth = 0;
- error(i,j) = (*layers[length-1].contents)(i,j) - truth;
- checknan(error(i,j), "gradient of final layer");
- }
- }
- int counter = 1;
- gradients.push_back(error);
- deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
- for (int i = length-2; i >= 1; i--) {
- gradients.push_back(cwise_product(gradients[counter-1] * layers[i].weights->transpose(),*layers[i].dZ));
- std::cout << layers[i-1].contents->transpose() * gradients[counter];
- deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
- counter++;
- }
-}
-
void Network::backpropagate()
{
std::vector<Eigen::MatrixXf> gradients;
@@ -390,10 +284,10 @@ void Network::backpropagate()
gradients.push_back(error);
deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
int counter = 1;
- for (int i = length-2; i >= 1; i--) {
- // TODO: Find nice way to add this
+ for (int i = length-2; i >= 1; i--) {
+ // TODO: Add nesterov momentum | -p B -t conundrum -t coding -m Without causing segmentation faults.
// (*layers[i].weights-((learning_rate * *layers[i].weights) + (0.9 * *layers[i].v))).transpose()
- //grad_calc(gradients, counter, i);
+ //grad_calc(gradients, counter, i)
gradients.push_back(cwise_product(gradients[counter-1] * layers[i].weights->transpose(), *layers[i].dZ));
deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
counter++;
@@ -408,7 +302,7 @@ void Network::backpropagate()
for (int j = 0; j < layers[length-2-i].contents->rows(); j++) {
for (int k = 0; k < layers[length-2-i].contents->cols(); k++) {
if ((*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha <= 0) {
- // Choice of using index i+1 here is questionable. TODO: REVIEW
+ // TODO: Review questionable code | -t quality -m Choice of using index i+1 here is sketchy.
sum += gradients[i+1](j,k) * (*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha;
}
}
@@ -431,74 +325,10 @@ void Network::backpropagate()
void Network::update_layer(float* vals, int datalen, int index)
{
- for (int i = 0; i < datalen; i++) (*layers[index].contents)((int)i / layers[index].contents->cols(),i%layers[index].contents->cols()) = vals[i];
-}
-
-int Network::next_batch()
-{
- char line[MAXLINE] = {' '};
- int inputs = layers[0].contents->cols();
- int datalen = batch_size * inputs;
- float batch[datalen];
- for (int i = 0; i < batch_size; i++) {
- fgets(line, MAXLINE, data);
- char *p;
- p = strtok(line,",");
- for (int j = 0; j < inputs; j++) {
- batch[j + (i * inputs)] = strtod(p, NULL);
- p = strtok(NULL,",");
- }
- (*labels)(i, 0) = strtod(p, NULL);
- }
- float* batchptr = batch;
- update_layer(batchptr, datalen, 0);
- return 0;
+ for (int i = 0; i < datalen; i++) (*layers[index].contents)(static_cast<int>(i / layers[index].contents->cols()), i%layers[index].contents->cols()) = vals[i];
}
-int prep_file(char* path, char* out_path)
-{
- FILE* rptr = fopen(path, "r");
- char line[MAXLINE];
- std::vector<std::string> lines;
- int count = 0;
- while (fgets(line, MAXLINE, rptr) != NULL) {
- lines.emplace_back(line);
- count++;
- }
- lines[lines.size()-1] = lines[lines.size()-1] + "\n";
- std::random_device rd;
- std::mt19937 g(rd());
- std::shuffle(lines.begin(), lines.end(), g);
- fclose(rptr);
- FILE* wptr = fopen(out_path, "w");
- for (std::string & i : lines) {
- const char* cstr = i.c_str();
- fprintf(wptr,"%s", cstr);
- }
- fclose(wptr);
- return count;
-}
-
-int split_file(char* path, int lines, float ratio)
-{
- FILE* src = fopen(path, "r");
- FILE* test = fopen(VAL_PATH, "w");
- FILE* train = fopen(TRAIN_PATH, "w");
- int switch_line = round(ratio * lines);
- char line[MAXLINE];
- int tests = 0;
- for (int i = 0; fgets(line, MAXLINE, src) != NULL; i++) {
- if (i > switch_line) {
- fprintf(test, "%s", line);
- tests++;
- }
- else fprintf(train, "%s", line);
- }
- fclose(src);
- fclose(test);
- fclose(train);
- return tests;
-}
+#include "data.cpp"
float Network::validate(char* path)
{
@@ -539,17 +369,12 @@ void Network::train()
float acc_sum = 0;
for (int i = 0; i <= instances-batch_size; i+=batch_size) {
if (early_stop == true && get_val_cost() < threshold) return;
- if (i != instances-batch_size) { // Don't try to advance batch on final batch.
- next_batch();
- }
+ if (i != instances-batch_size) next_batch();
feedforward();
backpropagate();
cost_sum += cost();
acc_sum += accuracy();
batches++;
- // if (i > batch_size * 10) {
- // exit(1);
- // }
}
epoch_acc = 1.0/((float) instances/batch_size) * acc_sum;
epoch_cost = 1.0/((float) instances/batch_size) * cost_sum;
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -26,7 +26,7 @@ public:
std::function<float(float)> activation;
std::function<float(float)> activation_deriv;
char activation_str[1024];
- // PReLU layers shouldn't be Layers but inherit from them! Fix me!!
+ // TODO: Fix PReLU layer inheritance | -p C -m PReLU layers shouldn't be Layers but inherit from them!
float alpha;
Layer(int rows, int columns, float a=0);
diff --git a/src/cnn.cpp b/src/cnn.cpp
@@ -302,8 +302,6 @@ std::vector<Eigen::MatrixXf> gradients;
deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
int counter = 1;
for (int i = length-2; i >= 1; i--) {
- // TODO: Find nice way to add this
- // *layers[i].weights-((learning_rate * *layers[i].weights) + (0.9 * *layers[i].v))).transpose()
gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
counter++;
@@ -343,7 +341,7 @@ std::vector<Eigen::MatrixXf> gradients;
conv_deltas.emplace_back(conv_layers[conv_layers.size()-1].input->rows() - gradients[length-1].rows()+1, conv_layers[conv_layers.size()-1].input->cols() - gradients[length-1].cols()+1);
for (int i = 0; i < conv_deltas[0].cols(); i+=conv_layers[conv_layers.size()-1].stride_len) {
for (int j = 0; j < conv_deltas[0].rows(); j+=conv_layers[conv_layers.size()-1].stride_len) {
- // Transpose here is sketchy
+ // TODO: Investigate legitimacy of tranpose | -t :quality:
conv_deltas[0](j,i) = (gradients[length-1] * (conv_layers[conv_layers.size()-1].input->block(j, i, gradients[length-1].rows(), gradients[length-1].cols())).transpose()).sum();
}
}
diff --git a/src/data.cpp b/src/data.cpp
@@ -0,0 +1,65 @@
+int Network::next_batch()
+{
+ char line[MAXLINE] = {' '};
+ int inputs = layers[0].contents->cols();
+ int datalen = batch_size * inputs;
+ float batch[datalen];
+ for (int i = 0; i < batch_size; i++) {
+ fgets(line, MAXLINE, data);
+ char *p;
+ p = strtok(line,",");
+ for (int j = 0; j < inputs; j++) {
+ batch[j + (i * inputs)] = strtod(p, NULL);
+ p = strtok(NULL,",");
+ }
+ (*labels)(i, 0) = strtod(p, NULL);
+ }
+ float* batchptr = batch;
+ update_layer(batchptr, datalen, 0);
+ return 0;
+}
+
+int prep_file(char* path, char* out_path)
+{
+ FILE* rptr = fopen(path, "r");
+ char line[MAXLINE];
+ std::vector<std::string> lines;
+ int count = 0;
+ while (fgets(line, MAXLINE, rptr) != NULL) {
+ lines.emplace_back(line);
+ count++;
+ }
+ lines[lines.size()-1] = lines[lines.size()-1] + "\n";
+ std::random_device rd;
+ std::mt19937 g(rd());
+ std::shuffle(lines.begin(), lines.end(), g);
+ fclose(rptr);
+ FILE* wptr = fopen(out_path, "w");
+ for (std::string & i : lines) {
+ const char* cstr = i.c_str();
+ fprintf(wptr,"%s", cstr);
+ }
+ fclose(wptr);
+ return count;
+}
+
+int split_file(char* path, int lines, float ratio)
+{
+ FILE* src = fopen(path, "r");
+ FILE* test = fopen(VAL_PATH, "w");
+ FILE* train = fopen(TRAIN_PATH, "w");
+ int switch_line = round(ratio * lines);
+ char line[MAXLINE];
+ int tests = 0;
+ for (int i = 0; fgets(line, MAXLINE, src) != NULL; i++) {
+ if (i > switch_line) {
+ fprintf(test, "%s", line);
+ tests++;
+ }
+ else fprintf(train, "%s", line);
+ }
+ fclose(src);
+ fclose(test);
+ fclose(train);
+ return tests;
+}
diff --git a/src/experimental/strassen.cpp b/src/experimental/strassen.cpp
@@ -46,13 +46,13 @@ Eigen::MatrixXf strassen_mul(Eigen::MatrixXf x, Eigen::MatrixXf y)
Eigen::MatrixXf m6 = (a.block(a.rows()-block_len,0, block_len, block_len) - a.block(0,0, block_len, block_len)) * (b.block(0,0, block_len, block_len) + b.block(0,b.cols()-block_len, block_len, block_len));
Eigen::MatrixXf m7 = (a.block(0,a.cols()-block_len, block_len, block_len) - a.block(a.rows()-block_len,a.cols()-block_len, block_len, block_len)) * (b.block(a.rows()-block_len,0, block_len, block_len) + b.block(b.rows()-block_len,b.cols()-block_len, block_len, block_len));
- std::cout << m1 + m4 - m5 + m7 << "\n\n" << m3+m5 << "\n\n" << m2+m4 << "\n\n" << m1-m2+m3+m6;
+ // std::cout << m1 + m4 - m5 + m7 << "\n\n" << m3+m5 << "\n\n" << m2+m4 << "\n\n" << m1-m2+m3+m6;
result.block(0,0, block_len, block_len) = m1 + m4 - m5 + m7;
result.block(0,result.cols()-block_len, block_len, block_len) = m3 + m5;
result.block(result.rows()-block_len,0, block_len, block_len) = m2 + m4;
result.block(result.rows()-block_len,result.cols()-block_len, block_len, block_len) = m1 -m2 + m3 + m6;
-
+ std::cout << "\nSTRASSEN_VER\n" << result << "\n\n\n";
return result;
}
@@ -72,6 +72,6 @@ int main()
Eigen::MatrixXf sproduct = strassen_mul(a, b);
auto strassen_end = std::chrono::high_resolution_clock::now();
- std::cout << "EIGEN: " << std::chrono::duration_cast<std::chrono::nanoseconds>(eigen_end - eigen_begin).count() / pow(10,9) << " STRASSEN: " << std::chrono::duration_cast<std::chrono::nanoseconds>(strassen_end - strassen_begin).count() / pow(10,9) << "\n";
- std::cout << "A:\n" << a << "\nB:\n" << b << "\nEigen:\n" << product << "\nStrassen:\n" << sproduct << "\n";
+ /* std::cout << "EIGEN: " << std::chrono::duration_cast<std::chrono::nanoseconds>(eigen_end - eigen_begin).count() / pow(10,9) << " STRASSEN: " << std::chrono::duration_cast<std::chrono::nanoseconds>(strassen_end - strassen_begin).count() / pow(10,9) << "\n"; */
+ /* std::cout << "A:\n" << a << "\nB:\n" << b << "\nEigen:\n" << product << "\nStrassen:\n" << sproduct << "\n"; */
}
diff --git a/src/optimizers.cpp b/src/optimizers.cpp
@@ -17,7 +17,7 @@ void Network::init_optimizer(char* name, ...)
*layers[length-2-i].m = (learning_rate * deltas[i]);
};
}
- // TODO: split into functions to remove reundant code
+ // TODO: Remove redundant code | -t quality -m Attempt split into functions to remove reundant code
if (strcmp(name, "nesterov") == 0) {
float beta = va_arg(args, double);
va_end(args);
@@ -47,8 +47,8 @@ void Network::init_optimizer(char* name, ...)
float beta1 = va_arg(args, double);
float beta2 = va_arg(args, double);
float epsilon = va_arg(args, double);
- // TODO: Add bias correction (requires figuring out measuring t)
- // TODO: cwiseProduct here is sketchy, look into me
+ // TODO: Add bias correction to adam | -t coding -m (requires figuring out measuring t)
+ // TODO: Investigate cwiseProduct in code | -t quality -m cwiseProduct here is sketchy, look into me
update = [this, beta1, beta2, epsilon](std::vector<Eigen::MatrixXf> deltas, int i) {
*layers[length-2-i].m = (beta1 * *layers[length-2-i].m) + ((1-beta1)*deltas[i]);
*layers[length-2-i].v = (beta2 * *layers[length-2-i].v) + (1-beta2)*(deltas[i].cwiseProduct(deltas[i]));
@@ -59,10 +59,10 @@ void Network::init_optimizer(char* name, ...)
float beta1 = va_arg(args, double);
float beta2 = va_arg(args, double);
float epsilon = va_arg(args, double);
- // TODO: Add bias correction for m (requires figuring out measuring t)
+ // TODO: Add bias correction for adamax | -t coding -m (requires figuring out measuring t)
update = [this, beta1, beta2, epsilon](std::vector<Eigen::MatrixXf> deltas, int i) {
*layers[length-2-i].m = (beta1 * *layers[length-2-i].m) + ((1-beta1)*deltas[i]);
- // FIXME: Use of .sum() here is incredibly questionable. Do this correctly.
+ // TODO: Fix Adamax calculations | -p C -t quality -m Use of .sum() here is incredibly questionable. Do this correctly.
if ((beta2 * *layers[length-2-i].v).sum() > deltas[i].array().abs().sum()) *layers[length-2-i].v = (beta2 * *layers[length-2-i].v);
else *layers[length-2-i].v = deltas[i].array().abs().matrix();
*layers[length-2-i].weights -= learning_rate * (layers[length-2-i].v->array().pow(-1).cwiseProduct(layers[length-2-i].m->array())).matrix();
diff --git a/src/pybind.cpp b/src/pybind.cpp
@@ -12,10 +12,9 @@ namespace py = pybind11;
PYBIND11_MODULE(mrbpnn, m) {
m.doc() = "Fast machine learning in C++"; // optional module docstring
-
py::class_<Network>(m, "Network")
.def(py::init<char*, int, float, float, int, float, float, bool, float>())
- .def("add_layer", &Network::add_layer, py::arg("nodes"), py::arg("activation"))
+ .def("add_layer", &Network::add_layer, py::arg("nodes"), py::arg("activation"), py::arg("activation_deriv"))
//.def("add_prelu_layer", &Network::add_prelu_layer, py::arg("nodes"), py::arg("a"))
.def("initialize", &Network::initialize)
//.def("init_decay", &Network::init_decay, py::arg("type"), py::arg("a_0"), py::arg("k"))
