commit 9493a2b16d0c16b70af2f5f5038bc8f0933db345
parent d3b6ee7546635fe8cf18969329bdf6af048963ee
Author: David Freifeld <freifeld.david@gmail.com>
Date: Sat, 25 Jul 2020 20:36:56 -0700
Various tweaks + moving to tensors?
Diffstat:
6 files changed, 40 insertions(+), 45 deletions(-)
diff --git a/Makefile b/Makefile
@@ -20,10 +20,10 @@ debug: $(GEN_FLAGS) = -Wall -U NDEBUG
fast: CXXFLAGS += $(GEN_FLAGS) -O3
fast: compile
-faster: CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/mr_bpnn_2.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -liomp5 -lpthread -lm -ldl -o mrbpnn`python3-config --extension-suffix` -O3 -mavx -mfma -march=native -mfpmath=sse -fno-pic -DMKL_ILP64 -D EIGEN_USE_MKL_ALL -D NDEBUG
+faster: CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/mr_bpnn_2.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -liomp5 -lpthread -lm -ldl -o mrbpnn`python3-config --extension-suffix` -O3 -mavx -mfma -march=native -mfpmath=sse -fno-pic -DMKL_ILP64 -D NDEBUG
faster: compile
-tradeoffs: CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/mr_bpnn_2.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -o mrbpnn`python3-config --extension-suffix` -O3 -mavx -mfma -march=native -mfpmath=sse -DMKL_ILP64 -qopenmp -fno-pic -qopt-calloc -qopt-prefetch -unroll-aggressive -qopt-calloc -use-intel-optimized-headers -ffast-math -no-prec-div -no-prec-sqrt -fimf-precision=low -fast-transcendentals -D EIGEN_USE_MKL_ALL -D NDEBUG #-qopt-report=5 -qopt-report-file=report
+tradeoffs: CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/mr_bpnn_2.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -o mrbpnn`python3-config --extension-suffix` -O3 -mavx -mfma -march=native -mfpmath=sse -DMKL_ILP64 -qopenmp -fno-pic -qopt-calloc -qopt-prefetch -unroll-aggressive -qopt-calloc -use-intel-optimized-headers -ffast-math -no-prec-div -no-prec-sqrt -fimf-precision=low -fast-transcendentals -D NDEBUG #-qopt-report=5 -qopt-report-file=report
tradeoffs: compile
@@ -31,4 +31,4 @@ reckless: CXXFLAGS = -O3
reckless: compile
compile:
- icpc $(CXXFLAGS) && rm ./mrbpnn/mrbpnn.cpython-37m-darwin.so ; cp ./mrbpnn.cpython-37m-darwin.so ./mrbpnn/mrbpnn.cpython-37m-darwin.s ; rm ./scripts/mrbpnn.cpython-37m-darwin.so ; cp ./mrbpnn.cpython-37m-darwin.so ./scripts/mrbpnn.cpython-37m-darwin.so
+ g++ $(CXXFLAGS) && rm ./mrbpnn/mrbpnn.cpython-37m-darwin.so ; cp ./mrbpnn.cpython-37m-darwin.so ./mrbpnn/mrbpnn.cpython-37m-darwin.s ; rm ./scripts/mrbpnn.cpython-37m-darwin.so ; cp ./mrbpnn.cpython-37m-darwin.so ./scripts/mrbpnn.cpython-37m-darwin.so
diff --git a/example.cpp b/example.cpp
@@ -13,10 +13,10 @@
double bench(int batch_sz)
{
- auto start = std::chrono::high_resolution_clock::now();
+ //auto start = std::chrono::high_resolution_clock::now();
Network net ("./data_banknote_authentication.txt", batch_sz, 0.05, 0.03, 0, 0.9);
net.add_layer(4, "linear");
- net.add_layer(6, "lecun_tanh");
+ net.add_layer(6, "relu");
net.add_layer(2, "linear");
// net.init_decay("step", 1, 2);
net.initialize();
@@ -34,9 +34,10 @@ double bench(int batch_sz)
}
// std::cout << *net.layers[net.length-1].contents << "\n\n";
// std::cout << *net.labels << "\n";
- auto end = std::chrono::high_resolution_clock::now();
+ //auto end = std::chrono::high_resolution_clock::now();
//net.list_net();
- return std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count() / pow(10,9);
+ return 0;
+ //return std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count() / pow(10,9);
}
int main()
diff --git a/scripts/example.py b/scripts/example.py
@@ -16,15 +16,15 @@ import numpy
import time
import wandb
-data_split = 0.75
+data_split = 0.9
-hyperparameter_defaults = dict(batch_size = 10,
+hyperparameter_defaults = dict(batch_size = 16,
hidden_layers = 1,
epochs = 50,
- learning_rate = 0.0155,
+ learning_rate = 0.05,
bias_lr = 0.03,
activation = "lecun_tanh",
- neurons = 10,
+ neurons = 6,
l = 0)
wandb.init(project="jacobian", config=hyperparameter_defaults)
@@ -35,7 +35,7 @@ net = mrbpnn.Network("../data_banknote_authentication.txt", config.batch_size, c
net.add_layer(4, "linear")
for i in range(config.hidden_layers):
net.add_layer(config.neurons, config.activation)
-net.add_layer(1, "resig")
+net.add_layer(2, "resig")
net.initialize()
for i in range(config.epochs):
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -10,6 +10,7 @@
#include <ctime>
#include <random>
#include <Eigen/MatrixFunctions>
+#include <Eigen/unsupported/CXX11/Tensor>
#define SHUFFLED_PATH "./shuffled.txt"
#define TEST_PATH "./test.txt"
@@ -378,7 +379,6 @@ float Network::test(char* path)
int label = -1;
for (int i = 0; i < batch_size; i++) {
fgets(line, MAXLINE, test_data);
- //if (strcmp(line, "\n")==0) continue;
char *p;
p = strtok(line,",");
for (int j = 0; j < inputs; j++) {
diff --git a/src/checks.cpp b/src/checks.cpp
@@ -16,8 +16,8 @@ void checks(Network net)
// list_net();
- Network copy1 = net;
- Network copy2 = net;
+ Network copy1 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ Network copy2 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
copy1.lambda += 1;
copy1.next_batch();
copy1.feedforward();
@@ -33,14 +33,14 @@ void checks(Network net)
// Check if zero cost is achievable on a batch
std::cout << "Zero-cost sanity check...";
- copy1 = net;
- copy1.lambda = 0;
- copy1.next_batch();
+ Network copy3 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ copy3.lambda = 0;
+ copy3.next_batch();
float finalcost;
for (int i = 0; i < 10000; i++) {
- copy1.feedforward();
- copy1.backpropagate();
- finalcost = copy1.cost();
+ copy3.feedforward();
+ copy3.backpropagate();
+ finalcost = copy3.cost();
if (finalcost <= ZERO_THRESHOLD) {
break;
}
@@ -54,18 +54,18 @@ void checks(Network net)
// list_net();
std::cout << "Gradient floating-point sanity check...";
- copy1 = net;
- copy1.next_batch();
- copy1.feedforward();
+ Network copy4 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ copy4.next_batch();
+ copy4.feedforward();
std::vector<Eigen::MatrixXf> gradients;
std::vector<Eigen::MatrixXf> deltas;
- Eigen::MatrixXf error = ((*copy1.layers[copy1.length-1].contents) - (*copy1.labels));
- gradients.push_back(error.cwiseProduct(*copy1.layers[copy1.length-1].dZ));
- deltas.push_back((*copy1.layers[copy1.length-2].contents).transpose() * gradients[0]);
+ Eigen::MatrixXf error = ((*copy4.layers[copy4.length-1].contents) - (*copy1.labels));
+ gradients.push_back(error.cwiseProduct(*copy4.layers[copy4.length-1].dZ));
+ deltas.push_back((*copy4.layers[copy4.length-2].contents).transpose() * gradients[0]);
int counter = 1;
- for (int i = copy1.length-2; i >= 1; i--) {
- gradients.push_back((gradients[counter-1] * copy1.layers[i].weights->transpose()).cwiseProduct(*copy1.layers[i].dZ));
- deltas.push_back(copy1.layers[i-1].contents->transpose() * gradients[counter]);
+ for (int i = copy4.length-2; i >= 1; i--) {
+ gradients.push_back((gradients[counter-1] * copy4.layers[i].weights->transpose()).cwiseProduct(*copy4.layers[i].dZ));
+ deltas.push_back(copy4.layers[i-1].contents->transpose() * gradients[counter]);
counter++;
}
auto check_gradients = [](std::vector<Eigen::MatrixXf> vec) -> bool {
@@ -89,10 +89,11 @@ void checks(Network net)
// list_net();
std::cout << "Expected loss sanity check...";
- copy1 = net;
- copy1.next_batch();
- copy1.feedforward();
- if (copy1.cost() <= 1) {
+
+ Network copy5 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ copy5.next_batch();
+ copy5.feedforward();
+ if (copy5.cost() <= 1) {
std::cout << " \u001b[32mPassed!\n\u001b[37m";
sanity_passed++;
}
@@ -101,17 +102,11 @@ void checks(Network net)
// list_net();
std::cout << "Layer updates sanity check...";
- copy1 = net;
- copy2 = net;
- copy1.next_batch();
- copy1.feedforward();
- copy2.next_batch();
- copy2.feedforward();
- copy1.backpropagate();
- int passed = 1;
- //list_net();
+ Network copy6 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
+ Network copy7 ("./data_banknote_authentication.txt", 16, 0.05, 0.03, 0, 0.9);
//copy2.list_net();
//copy1.list_net();
+ int passed;
for (int i = 0; i < copy1.layers.size()-1; i++) {
if (*copy1.layers[i].weights == *copy2.layers[i].weights) {
// std::cout << *copy2.layers[i].weights <<"uninitweight\n\n";
diff --git a/src/utils.cpp b/src/utils.cpp
@@ -16,13 +16,12 @@
#include <unistd.h>
#include <sys/stat.h>
#include <Eigen/Dense>
+#include <Eigen/MatrixFunctions>
// A bunch of hardcoded activation functions. Avoids much of the slowness of custom functions.
// Although the std::function makes it not the fastest way, the functionality is worth it.
// Yes, these functions may be a frustrating to read but they're just equations and I want to conserve space.
-//float tanhapprox(float x) {return x - (1/3 * pow(x, 3)) + (2/15 * pow(x, 5)) - (17/315 * pow(x, 7));}
-
float sigmoid(float x) {return 1.0/(1+exp(-x));}
float sigmoid_deriv(float x) {return 1.0/(1+exp(-x)) * (1 - 1.0/(1+exp(x)));}
