commit 3dfec848dd016155442d71b2fb08578e21af0355
parent fc7416d67128f4051248de31c0fa908eb4f79379
Author: David Freifeld <freifeld.david@gmail.com>
Date: Fri, 7 Aug 2020 19:20:06 -0700
More stability in checks + code
Diffstat:
6 files changed, 21 insertions(+), 56 deletions(-)
diff --git a/.github/workflows/macos-latest.yml b/.github/workflows/macos-latest.yml
@@ -14,7 +14,7 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: install
- run: brew install eigen
+ run: git clone https://gitlab.com/libeigen/eigen.git && cd eigen && cp -R Eigen /usr/local/include/ && cd ..
# python 3 -m pip install pytest
# git clone https://github.com/pybind/pybind11.git
# mkdir build
diff --git a/.github/workflows/ubuntu-latest.yml b/.github/workflows/ubuntu-latest.yml
@@ -14,9 +14,7 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: install
- run:
- sudo apt install libeigen3-dev
- sudo apt install cmake
+ run: git clone https://gitlab.com/libeigen/eigen.git
# python 3 -m pip install pytest
# git clone https://github.com/pybind/pybind11.git
# mkdir build
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -14,9 +14,9 @@ endif()
if (FAST)
set(COMPILE_FLAGS "${COMPILE_FLAGS} -O3")
elseif (FASTER)
- set(COMPILE_FLAGS "${COMPILE_FLAGS} -mavx -O3 -mavx -mfma -march=native -mfpmath=sse -fno-pic -DMKL_ILP64 -D NDEBUG")
+ set(COMPILE_FLAGS "${COMPILE_FLAGS} -mavx -O3 -mavx -mfma -march=native -mfpmath=sse -fno-pic -DMKL_ILP64")
elseif (TRADEOFFS)
- set(COMPILE_FLAGS "${COMPILE_FLAGS} -mavx -O3 -mavx -msse2 -msse3 -march=native -mfpmath=sse -DMKL_ILP64 -fno-pic -ffast-math -D NDEBUG -ffast-math")
+ set(COMPILE_FLAGS "${COMPILE_FLAGS} -mavx -O3 -mavx -msse2 -msse3 -march=native -mfpmath=sse -DMKL_ILP64 -fno-pic -ffast-math -ffast-math")
elseif (RECKLESS)
set(COMPILE_FLAGS "${COMPILE_FLAGS} -mavx -O3 -mavx -msse2 -msse3 -march=native -mfpmath=sse -DMKL_ILP64 -fno-pic -ffast-math -D NDEBUG -ffast-math -D RECKLESS")
endif()
diff --git a/example.cpp b/example.cpp
@@ -20,7 +20,7 @@ double bench(int batch_sz, int epochs)
auto start = std::chrono::high_resolution_clock::now();
Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 2, 0, 0.9);
net.add_layer(4, "linear");
- net.add_layer(5, "relu");
+ net.add_layer(5, "lecun_tanh");
net.add_layer(2, "linear");
net.initialize();
for (int i = 0; i < epochs; i++) {
@@ -33,29 +33,4 @@ double bench(int batch_sz, int epochs)
int main(int argc, char** argv)
{
std::cout << bench(strtol(argv[1], NULL, 10), strtol(argv[2], NULL, 10)) << "\n";
- // show_console_cursor(false);
- // BlockProgressBar bar{
- // option::BarWidth{80},
- // option::Start{"["},
- // option::End{"]"},
- // option::ForegroundColor{Color::white} ,
- // option::FontStyles{std::vector<FontStyle>{FontStyle::bold}}
- // };
- // Network net ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
- // net.add_layer(4, "linear");
- // net.add_layer(5, "relu");
- // net.add_layer(2, "linear");
- // net.initialize();
- // bar.set_option(option::PostfixText{"Starting train"});
- // for (int i = 0; i < 5000; i++) {
- // char msg[32];
- // sprintf(msg, "Starting epoch %i", i);
- // std::string str(msg);
- // bar.set_option(option::PostfixText{str});
- // net.train();
- // bar.set_progress((float)i/50 * 100);
- // }
- // bar.set_progress(100); // Ensure we are done.
- // std::cout << "\nFinal cost: " << net.get_cost() << " Final validation cost:" << net.get_val_cost() << "\n";
- // show_console_cursor(true);
}
diff --git a/readme.md b/readme.md
@@ -98,7 +98,7 @@ The five main configurations correspond to differing levels of optimization.
- `cmake . -DFAST=ON`: Enables the O3 optimization layer in the compiler.
- `cmake . -DFASTER=ON`: Enables O3 as well as extra individual flags.
- `cmake . -DTRADEOFFS=ON`: All previous optimizations as well as ones that sacrifice precision.
-- `cmake . -DRECKLESS=ON`: Like `TRADEOFFS`, but defines the RECKLESS macro which skips all checks within the code.
+- `cmake . -DRECKLESS=ON`: Like `TRADEOFFS`, but defines the RECKLESS macro (and NDEBUG) which skips all checks within the code.
One you've selected a main optimization level, extra configurations can be passed in.
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -26,7 +26,6 @@
//#include "checks.cpp"
Layer::Layer(int batch_sz, int nodes, float a)
-
:alpha(a)
{
contents = new Eigen::MatrixXf (batch_sz, nodes);
@@ -64,12 +63,17 @@ void Layer::init_weights(Layer next)
Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, int regularization, float l, float ratio, bool early_exit, float cutoff)
:lambda(l), learning_rate(learn_rate), bias_lr(bias_rate), batch_size(batch_sz), reg_type(regularization), early_stop(early_exit), threshold(cutoff)
{
+ assert( > 0 || batch_size < total_instances);
assert(reg_type == 1 || reg_type == 2); // L1 and L2 are only relevant regularizations
- int total_instances = prep_file(path, SHUFFLED_PATH);
- val_instances = split_file(SHUFFLED_PATH, total_instances, ratio);
+ try {
+ int total_instances = prep_file(path, SHUFFLED_PATH);
+ val_instances = split_file(SHUFFLED_PATH, total_instances, ratio);
+ data = fopen(TRAIN_PATH, "r");
+ val_data = fopen(VAL_PATH, "r");
+ }
+ catch (...) std::cout << "Exception occured when loading data file." << "\n";
instances = total_instances - val_instances;
- data = fopen(TRAIN_PATH, "r");
- val_data = fopen(VAL_PATH, "r");
+ assert(batch_size > 0 || batch_size < instances);
decay = [this]() -> void {
learning_rate = learning_rate;
};
@@ -89,26 +93,27 @@ void Network::init_decay(char* type, ...)
learning_rate = a_0 * learning_rate/k;
};
}
- if (strcmp(type, "exp") == 0) {
+ else if (strcmp(type, "exp") == 0) {
float a_0 = va_arg(args, double);
float k = va_arg(args, double);
decay = [this, a_0, k]() -> void {
learning_rate = a_0 * exp(-k * epochs);
};
}
- if (strcmp(type, "frac") == 0) {
+ else if (strcmp(type, "frac") == 0) {
float a_0 = va_arg(args, double);
float k = va_arg(args, double);
decay = [this, a_0, k]() -> void {
learning_rate = a_0 / (1+(k * epochs));
};
}
- if (strcmp(type, "linear") == 0) {
+ else if (strcmp(type, "linear") == 0) {
int max_ep = va_arg(args, double);
decay = [this, max_ep]() -> void {
learning_rate = 1 - epochs/max_ep;
};
}
+ else std::cout << "Invalid << "\n";
va_end(args);
}
@@ -190,7 +195,7 @@ void Network::add_layer(int nodes, char* name)
layers[length-1].activation_deriv = rectifier(sigmoid_deriv);
}
else {
- std::cout << "Warning! Incorrect activation specified. Exiting...\n\nIf this is coming up and you don't know why, try defining your own activation function.\n";
+ std::cout << "Warning! Incorrect activation specified. Exiting.\n";
exit(1);
}
}
@@ -229,7 +234,6 @@ void Network::feedforward()
(*layers[length-1].contents)(j,k) = layers[length-1].activation((*layers[length-1].contents)(j,k));
}
}
- // std::cout << "begin softmax" << "\n";
for (int i = 0; i < layers[length-1].contents->rows(); i++) {
Eigen::MatrixXf m = layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols());
Eigen::MatrixXf::Index maxRow, maxCol;
@@ -239,14 +243,11 @@ void Network::feedforward()
float sum = avx_exp(m).sum();
m = avx_cdiv(avx_exp(m), sum);
#else
- // std::cout << "begin sum" << "\n";
- // std::cout << m << "\n";
float sum = 0;
for (int j = 0; j < layers[length-1].contents->cols(); j++) {
checknan(m(0,j), "input of Softmax operation");
sum += exp(m(0,j));
}
- // std::cout << "begin norm" << "\n";
for (int j = 0; j < layers[length-1].contents->cols(); j++) {
m(0,j) = exp(m(0,j))/sum;
checknan(m(0,j), "output of Softmax operation");
@@ -392,18 +393,11 @@ void Network::backpropagate()
deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
counter++;
}
- // std::cout << deltas[0] << "\n\nNUMERIC\n\n" << numerical_grad(1, 0.000001) <<"\n\n\n-----------\n\n\n";
for (int i = 0; i < length-1; i++) {
update(deltas, i);
- // *layers[length-2-i].weights -= (0.9 * *layers[length-2-i].v) + (learning_rate * deltas[i]);
- // *layers[length-2-i].v = (learning_rate * deltas[i]);
-
if (reg_type == 2) *layers[length-2-i].weights -= ((lambda/batch_size) * (*layers[length-2-i].weights));
else if (reg_type == 1) *layers[length-2-i].weights -= ((lambda/(2*batch_size)) * l1_deriv(*layers[length-2-i].weights));
-
*layers[length-1-i].bias -= bias_lr * gradients[i];
- // std::cout << *layers[length-2-i].v << "\n\n" << *layers[length-2-i].weights << "\n\n" << deltas[i] << "\n\n\n\n";
-
if (strcmp(layers[length-2-i].activation_str, "prelu") == 0) {
float sum = 0;
for (int j = 0; j < layers[length-2-i].contents->rows(); j++) {
@@ -432,9 +426,7 @@ void Network::backpropagate()
void Network::update_layer(float* vals, int datalen, int index)
{
- for (int i = 0; i < datalen; i++) {
- (*layers[index].contents)((int)i / layers[index].contents->cols(),i%layers[index].contents->cols()) = vals[i];
- }
+ for (int i = 0; i < datalen; i++) (*layers[index].contents)((int)i / layers[index].contents->cols(),i%layers[index].contents->cols()) = vals[i];
}
int Network::next_batch()
