commit 3fbc813c2f8a8397bd4d250d4f2d6a4c262e45ab
parent ef8e1d8448ec1dcac13f868ba3e90461ebc7d558
Author: David Freifeld <freifeld.david@gmail.com>
Date: Mon, 3 Aug 2020 18:26:26 -0700
Added early stopping + fixed makefile
Diffstat:
5 files changed, 19 insertions(+), 11 deletions(-)
diff --git a/Makefile b/Makefile
@@ -12,7 +12,7 @@
GEN_FLAGS = -fpic
CXX = /usr/local/opt/llvm/bin/clang
-CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/mr_bpnn_2.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -o mrbpnn`python3-config --extension-suffix`
+CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/pybind.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -o mrbpnn`python3-config --extension-suffix`
LDFLAGS = -L/usr/local/opt/llvm/lib -lc++
all: compile
@@ -26,11 +26,11 @@ test: compile
fast: CXXFLAGS += $(GEN_FLAGS) -O3
fast: compile
-faster: CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/mr_bpnn_2.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -o mrbpnn`python3-config --extension-suffix` -O3 -mavx -msse2 -msse3 -fopenmp -march=native -mfpmath=sse -fno-pic -DMKL_ILP64 -D NDEBUG -D EIGEN_USE_BLAS ${MKLROOT}/lib/libmkl_intel_ilp64.a -m64 -I${MKLROOT}/include ${MKLROOT}/lib/libmkl_intel_thread.a ${MKLROOT}/lib/libmkl_core.a -liomp5 -lpthread -lm -ldl
+faster: CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/pybind.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -o mrbpnn`python3-config --extension-suffix` -O3 -mavx -msse2 -msse3 -fopenmp -march=native -mfpmath=sse -fno-pic -DMKL_ILP64 -D NDEBUG -D EIGEN_USE_BLAS ${MKLROOT}/lib/libmkl_intel_ilp64.a -m64 -I${MKLROOT}/include ${MKLROOT}/lib/libmkl_intel_thread.a ${MKLROOT}/lib/libmkl_core.a -liomp5 -lpthread -lm -ldl
faster: LDFLAGS = -lpthread -lm -ldl -lblas
faster: compile
-tradeoffs: CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/mr_bpnn_2.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -o mrbpnn`python3-config --extension-suffix` -O3 -mavx -msse2 -msse3 -march=native -mfpmath=sse -DMKL_ILP64 -fno-pic -ffast-math -D NDEBUG #-qopt-report=5 -qopt-report-file=report
+tradeoffs: CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/pybind.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -o mrbpnn`python3-config --extension-suffix` -O3 -mavx -msse2 -msse3 -march=native -mfpmath=sse -DMKL_ILP64 -fno-pic -ffast-math -D NDEBUG #-qopt-report=5 -qopt-report-file=report
tradeoffs: compile
diff --git a/example.cpp b/example.cpp
@@ -9,6 +9,7 @@
#include <indicators/progress_bar.hpp>
#include <indicators/block_progress_bar.hpp>
using namespace indicators;
+
#include "./src/bpnn.hpp"
#include "./src/utils.hpp"
#include "unistd.h"
@@ -39,18 +40,21 @@ int main()
option::ForegroundColor{Color::white} ,
option::FontStyles{std::vector<FontStyle>{FontStyle::bold}}
};
- Network net ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9);
+ Network net ("./data_banknote_authentication.txt", 16, 0.0155, 0.03, 2, 0, 0.9, true, 0.0001);
net.add_layer(4, "linear");
net.add_layer(5, "relu");
net.add_layer(2, "linear");
net.initialize();
bar.set_option(option::PostfixText{"Starting train"});
for (int i = 0; i < 5000; i++) {
- net.train();
char msg[32];
- sscanf(msg, "Finished epoch %i", i);
+ sprintf(msg, "Starting epoch %i", i);
std::string str(msg);
- bar.set_option(option::PostfixText{"#po0fso"});
+ bar.set_option(option::PostfixText{str});
+ net.train();
bar.set_progress((float)i/5000 * 100);
}
+ bar.set_progress(100); // Ensure we are done.
+ std::cout << "\nFinal cost: " << net.get_cost() << " Final validation cost:" << net.get_val_cost() << "\n";
+ show_console_cursor(true);
}
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -54,8 +54,8 @@ void Layer::init_weights(Layer next)
}
}
-Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, int regularization, float l, float ratio)
- :lambda(l), learning_rate(learn_rate), bias_lr(bias_rate), batch_size(batch_sz), reg_type(regularization)
+Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, int regularization, float l, float ratio, bool early_exit, float cutoff)
+ :lambda(l), learning_rate(learn_rate), bias_lr(bias_rate), batch_size(batch_sz), reg_type(regularization), early_stop(early_exit), threshold(cutoff)
{
assert(reg_type == 1 || reg_type == 2); // L1 and L2 are only relevant regularizations
int total_instances = prep_file(path, SHUFFLED_PATH);
@@ -536,6 +536,7 @@ void Network::train()
float cost_sum = 0;
float acc_sum = 0;
for (int i = 0; i <= instances-batch_size; i+=batch_size) {
+ if (early_stop == true && get_val_cost() < threshold) return;
if (i != instances-batch_size) { // Don't try to advance batch on final batch.
next_batch();
}
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -68,7 +68,7 @@ public:
std::function<void(std::vector<Eigen::MatrixXf>, int, int)> grad_calc;
std::function<void(std::vector<Eigen::MatrixXf>, int)> update;
- Network(char* path, int batch_sz, float learn_rate, float bias_rate, int regularization, float l, float ratio);
+ Network(char* path, int batch_sz, float learn_rate, float bias_rate, int regularization, float l, float ratio, bool early_exit=true, float cutoff=0);
void add_layer(int nodes, char* activation);
void add_prelu_layer(int nodes, float a);
void init_decay(char* type, ...);
@@ -80,6 +80,9 @@ public:
void feedforward();
void list_net();
+ bool early_stop;
+ float threshold;
+
float cost();
float accuracy();
void backpropagate();
diff --git a/src/pybind.cpp b/src/pybind.cpp
@@ -14,7 +14,7 @@ PYBIND11_MODULE(mrbpnn, m) {
m.doc() = "Fast machine learning in C++"; // optional module docstring
py::class_<Network>(m, "Network")
- .def(py::init<char*, int, float, float, int, float, float>())
+ .def(py::init<char*, int, float, float, int, float, float, bool, float>())
.def("add_layer", &Network::add_layer, py::arg("nodes"), py::arg("activation"))
.def("add_prelu_layer", &Network::add_prelu_layer, py::arg("nodes"), py::arg("a"))
.def("initialize", &Network::initialize)
