jacobian

commit 761debbb9ef1e9cc9c2c5f86b07a4f90ed9b204d
parent 7a768c84a15b1b8c178cac3f12b3a02350aa2d14
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sun, 28 Mar 2021 19:23:50 -0700

Port decay functions and respective python bindings

Diffstat:
M
src/bpnn.cpp
 | 
44
++++++++++++++++++++++++++++++++++++++++++--
M
src/bpnn.hpp
 | 
12
+++++++++---
M
src/pybind.cpp
 | 
5
+++++

3 files changed, 56 insertions(+), 5 deletions(-)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -55,7 +55,7 @@ Network::Network(const char* path, int batch_sz, float learn_rate, float bias_ra
     data = open(TRAIN_BIN_PATH, O_RDONLY | O_NONBLOCK);
     val_data = open(VAL_BIN_PATH, O_RDONLY | O_NONBLOCK);
     instances = total_instances - val_instances;
-    decay = []() -> void {};
+    decay = [](float& learning_rate) -> void {};
     update = [](const Layer& layer, const Eigen::MatrixXf delta, const float learning_rate) {
         *layer.weights -= (learning_rate * delta);
     };
@@ -131,6 +131,46 @@ void Network::feedforward()
     softmax();
 }
 
+std::function<void(float&)> decays::step(float a_0, float k)
+{
+    return [a_0, k](float& learning_rate) -> void {
+        learning_rate = a_0 * learning_rate/k;
+    };
+}
+
+std::function<void(float&)> decays::exponential(float a_0, float k)
+{
+    int epochs = 0;
+    return [a_0, k, epochs](float& learning_rate) mutable -> void {
+        learning_rate = a_0 * exp(-k * epochs);
+        epochs++;
+    };
+}
+
+std::function<void(float&)> decays::fractional(float a_0, float k)
+{
+    int epochs = 0;
+    return [a_0, k, epochs](float& learning_rate) mutable -> void {
+        learning_rate = a_0 / (1+(k * epochs));
+        epochs++;
+    };
+}
+
+std::function<void(float&)> decays::linear(int max_ep)
+{
+    int epochs = 0;
+    return [max_ep, epochs](float& learning_rate) mutable -> void {
+        learning_rate = 1 - epochs/max_ep;
+        epochs++;
+    };
+}
+
+
+void Network::init_decay(std::function<void(float&)> f)
+{
+    decay = f;
+}
+
 void Network::list_net()
 {
     Expects(length > 1);
@@ -272,7 +312,7 @@ void Network::train()
     if (silenced == false) printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n", epochs, epoch_cost, epoch_acc, val_cost, val_acc);
     batches=1;
     data = open(TRAIN_BIN_PATH, O_RDONLY | O_NONBLOCK);
-    decay();
+    decay(learning_rate);
     epochs++;
     Ensures(lseek(data, 0, SEEK_CUR) == 0);
 }
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -43,7 +43,7 @@ protected:
     float epoch_cost;
     float val_acc;
     float val_cost;
-    std::function<void(void)> decay;
+    std::function<void(float&)> decay;
     std::function<void(std::vector<Eigen::MatrixXf>, int, int)> grad_calc;
     std::function<void(Layer&, Eigen::MatrixXf, float)> update;
     void next_batch(int fd);
@@ -73,6 +73,7 @@ public:
     void add_layer(int nodes, std::function<float(float)> activation, std::function<float(float)> activation_deriv);
     void initialize();
     void init_optimizer(std::function<void(Layer&, Eigen::MatrixXf, float)> f);
+    void init_decay(std::function<void(float&)> f);
     void set_activation(int index, std::function<float(float)> custom, std::function<float(float)> custom_deriv);
     void feedforward();
     void softmax();
@@ -102,6 +103,13 @@ std::function<void(Layer&, Eigen::MatrixXf, float)> adam(float beta1, float beta
 std::function<void(Layer&, Eigen::MatrixXf, float)> adamax(float beta1, float beta2, float epsilon);
 }
 
+namespace decays {
+std::function<void(float&)> step(float a_0, float k);
+std::function<void(float&)> exponential(float a_0, float k);
+std::function<void(float&)> fractional(float a_0, float k);
+std::function<void(float&)> linear(int max_ep);
+}
+
 #define MAXLINE 1024
 
 #if (!RECKLESS)
@@ -119,7 +127,5 @@ std::function<void(Layer&, Eigen::MatrixXf, float)> adamax(float beta1, float be
 #define TRAIN_PATH "./train.txt"
 #define VAL_BIN_PATH "./test.bin"
 #define TRAIN_BIN_PATH "./train.bin"
-#define VAL_LZ4_PATH "./test.lz4"
-#define TRAIN_LZ4_PATH "./train.lz4"
 
 #endif /* MODULE_H */
diff --git a/src/pybind.cpp b/src/pybind.cpp
@@ -33,6 +33,7 @@ PYBIND11_MODULE(_jacobian, m)
              py::arg("activation"), py::arg("activation_deriv"))
         .def("initialize", &Network::initialize)
         .def("init_optimizer", &Network::init_optimizer, py::arg("optimizer"))
+        .def("init_decay", &Network::init_decay, py::arg("decay"))
         .def("set_activation", &Network::set_activation,
              py::arg("index"), py::arg("custom"),
              py::arg("custom_deriv"))
@@ -54,4 +55,8 @@ PYBIND11_MODULE(_jacobian, m)
     m.def("demon", &optimizers::demon, py::arg("beta"), py::arg("max_ep"));
     m.def("adam", &optimizers::adam, py::arg("beta1"), py::arg("beta2"), py::arg("epsilon"));
     m.def("adamax", &optimizers::adamax, py::arg("beta1"), py::arg("beta2"), py::arg("epsilon"));
+    m.def("step", &decays::step, py::arg("a_0"), py::arg("k"));
+    m.def("exponential", &decays::exponential, py::arg("a_0"), py::arg("k"));
+    m.def("fractional", &decays::fractional, py::arg("a_0"), py::arg("k"));
+    m.def("linear", &decays::linear, py::arg("max_ep"));
 }