jacobian

commit e056a6ba7f7170df3b6ee10696ca25acd6409fed
parent a6b10cf43deeda18f31b71984bc892dec95d7cc9
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Thu, 30 Jul 2020 15:17:06 -0700

PReLU now supported (Python is buggy tho)

Diffstat:
M
example.cpp
 | 
2
+-
M
src/bpnn.cpp
 | 
9
+++------
M
src/mr_bpnn_2.cpp
 | 
1
+

3 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -16,7 +16,7 @@ double bench(int batch_sz)
   auto start = std::chrono::high_resolution_clock::now();
   Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 0, 0.9);
   net.add_layer(4, "linear");
-  net.add_prelu_layer(5, 0.01);
+  net.add_layer(5, "relu");
   net.add_layer(2, "linear");
   net.initialize();
   for (int i = 0; i < 50; i++) {
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -302,16 +302,14 @@ void Network::backpropagate()
     *layers[length-1-i].bias -= bias_lr * gradients[i];
     if (strcmp(layers[length-2-i].activation_str, "prelu") == 0) {
       float sum = 0;
-      std::cout << "\n\nGRAD:\n\n" << gradients[i] << "\n\n\n";
       for (int j = 0; j < layers[length-2-i].contents->rows(); j++) {
         for (int k = 0; k < layers[length-2-i].contents->cols(); k++) {
           if ((*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha <= 0) {
-            sum += gradients[i](j,k) * (*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha;
-            std::cout << "SUM += " << gradients[i](j,k) << " * " << (*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha << "\n";
+            // Choice of using index i+1 here is questionable. TODO: REVIEW
+            sum += gradients[i+1](j,k) * (*layers[length-2-i].contents)(j,k)/layers[length-2-i].alpha;
           }
         }
       }
-      std::cout << "SUM: " << sum << "\n";
       layers[length-2-i].alpha += learning_rate * sum;
       float a = layers[length-2-i].alpha;
       layers[length-2-i].activation = [a](float x) -> float
@@ -447,12 +445,11 @@ void Network::train()
     cost_sum += cost();
     acc_sum += accuracy();
     batches++;
-    list_net();
     // if (i > batch_size * 10) {
     //   list_net();
     //   exit(1);
     // }
-    layers[10000000].alpha = 2;
+    //    layers[10000000].alpha = 2;
   }
   epoch_acc = 1.0/((float) instances/batch_size) * acc_sum;
   epoch_cost = 1.0/((float) instances/batch_size) * cost_sum;
diff --git a/src/mr_bpnn_2.cpp b/src/mr_bpnn_2.cpp
@@ -118,6 +118,7 @@ PYBIND11_MODULE(mrbpnn, m) {
   py::class_<Network>(m, "Network")
     .def(py::init<char*, int, float, float, float, float>())
     .def("add_layer", &Network::add_layer, py::arg("nodes"), py::arg("activation"))
+    .def("add_prelu_layer", &Network::add_prelu_layer, py::arg("nodes"), py::arg("a"))
     .def("initialize", &Network::initialize)
     .def("init_decay", &Network::init_decay, py::arg("type"), py::arg("a_0"), py::arg("k"))
     .def("set_activation", &Network::set_activation)