jacobian

commit 9b40262219fb576b8b7c887810334d9575458332
parent 4032d3c1402eedf8412842dc8c7a25936794a302
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sun, 28 Jun 2020 17:43:05 -0700

Adagrad struggles

Diffstat:
M
bpnn.cpp
 | 
20
+++++++++++++++-----
M
bpnn.hpp
 | 
1
+
M
example.cpp
 | 
4
++--
M
example.py
 | 
4
++--

4 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/bpnn.cpp b/bpnn.cpp
@@ -40,6 +40,7 @@ Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate)
   bias_lr = bias_rate;
   instances = prep_file(path, "./shuffled.txt");
   length = 0;
+  t = 0;
   batch_size = batch_sz;
   data = fopen("./shuffled.txt", "r");
   batches = 0;
@@ -172,12 +173,20 @@ void Network::backpropagate()
   }
   for (int i = 0; i < length-1; i++) {
     Eigen::MatrixXd gradient = gradients[i];
-    Eigen::MatrixXd sum (layers[length-2-i].weights->rows(), layers[length-2-i].weights->cols());
-    for (int j = 0; j < layers[length-2-i].prev_updates.size(); j++) {
-      sum = sum + layers[length-2-i].prev_updates[j].cwiseProduct(layers[length-2-i].prev_updates[j]); 
+    std::cout << t <<"\n";
+    if (t > 0) {
+      Eigen::MatrixXd sum (layers[length-2-i].weights->rows(), layers[length-2-i].weights->cols());
+      for (int j = 0; j < layers[length-2-i].prev_updates.size(); j++) {
+        sum = sum + layers[length-2-i].prev_updates[j]; 
+      }
+      layers[length-2-i].prev_updates.emplace_back(((1/learning_rate) * sum.cwiseSqrt()).cwiseProduct(deltas[i]));
+      std::cout << layers[length-2-i].prev_updates[layers[length-2-i].prev_updates.size()-1] << "\n\nSUM\n\n" << sum << "\n\n\n";
+      *layers[length-2-i].weights -= layers[length-2-i].prev_updates[layers[length-2-i].prev_updates.size()-1];
+    }
+    else {
+      std::cout << "AAAA" << learning_rate * deltas[i] << "\n\n";
+      *layers[length-2-i].weights -= learning_rate * deltas[i];
     }
-    layers[length-2-i].prev_updates.emplace_back(((1/learning_rate) * sum.cwiseSqrt()).transpose() * deltas[i]);
-   *layers[length-2-i].weights -= layers[length-2-i].prev_updates[layers[length-2-i].prev_updates.size()];
     *layers[length-1-i].bias -= bias_lr * gradients[i];
   }
 }
@@ -295,6 +304,7 @@ void Network::train(int total_epochs)
       cost_sum += cost();
       acc_sum += accuracy();
       batches++;
+      t++;
     }
     epoch_accuracy = 1.0/((float) instances/batch_size) * acc_sum;
     epoch_cost = 1.0/((float) instances/batch_size) * cost_sum;
diff --git a/bpnn.hpp b/bpnn.hpp
@@ -36,6 +36,7 @@ public:
 
   std::vector<Layer> layers;
   int length;
+  int t;
 
   float learning_rate;
   float bias_lr;
diff --git a/example.cpp b/example.cpp
@@ -11,8 +11,8 @@ int main()
   net.add_layer(1, "resig");
   net.initialize();
   //  net.list_net();
-  net.train(50);
-  net.list_net();
+  net.train(1);
+  //  net.list_net();
   //printf("%i\n", wc("./data_banknote_authentication.txt"));
 
   // double x = 0.4235;
diff --git a/example.py b/example.py
@@ -4,13 +4,13 @@ import time
 
 def bench():
     init = time.time()
-    net = mrbpnn.Network("./data_banknote_authentication.txt", 10, 0.0155, 0.03);
+    net = mrbpnn.Network("./extra.txt", 10, 0.001, 0.03);
     net.add_layer(4, "linear");
     net.add_layer(5, "relu");
     net.add_layer(1, "resig");
     net.initialize();
     initend = time.time()
-    net.train(50);
+    net.train(1);
     end = time.time()
     return (end-init)
 #    print("%s: init %s" % (end-init, initend-init))