jacobian

commit 306bd1e7e000e882c583cd347abaf140d507492f
parent a2d4396fe7ec2feb606162f0201b8c725bef7970
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Wed,  5 Aug 2020 17:20:14 -0700

Copious amounts of errors

Diffstat:
M
src/bpnn.cpp
 | 
11
+++++------
M
src/utils.cpp
 | 
9
++++++---
M
src/utils.hpp
 | 
2
+-

3 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -67,7 +67,7 @@ Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, in
         learning_rate = learning_rate;
     };
     grad_calc = [this](std::vector<Eigen::MatrixXf> gradients, int i, int counter) -> void {
-        gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
+        gradients.push_back(avx_product(gradients[counter-1] * layers[i].weights->transpose(), *layers[i].dZ));
     };
     update = [this](std::vector<Eigen::MatrixXf> deltas, int i) {
         *layers[length-2-i].weights -= (learning_rate * deltas[i]);
@@ -236,7 +236,7 @@ void Network::feedforward()
         // std::cout << "\nGETTING SUM\n";
         for (int j = 0; j < layers[length-1].contents->cols(); j++) {
             checknan(m(0,j), "input to final layer");
-            sum += exp(m(0,j));
+            sum += exp(m(0,j));.
             // std::cout << "Adding " << exp(m(0,j)) << "(aka e^"<< m(0, j) << ")\n";
             checknan(sum, "sum in Softmax operation");
         }
@@ -280,7 +280,7 @@ float Network::cost()
         checknan(tempsum, "total summation inside cost calculation");
     }
     for (int i = 0; i < layers.size()-1; i++) {
-        if (reg_type == 2) reg += (layers[i].weights->cwiseProduct(*layers[i].weights)).sum();
+        if (reg_type == 2) reg += avx_product(*layers[i].weights,*layers[i].weights).sum();
         else if (reg_type == 1) reg += (layers[i].weights->array().abs().matrix()).sum();
     }
     return ((1.0/batch_size) * sum) + (1/2*lambda*reg);
@@ -355,7 +355,7 @@ void Network::grad_check()                      \
     gradients.push_back(error);
     deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
     for (int i = length-2; i >= 1; i--) {
-        gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
+        gradients.push_back(avx_product(gradients[counter-1] * layers[i].weights->transpose(),*layers[i].dZ));
         std::cout << layers[i-1].contents->transpose() * gradients[counter];
         deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
         counter++;
@@ -383,7 +383,6 @@ void Network::backpropagate()
         // TODO: Find nice way to add this
         // (*layers[i].weights-((learning_rate * *layers[i].weights) + (0.9 * *layers[i].v))).transpose()
         grad_calc(gradients, counter, i);
-        gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
         deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
         counter++;
     }
@@ -552,7 +551,7 @@ void Network::train()
     epoch_acc = 1.0/((float) instances/batch_size) * acc_sum;
     epoch_cost = 1.0/((float) instances/batch_size) * cost_sum;
     validate(VAL_PATH);
-    //printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n", epochs, epoch_cost, epoch_acc, val_cost, val_acc);
+    printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n", epochs, epoch_cost, epoch_acc, val_cost, val_acc);
     batches=1;
     rewind(data);
     decay();
diff --git a/src/utils.cpp b/src/utils.cpp
@@ -88,17 +88,20 @@ std::function<float(float)> rectifier(float (*activation)(float))
 }
 
 // Intel intrinsics for the win!
+// TODO: Investigate weird memory problems!
 Eigen::MatrixXf avx_product(Eigen::MatrixXf a, Eigen::MatrixXf b)
 {
 #ifndef RECKLESS
     assert(a.rows() == b.rows() && a.cols() == b.rows());
 #endif
-    float arr1[(((a.rows() * a.cols()) % 8) * 8) + 8] = a.data();
-    float arr2[(((b.rows() * b.cols()) % 8) * 8) + 8] = b.data();
+    float arr1[(((a.rows() * a.cols()) % 8) * 8) + 8];
+    memcpy(arr1, a.data(), sizeof(float)*a.cols()*a.rows());    
+    float arr2[(((b.rows() * b.cols()) % 8) * 8) + 8];
+    memcpy(arr1, b.data(), sizeof(float)*b.cols()*a.rows());    
     for (int i = 0; i < (((a.rows() * a.cols()) % 8) * 8) + 8; i++) {
         _mm256_store_ps(arr1, _mm256_mul_ps(_mm256_load_ps(arr1+i*8),
                                             _mm256_load_ps(arr2+i*8)));
     }
-    Eigen::MatrixXf dst (vec1, a.rows(), a.cols());
+    Eigen::Map<Eigen::MatrixXf> dst (arr1, a.rows(), a.cols());
     return dst;
 }
diff --git a/src/utils.hpp b/src/utils.hpp
@@ -8,7 +8,7 @@
 #ifndef UTILS_H
 #define UTILS_H
 
-#include <functional
+#include <functional>
 #include <immintrin.h>
 
 // A zoo of activation functions.