jacobian

commit 0941c3bff9e747ff12392e1040712bb7ef31cba3
parent 4dce7237a77b0153ce67d52849f503245107a8a0
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Thu,  6 Aug 2020 00:18:41 -0700

Fixes + AVX exp

Diffstat:
M
src/bpnn.cpp
 | 
12
+-----------
M
src/utils.cpp
 | 
28
+++++++++++++---------------
M
src/utils.hpp
 | 
1
+

3 files changed, 15 insertions(+), 26 deletions(-)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -66,10 +66,6 @@ Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, in
     decay = [this]() -> void {
         learning_rate = learning_rate;
     };
-    grad_calc = [this](std::vector<Eigen::MatrixXf> gradients, int i, int counter) -> void {
-        //gradients.push_back(avx_product(gradients[counter-1] * layers[i].weights->transpose(), *layers[i].dZ));
-        gradients.push_back((gradients[counter-1] * layers[i].weights->transpose()).cwiseProduct(*layers[i].dZ));
-    };
     update = [this](std::vector<Eigen::MatrixXf> deltas, int i) {
         *layers[length-2-i].weights -= (learning_rate * deltas[i]);
     };
@@ -229,18 +225,12 @@ void Network::feedforward()
     }
     //  std::cout << "\nSOFTMAX INPUT\n" << *layers[length-1].contents << "\n\n";
     for (int i = 0; i < layers[length-1].contents->rows(); i++) {
-        float sum = 0;
         Eigen::MatrixXf m = layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols());
         Eigen::MatrixXf::Index maxRow, maxCol;
         float max = m.maxCoeff(&maxRow, &maxCol);
         m = (m.array() - max).matrix();
         // std::cout << "\nGETTING SUM\n";
-        for (int j = 0; j < layers[length-1].contents->cols(); j++) {
-            checknan(m(0,j), "input to final layer");
-            sum += exp(m(0,j));
-            // std::cout << "Adding " << exp(m(0,j)) << "(aka e^"<< m(0, j) << ")\n";
-            checknan(sum, "sum in Softmax operation");
-        }
+        float sum = avx_exp(m).sum();
         //std::cout << "\nFINAL ACTIVATION\n";
         for (int j = 0; j < layers[length-1].contents->cols(); j++) {
             m(0,j) = exp(m(0,j))/sum;
diff --git a/src/utils.cpp b/src/utils.cpp
@@ -89,7 +89,6 @@ std::function<float(float)> rectifier(float (*activation)(float))
 }
 
 // Intel intrinsics for the win!
-// TODO: Investigate weird memory problems!
 Eigen::MatrixXf avx_product(Eigen::MatrixXf a, Eigen::MatrixXf b)
 {
 #ifndef RECKLESS
@@ -97,25 +96,24 @@ Eigen::MatrixXf avx_product(Eigen::MatrixXf a, Eigen::MatrixXf b)
 #endif
     int size = ((a.rows() * a.cols()) + 7) & (-8);
     float arr1[size];
-    std::cout << a.cols()*a.rows() <<"\n";
     memcpy(arr1, a.data(), sizeof(float)*a.cols()*a.rows());    
     float arr2[size];
     memcpy(arr2, b.data(), sizeof(float)*b.cols()*b.rows());
-    for (int i = 0; i < a.cols()*a.rows(); i++) std::cout << arr1[i] << " ";
-    std::cout << "\n";
-    for (int i = 0; i < a.cols()*a.rows(); i++) std::cout << arr2[i] << " ";
-    std::cout << "\n\n";
     for (int i = 0; i < size/8; i++) {
-        __m256 product = _mm256_mul_ps(_mm256_load_ps(arr1+i*8), _mm256_load_ps(arr2+i*8));
-        std::cout << "Product:\n";
-        for (int i = 0; i < 8; i++) std::cout << product[i] << " ";
-        std::cout << "\n";
-        _mm256_store_ps(arr1, product);
+        _mm256_store_ps(arr1+i*8, _mm256_mul_ps(_mm256_load_ps(arr1+i*8), _mm256_load_ps(arr2+i*8)));
     }
-    std::cout << "Product (final):\n";
-    for (int i = 0; i < a.cols()*a.rows(); i++) std::cout << arr1[i] << " ";
-    std::cout << "\n\n\n\n";
     Eigen::Map<Eigen::MatrixXf> dst (arr1, a.rows(), a.cols());
-    //std::cout << a << "\n\n" <<  b << "\n\n" << dst << "\n\n\n\n";
+    return dst;
+}
+
+Eigen::MatrixXf avx_exp(Eigen::MatrixXf m)
+{
+    int size = ((m.rows() * m.cols()) + 7) & (-8);
+    float arr1[size];
+    memcpy(arr1, m.data(), sizeof(float)*m.cols()*m.rows());    
+    for (int i = 0; i < size/8; i++) {
+        _mm256_store_ps(arr1+i*8, _mm256_exp_ps(_mm256_load_ps(arr1+i*8)));
+    }
+    Eigen::Map<Eigen::MatrixXf> dst (arr1, m.rows(), m.cols());
     return dst;
 }
diff --git a/src/utils.hpp b/src/utils.hpp
@@ -39,6 +39,7 @@ float leaky_relu_deriv(float x);
 std::function<float(float)> rectifier(float (*activation)(float));
 
 Eigen::MatrixXf avx_product(Eigen::MatrixXf a, Eigen::MatrixXf b);
+Eigen::MatrixXf avx_exp(Eigen::MatrixXf m);
 
 Eigen::MatrixXf strassen_mul(Eigen::MatrixXf a, Eigen::MatrixXf b);