jacobian

commit 9dfb33b3ee210f411086cf4241cff13b89cc35e5
parent a307431fa605fefd2ba3b4a7cfca9514a8ee4949
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Thu,  6 Aug 2020 21:42:03 -0700

Added ability to toggle explicit AVX functions

Diffstat:
M
example.cpp
 | 
2
+-
M
src/bpnn.cpp
 | 
34
+++++++++++++++++++++-------------
M
src/utils.cpp
 | 
1
+

3 files changed, 23 insertions(+), 14 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -32,7 +32,7 @@ double bench(int batch_sz)
 
 int main()
 {
-    std::cout << bench(16) << "\n";
+    std::cout << bench(10) << "\n";
     // show_console_cursor(false);
     // BlockProgressBar bar{
     //   option::BarWidth{80},
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -17,6 +17,12 @@
 #define VAL_PATH "./test.txt"
 #define TRAIN_PATH "./train.txt"
 
+#if (AVX)
+#define cwise_product(a,b) avx_product(a, b)
+#else
+#define cwise_product(a,b) (a).cwiseProduct(b)
+#endif
+
 //#include "checks.cpp"
 
 Layer::Layer(int batch_sz, int nodes, float a)
@@ -217,30 +223,32 @@ void Network::feedforward()
     }
     for (int j = 0; j < layers[length-1].contents->rows(); j++) {
         if (strcmp(layers[length-1].activation_str, "linear") == 0) break;
-#pragma omp simd
         for (int k = 0; k < layers[length-1].contents->cols(); k++) {
             (*layers[length-1].dZ)(j,k) = layers[length-1].activation_deriv((*layers[length-1].contents)(j,k));
             (*layers[length-1].contents)(j,k) = layers[length-1].activation((*layers[length-1].contents)(j,k));
         }
     }
-    //  std::cout << "\nSOFTMAX INPUT\n" << *layers[length-1].contents << "\n\n";
     for (int i = 0; i < layers[length-1].contents->rows(); i++) {
         Eigen::MatrixXf m = layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols());
         Eigen::MatrixXf::Index maxRow, maxCol;
         float max = m.maxCoeff(&maxRow, &maxCol);
         m = (m.array() - max).matrix();
-        // std::cout << "\nGETTING SUM\n";
+#if (AVX)
         float sum = avx_exp(m).sum();
-        //std::cout << "\nFINAL ACTIVATION\n";
-        // for (int j = 0; j < layers[length-1].contents->cols(); j++) {
-        //     m(0,j) = exp(m(0,j))/sum;
-        //     //      std::cout << "Calculating " << exp(m(0,j)) << "/" << sum << " to be " << (*layers[length-1].contents)(i,j) << "(aka " << test<<")\n";
-        //     checknan(m(0,j), "output of Softmax operation");
-        // }
         m = avx_cdiv(avx_exp(m), sum);
+#else
+        float sum = 0;
+        for (int j = 0; j < layers[length-1].contents->cols(); j++) {
+            checknan(m(0,j), "input of Softmax operation");
+            sum += exp(m(0,j));
+        }
+        for (int j = 0; j < layers[length-1].contents->cols(); j++) {
+            m(0,j) = exp(m(0,j))/sum;
+            checknan(m(0,j), "output of Softmax operation");
+        }
+#endif
         layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols()) = m;
     }
-    // std::cout << "\n\n";
 }
 
 void Network::list_net()
@@ -272,7 +280,7 @@ float Network::cost()
         checknan(tempsum, "total summation inside cost calculation");
     }
     for (int i = 0; i < layers.size()-1; i++) {
-        if (reg_type == 2) reg += avx_product(*layers[i].weights,*layers[i].weights).sum();
+        if (reg_type == 2) reg += cwise_product(*layers[i].weights,*layers[i].weights).sum();
         else if (reg_type == 1) reg += (layers[i].weights->array().abs().matrix()).sum();
     }
     return ((1.0/batch_size) * sum) + (1/2*lambda*reg);
@@ -347,7 +355,7 @@ void Network::grad_check()                      \
     gradients.push_back(error);
     deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
     for (int i = length-2; i >= 1; i--) {
-        gradients.push_back(avx_product(gradients[counter-1] * layers[i].weights->transpose(),*layers[i].dZ));
+        gradients.push_back(cwise_product(gradients[counter-1] * layers[i].weights->transpose(),*layers[i].dZ));
         std::cout << layers[i-1].contents->transpose() * gradients[counter];
         deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
         counter++;
@@ -375,7 +383,7 @@ void Network::backpropagate()
         // TODO: Find nice way to add this
         // (*layers[i].weights-((learning_rate * *layers[i].weights) + (0.9 * *layers[i].v))).transpose()
         //grad_calc(gradients, counter, i);
-        gradients.push_back(avx_product(gradients[counter-1] * layers[i].weights->transpose(), *layers[i].dZ));
+        gradients.push_back(cwise_product(gradients[counter-1] * layers[i].weights->transpose(), *layers[i].dZ));
         deltas.push_back(layers[i-1].contents->transpose() * gradients[counter]);
         counter++;
     }
diff --git a/src/utils.cpp b/src/utils.cpp
@@ -99,6 +99,7 @@ Eigen::MatrixXf avx_product(Eigen::MatrixXf a, Eigen::MatrixXf b)
         _mm256_store_ps(a.data()+i*8, _mm256_mul_ps(_mm256_load_ps(a.data()+i*8), _mm256_load_ps(b.data()+i*8)));
     }
     for (int i = size-8; i < a.cols()*a.rows(); i++) *(a.data()+i) = *(a.data()+i) * *(b.data()+i);
+    b = a.cwiseProduct(b);
     return a;
 }