jacobian

commit 4d4c745405fa76bf09a606336bf60e4f8164ea38
parent 9dfb33b3ee210f411086cf4241cff13b89cc35e5
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Thu,  6 Aug 2020 23:18:17 -0700

Addressed change of plans + tweaks

Diffstat:
M
example.cpp
 | 
4
++--
M
readme.md
 | 
4
+++-
M
src/bpnn.cpp
 | 
5
+++++
M
src/utils.cpp
 | 
4
+++-

4 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/example.cpp b/example.cpp
@@ -23,7 +23,7 @@ double bench(int batch_sz)
     net.add_layer(5, "relu");
     net.add_layer(2, "linear");
     net.initialize();
-    for (int i = 0; i < 50; i++) {
+    for (int i = 0; i < 1000; i++) {
         net.train();
     }
     auto end = std::chrono::high_resolution_clock::now();
@@ -32,7 +32,7 @@ double bench(int batch_sz)
 
 int main()
 {
-    std::cout << bench(10) << "\n";
+    std::cout << bench(16) << "\n";
     // show_console_cursor(false);
     // BlockProgressBar bar{
     //   option::BarWidth{80},
diff --git a/readme.md b/readme.md
@@ -12,7 +12,7 @@ Jacobian is a work-in-progress machine learning library written in C++ designed 
 ## The Chopping Block
 What's next for Jacobian?
 
-1. **Full AVX support.** This will make time-consuming parts of the code like activation functions (most notably softmax) and coefficient-wise multiplications 2-4x faster. This will only work on processors supporting AVX instructions.
+1. **Polish.** Preparing for an 1.0 release and ensuring Jacobian actually works well and can be reliably used are on the agenda. This would comprise of proper gradient and sanity checks, testing Jacobian on a variety of classification problems, lots of bugfixes, and starting to prioritize stability of the master branch (i.e. consistently using branches for new features, using Github Workflows for automated checks, improving the build system with CMake...).
 
 2. **Usable CNNs.** CNNs have been supported for a bit now, but in their current state have not been very practical or stable. Fixes and additions to the CNN backprop, proper pooling layer support, and tensors will enable CNNs to behave more like their stable ANN brethren. Expect small CNN-oriented features as well.
 
@@ -22,6 +22,8 @@ What's next for Jacobian?
 
 **Some smaller features** coming up may include a proper Python install via `pip`, more complex gradient descent optimizers, proper sanity/gradient checks, and more in-depth benchmarks.
 
+**Shelved right now** is AVX support (and more explicit SIMD calls), as it seems that the O3 optimization layer does a better job of this on its own. Lower level optimizations as a whole are on pause as investigations into which types of optimizations are valuable as well as reliable methods of benchmarking small changes in speed are required. 
+
 ## Benchmarks
 
 One of the tradeoffs of Jacobian is that as of now it doesn't train nearly as close to perfection as other available libraries nor does it maximize accuracy as much (with the benefit being the added speed). Here's a graph of Jacobian's model metrics over epochs on a simple task (banknote dataset with batch size 16) as compared to other libraries.
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -26,6 +26,7 @@
 //#include "checks.cpp"
 
 Layer::Layer(int batch_sz, int nodes, float a)
+    
     :alpha(a)
 {
     contents = new Eigen::MatrixXf (batch_sz, nodes);
@@ -228,6 +229,7 @@ void Network::feedforward()
             (*layers[length-1].contents)(j,k) = layers[length-1].activation((*layers[length-1].contents)(j,k));
         }
     }
+    //    std::cout << "begin softmax" << "\n";
     for (int i = 0; i < layers[length-1].contents->rows(); i++) {
         Eigen::MatrixXf m = layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols());
         Eigen::MatrixXf::Index maxRow, maxCol;
@@ -237,11 +239,14 @@ void Network::feedforward()
         float sum = avx_exp(m).sum();
         m = avx_cdiv(avx_exp(m), sum);
 #else
+        // std::cout << "begin sum" << "\n";
+        // std::cout << m << "\n";
         float sum = 0;
         for (int j = 0; j < layers[length-1].contents->cols(); j++) {
             checknan(m(0,j), "input of Softmax operation");
             sum += exp(m(0,j));
         }
+        //  std::cout << "begin norm" << "\n";
         for (int j = 0; j < layers[length-1].contents->cols(); j++) {
             m(0,j) = exp(m(0,j))/sum;
             checknan(m(0,j), "output of Softmax operation");
diff --git a/src/utils.cpp b/src/utils.cpp
@@ -88,11 +88,12 @@ std::function<float(float)> rectifier(float (*activation)(float))
     return rectified;
 }
 
+#if (AVX)
 // Intel intrinsics for the win!
 Eigen::MatrixXf avx_product(Eigen::MatrixXf a, Eigen::MatrixXf b)
 {
 #ifndef RECKLESS
-    assert(a.rows() == b.rows() && a.cols() == b.rows());
+    assert(a.rows() == b.rows() && a.cols() == b.cols());
 #endif
     int size = ((a.rows() * a.cols()) + 7) & (-8);
     for (int i = 0; i < (size-8)/8; i++) {
@@ -166,3 +167,4 @@ Eigen::MatrixXf avx_cosh(Eigen::MatrixXf m)
     for (int i = size-8; i < m.cols()*m.rows(); i++) *(m.data()+i) = cosh(*(m.data()+i));
     return m;
 }
+#endif