commit a3114188ddc6ff1e853b331730bb7b3bad85eb0f
parent f6a070985611b40ed2d55e31cf142e3909396a86
Author: David Freifeld <freifeld.david@gmail.com>
Date: Tue, 14 Jul 2020 19:42:40 -0700
Working on speedups + icpc works without MKL
Diffstat:
5 files changed, 40 insertions(+), 38 deletions(-)
diff --git a/Makefile b/Makefile
@@ -3,7 +3,6 @@
# Jacobian
#
# Created by David Freifeld
-# Copyright © 2020 David Freifeld. All rights reserved.
#
# ---------
# TODO:
@@ -21,10 +20,10 @@ debug: $(GEN_FLAGS) = -Wall -U NDEBUG
fast: CXXFLAGS += $(GEN_FLAGS) -O3
fast: compile
-faster: CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/mr_bpnn_2.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a ${MKLROOT}/lib/libmkl_intel_ilp64.a ${MKLROOT}/lib/libmkl_intel_thread.a ${MKLROOT}/lib/libmkl_core.a -liomp5 -lpthread -lm -ldl -o mrbpnn`python3-config --extension-suffix` -O3 -mavx -mfma -march=native -mfpmath=sse -fno-pic -DMKL_ILP64 -I${MKLROOT}/include -D EIGEN_USE_MKL_ALL -D NDEBUG
+faster: CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/mr_bpnn_2.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -liomp5 -lpthread -lm -ldl -o mrbpnn`python3-config --extension-suffix` -O3 -mavx -mfma -march=native -mfpmath=sse -fno-pic -DMKL_ILP64 -D EIGEN_USE_MKL_ALL -D NDEBUG
faster: compile
-tradeoffs: CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/mr_bpnn_2.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a ${MKLROOT}/lib/libmkl_intel_ilp64.a ${MKLROOT}/lib/libmkl_intel_thread.a ${MKLROOT}/lib/libmkl_core.a -liomp5 -lpthread -lm -ldl -o mrbpnn`python3-config --extension-suffix` -O3 -mavx -mfma -march=native -mfpmath=sse -DMKL_ILP64 -I${MKLROOT}/include -qopenmp -fno-pic -qopt-calloc -qopt-prefetch -unroll-aggressive -qopt-calloc -use-intel-optimized-headers -ffast-math -no-prec-div -no-prec-sqrt -fimf-precision=low -fast-transcendentals -D EIGEN_USE_MKL_ALL -D NDEBUG #-qopt-report=5 -qopt-report-file=report
+tradeoffs: CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/mr_bpnn_2.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -o mrbpnn`python3-config --extension-suffix` -O3 -mavx -mfma -march=native -mfpmath=sse -DMKL_ILP64 -qopenmp -fno-pic -qopt-calloc -qopt-prefetch -unroll-aggressive -qopt-calloc -use-intel-optimized-headers -ffast-math -no-prec-div -no-prec-sqrt -fimf-precision=low -fast-transcendentals -D EIGEN_USE_MKL_ALL -D NDEBUG #-qopt-report=5 -qopt-report-file=report
tradeoffs: compile
@@ -32,4 +31,4 @@ reckless: CXXFLAGS = -O3
reckless: compile
compile:
- g++ $(CXXFLAGS) && rm ./mrbpnn/mrbpnn.cpython-37m-darwin.so ; cp ./mrbpnn.cpython-37m-darwin.so ./mrbpnn/mrbpnn.cpython-37m-darwin.s ; rm ./scripts/mrbpnn.cpython-37m-darwin.so ; cp ./mrbpnn.cpython-37m-darwin.so ./scripts/mrbpnn.cpython-37m-darwin.so
+ icpc $(CXXFLAGS) && rm ./mrbpnn/mrbpnn.cpython-37m-darwin.so ; cp ./mrbpnn.cpython-37m-darwin.so ./mrbpnn/mrbpnn.cpython-37m-darwin.s ; rm ./scripts/mrbpnn.cpython-37m-darwin.so ; cp ./mrbpnn.cpython-37m-darwin.so ./scripts/mrbpnn.cpython-37m-darwin.so
diff --git a/example.cpp b/example.cpp
@@ -28,7 +28,7 @@ double bench(int batch_sz)
// net.backpropagate();
// std::cout << net.cost() << " " << net.accuracy() << "\n";
// }
- for (int i = 0; i < 1; i++) {
+ for (int i = 0; i < 50; i++) {
net.train();
// net.list_net();
}
diff --git a/scripts/benchmark.py b/scripts/benchmark.py
@@ -16,26 +16,29 @@ def bench(batch_sz, layers):
net = mrbpnn.Network("../data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 0, 0.9)
net.add_layer(4, "linear")
for i in range(layers):
- net.add_layer(5, "relu")
- net.add_layer(1, "resig")
+ net.add_layer(5, "lecun_tanh")
+ net.add_layer(2, "linear")
net.initialize()
initend = time.time()
- for i in range(1):
+ for i in range(50):
net.train()
end = time.time()
return (end-init)
-x = []
-y = []
-i = 1
-while(i < 125):
- print(i)
- y.append(bench(i, 1))
- x.append(i)
- i+=1;
-plt.plot(x,y, label = "Jacobian (Sequential)")
+for i in range(5):
+ print(bench(10,1))
+
+# x = []
+# y = []
+# i = 1
+# while(i < 125):
+# print(i)
+# y.append(bench(i, 1))
+# x.append(i)
+# i+=1;
+# plt.plot(x,y, label = "Jacobian (Sequential)")
# plt.plot(otherlist, [2.3994078636169434, 1.2735769748687744, 1.0030598640441895, 0.8972160816192627, 0.801548957824707, 0.752018928527832, 0.7073678970336914, 0.6857280731201172, 0.6707980632781982, 0.6421489715576172, 0.6614980697631836, 0.6403779983520508, 0.7251319885253906, 0.6796879768371582, 0.6601080894470215, 0.6711599826812744, 0.6432759761810303, 0.6491389274597168, 0.6762490272521973, 0.6859049797058105, 0.7067179679870605, 0.7142889499664307, 0.7258059978485107, 0.7868969440460205, 0.7326970100402832, 0.7365641593933105, 0.7576079368591309, 0.7772500514984131, 0.8062641620635986, 0.7768490314483643, 0.8253629207611084, 0.8264601230621338, 0.8459320068359375, 0.9670729637145996, 0.8388969898223877, 0.9129719734191895, 0.9009649753570557, 0.8916170597076416, 0.8926799297332764, 0.9171609878540039, 0.9242072105407715, 0.9534740447998047, 0.947465181350708, 0.9723358154296875, 1.018247127532959, 1.1208629608154297, 1.014026165008545, 1.034980058670044, 1.0626468658447266, 1.080394983291626, 1.0627479553222656, 1.0839190483093262, 1.0938241481781006, 1.127730131149292, 1.1265759468078613, 1.136888027191162, 1.140428066253662, 1.1712510585784912, 1.206390142440796, 1.2087180614471436, 1.4066569805145264, 1.2425589561462402, 1.280066967010498, 1.2891559600830078, 1.3243582248687744, 1.3152379989624023], label="Keras")
-plt.legend()
-plt.show()
-print(y)
+# plt.legend()
+# plt.show()
+# print(y)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -169,11 +169,11 @@ void Network::feedforward()
// std::cout << "\nSOFTMAX INPUT\n" << *layers[length-1].contents << "\n\n";
for (int i = 0; i < layers[length-1].contents->rows(); i++) {
float sum = 0;
- Eigen::MatrixXf m = *layers[length-1].contents;//->block(i,0,1,layers[length-1].contents->cols());
- // Eigen::MatrixXf::Index maxRow, maxCol;
- // float max = m.maxCoeff(&maxRow, &maxCol);
- // m = (m.array() - max).matrix();
- // std::cout << "\nGETTING SUM\n";
+ Eigen::MatrixXf m = layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols());
+ Eigen::MatrixXf::Index maxRow, maxCol;
+ float max = m.maxCoeff(&maxRow, &maxCol);
+ m = (m.array() - max).matrix();
+ // std::cout << "\nGETTING SUM\n";
for (int j = 0; j < layers[length-1].contents->cols(); j++) {
checknan(m(0,j), "input to final layer");
sum += exp(m(0,j));
@@ -182,8 +182,7 @@ void Network::feedforward()
}
// std::cout << "\nFINAL ACTIVATION\n";
for (int j = 0; j < layers[length-1].contents->cols(); j++) {
- (*layers[length-1].contents)(i,j) = exp(m(0,j))/sum;
- float test = exp(m(0,j))/sum;
+ m(0,j) = exp(m(0,j))/sum;
// std::cout << "Calculating " << exp(m(0,j)) << "/" << sum << " to be " << (*layers[length-1].contents)(i,j) << "(aka " << test<<")\n";
checknan(m(0,j), "output of Softmax operation");
}
@@ -249,26 +248,26 @@ void Network::backpropagate()
std::vector<Eigen::MatrixXf> gradients;
std::vector<Eigen::MatrixXf> deltas;
Eigen::MatrixXf error (layers[length-1].contents->rows(), layers[length-1].contents->cols());
- std::cout << "\nTRUTH:\n";
+ // std::cout << "\nTRUTH:\n";
for (int i = 0; i < error.rows(); i++) {
for (int j = 0; j < error.cols(); j++) {
float truth;
if (j==(*labels)(i,0)) truth = 1;
else truth = 0;
- std::cout << truth << " ";
+ // std::cout << truth << " ";
error(i,j) = (*layers[length-1].contents)(i,j) - truth;
checknan(error(i,j), "gradient of final layer");
// std::cout << truth << "[as label is "<< (*labels)(i,0) <<"] - " << (*layers[length-1].contents)(i,j) << "[aka index " << i << " " << j << "] = " << error(i,j) << "\n";
}
- std::cout << "\n";
- }
- std::cout << "\n\n";
- std::cout << "\nLABELS:\n";
- std::cout << *labels << "\n\n";
- std::cout << "\nPREDICTION:\n";
- std::cout << (*layers[length-1].contents) << "\n\n";
- std::cout << "\nERR:\n";
- std::cout << error << "\n\n";
+ // std::cout << "\n";
+ }
+ // std::cout << "\n\n";
+ // std::cout << "\nLABELS:\n";
+ // std::cout << *labels << "\n\n";
+ // std::cout << "\nPREDICTION:\n";
+ // std::cout << (*layers[length-1].contents) << "\n\n";
+ // std::cout << "\nERR:\n";
+ // std::cout << error << "\n\n";
gradients.push_back(error);
deltas.push_back((*layers[length-2].contents).transpose() * gradients[0]);
int counter = 1;
diff --git a/src/checks.cpp b/src/checks.cpp
@@ -78,6 +78,7 @@ void checks(Network net)
}
}
}
+ return false;
};
if (check_gradients(gradients) == false && check_gradients(deltas) == false) {
std::cout << " \u001b[32mPassed!\n\u001b[37m";
