commit 1f82b6a4b906d43fb192e6203606dfef174a2290
parent 58fc2a8f6a8dd4bf58089f58918a07c2659e3a11
Author: David Freifeld <freifeld.david@gmail.com>
Date: Thu, 6 Aug 2020 13:22:38 -0700
Tweaking to make things work
Diffstat:
6 files changed, 30 insertions(+), 16 deletions(-)
diff --git a/Makefile b/Makefile
@@ -11,9 +11,9 @@
GEN_FLAGS = -fpic
-CXX = /usr/local/opt/llvm/bin/clang
+CXX = icpc
CXXFLAGS = -shared -std=c++17 -undefined dynamic_lookup `python3 -m pybind11 --includes` ./src/pybind.cpp ./src/bpnn.cpp ./src/utils.cpp mapreduce.a -o mrbpnn`python3-config --extension-suffix`
-LDFLAGS = -L/usr/local/opt/llvm/lib -lc++
+#LDFLAGS = -L/usr/local/opt/llvm/lib -lc++
all: compile
diff --git a/example.cpp b/example.cpp
@@ -30,9 +30,15 @@ double bench(int batch_sz)
return std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count() / pow(10,9);
}
+//[0.338843, 0.202843, 0.187749, 0.158513, 0.130754, 0.122713, 0.116193, 0.106685, 0.105314, 0.104613, 0.107242, 0.10783, 0.10349, 0.100509, 0.0988825, 0.0957273, 0.094987, 0.0946438, 0.101784, 0.116543, 0.113084, 0.0965181, 0.0959425, 0.0954423, 0.0916117, 0.0872652, 0.0861427, 0.101456, 0.106722, 0.105607, 0.0966957, 0.0895351, 0.091384, 0.0894539, 0.0905296, 0.0963364, 0.0844589, 0.0938695, 0.0857902, 0.0871551, 0.0858183, 0.0905346, 0.0885799, 0.0928537, 0.0801082, 0.0865322, 0.0844391, 0.0943158, 0.0975565, 0.0880452, 0.0845807, 0.0831063, 0.0826122, 0.0921348, 0.0848095, 0.0837274, 0.0826749, 0.0881116, 0.0824056, 0.0818347, 0.082213, 0.0876547, 0.0855439, 0.082457, 0.0849121, 0.100939, 0.0834156, 0.0801869, 0.0808713, 0.0837663, 0.0879877, 0.0846778, 0.0764281, 0.0786299, 0.094567, 0.0824965, 0.0871234, 0.0843408, 0.0803826, 0.0799133, 0.0771855, 0.0746452, 0.0768267, 0.0835015, 0.0809601, 0.0821129, 0.0744632, 0.0764572, 0.0803296, 0.0941584, 0.0835956, 0.0888951, 0.0896963, 0.0836287, 0.0783914, 0.0764656, 0.077771, 0.0829365, 0.086286, 0.107995, 0.0956257, 0.0777138, 0.0817694, 0.0835353, 0.0772414, 0.0798855, 0.0794126, 0.0793686, 0.08123, 0.078664, 0.0748174, 0.0748096, 0.0744324, 0.0859672, 0.0784301, 0.07825, 0.081949, 0.0845954, 0.0824774, 0.081973, 0.0779182, 0.0739509, 0.0756138, 0.0741541, 0.0754887, 0.0791503, 0.0839766, 0.0787083, 0.0769964, 0.0784719, 0.0839921, 0.0857664, 0.0931598, 0.0766591, 0.0684099, 0.0698501, 0.0665093, 0.0677842, 0.0685096, 0.0764642, 0.0716012, 0.073328, 0.0732893, 0.0712251, 0.0708647, 0.0711794, 0.0728059, 0.0802736, 0.079377, 0.0792339, 0.0774781, 0.06505, 0.0755057, 0.0705385, 0.0740816, 0.0757598, 0.066424, 0.0755384, 0.0674416, 0.0700981, 0.068384, 0.0686367, 0.0735966, 0.0747738, 0.0723109, 0.0704314, 0.0788959, 0.073288, 0.0768479, 0.0746643, 0.0744588, 0.0801603, 0.0654415, 0.0655372, 0.0654843, 0.0657332, 0.0684429, 0.0673494, 0.072845, 0.079189, 0.0827985, 0.069944, 0.0699655, 0.0708648, 0.0749739, 0.0745345, 0.0730975, 0.0665135, 0.075432, 0.0710865, 0.0740866, 0.0689247, 0.0694628, 0.0726771, 0.0789833, 0.0747874, 0.074542, 0.0728001, 0.0715164, 0.0723057, 0.0747553, 0.0614433, 0.0615861, 0.0614241, 0.0624554, 0.062871, 0.061635, 0.0634367, 0.0664956, 0.0695693, 0.0671272, 0.0716586, 0.0817452, 0.084442, 0.0802537, 0.0906243, 0.0905003, 0.0732237, 0.0753693, 0.0665925, 0.0698621, 0.0729593, 0.0837391, 0.0716365, 0.0689702, 0.0691745, 0.0749986, 0.0800126, 0.0769108, 0.0727117, 0.0776226, 0.0745245, 0.0723034, 0.0714852, 0.0782589, 0.0798153, 0.0743941, 0.0736614, 0.0731142, 0.0735985, 0.0750191, 0.0698225, 0.0630399, 0.0635404, 0.0617813, 0.0602284, 0.0639944, 0.0649009, 0.0606379, 0.0608289, 0.0656081, 0.0629021, 0.0644216, 0.0618588, 0.0617847, 0.0635597, 0.062763, 0.0619538, 0.0676795, 0.0623012, 0.0679293, 0.0672724, 0.0650759, 0.0633218, 0.0642987, 0.0688075, 0.0675989, 0.0675843, 0.0667547, 0.0683742, 0.0656674, 0.0661296, 0.0665096, 0.0678543, 0.0702562, 0.0744055, 0.0656165, 0.0704861, 0.0772736, 0.0702327, 0.0721099, 0.0689054, 0.0694782, 0.0706254, 0.0697149, 0.0716915, 0.0766274, 0.0747415, 0.0717924, 0.0778832, 0.0728482, 0.0730464, 0.074307, 0.0801752, 0.0893201, 0.0740546, 0.0765343, 0.0758259, 0.0770126, 0.0762224, 0.0768883, 0.0769427, 0.0567528, 0.0566555, 0.058204, 0.0580447, 0.0563094, 0.0577655, 0.0677613, 0.0592863, 0.0616723, 0.0670406, 0.0622165, 0.0592929, 0.0608273, 0.066335, 0.0602811, 0.0603421, 0.0591458, 0.0584358, 0.0597496, 0.0591694, 0.0593813, 0.0597292, 0.0641421, 0.063907, 0.0614301, 0.0611214, 0.0628714, 0.0647676, 0.0688876, 0.0594709, 0.061719, 0.0623248, 0.0615561, 0.061157, 0.0626794, 0.0627776, 0.0619415, 0.0623166, 0.0675196, 0.0627493, 0.0618859, 0.0615881, 0.0630554, 0.0618618, 0.0625119, 0.068445, 0.0707345, 0.0640439, 0.0642581, 0.0617944, 0.0638289, 0.0632544, 0.062951, 0.0649969, 0.0702654, 0.0668409, 0.0652916, 0.0629443, 0.0682856, 0.071953, 0.0656666, 0.0697412, 0.082659, 0.0750127, 0.0722331, 0.0739607, 0.0772765, 0.0779399, 0.0776198, 0.0686137, 0.0685726, 0.0699783, 0.0696944, 0.0696976, 0.0681523, 0.0795547, 0.0753863, 0.0761005, 0.0757262, 0.0734613, 0.0697908, 0.0718286, 0.0763912, 0.0702223, 0.0708423, 0.0709779, 0.0702868, 0.0711218, 0.0695263, 0.0745161, 0.0728829, 0.0716398, 0.0712407, 0.0715007, 0.0791072, 0.0836174, 0.0781383, 0.0748312, 0.0771244, 0.0772551, 0.0776821, 0.0579481, 0.0575632, 0.0539092, 0.0608686, 0.053059, 0.052635, 0.053882, 0.054488, 0.0535827, 0.0545368, 0.0612481, 0.0610548, 0.0577176, 0.0577156, 0.0538941, 0.0549249, 0.0522449, 0.0523852, 0.0548231, 0.0508234, 0.0511824, 0.0528544, 0.055825, 0.0551701, 0.0520828, 0.0542695, 0.0507181, 0.0533525, 0.0518795, 0.0599537, 0.053585, 0.053546, 0.0562524, 0.0557106, 0.0569717, 0.0546952, 0.0556468, 0.0562887, 0.0594845, 0.0588476, 0.0603245, 0.0616387, 0.0666848, 0.0561321, 0.0540465, 0.0580248, 0.0589208, 0.0554689, 0.0563942, 0.0542415, 0.0543489, 0.0548278, 0.0629161, 0.058881, 0.0577268, 0.0567686, 0.0542471, 0.0568719, 0.0603566, 0.0585739, 0.0613844, 0.0620546, 0.059745, 0.0648944, 0.0613882, 0.0598974, 0.0594086, 0.0575917, 0.0587733, 0.0568479, 0.0567795, 0.0586232, 0.0625251, 0.0665263, 0.061375, 0.0615487, 0.058287, 0.0583547, 0.0607558, 0.0623474, 0.0672509, 0.0635449, 0.0638978, 0.0647034, 0.0704784, 0.072521, 0.0618658, 0.0612505, 0.0601244, 0.0601935, 0.0632679, 0.0630442, 0.0614066, 0.0674508, 0.0667053, 0.0659775, 0.0645059, 0.0624362, 0.0618998, 0.0613545, 0.0617629, 0.0652347, 0.0622766, 0.0619601, 0.0659781, 0.0619818, 0.0627037, 0.0625741, 0.0676651, 0.073342, 0.0705372, 0.0822286, 0.0660385, 0.065279, 0.0655787, 0.0655194, 0.0659903, 0.0673343, 0.0694705, 0.0713905, 0.0658311, 0.0643639, 0.0655, 0.0661216, 0.0694046, 0.0753305, 0.105748, 0.098237, 0.0747157, 0.0785219, 0.0682795, 0.0672818, 0.0675554, 0.0659002, 0.0633071, 0.0673199, 0.0674547, 0.0671711, 0.0681841, 0.0735241, 0.0668153, 0.0696955, 0.0711496, 0.0715202, 0.0653737, 0.069104, 0.0702511, 0.070219, 0.0678985, 0.0667064, 0.0691054, 0.0689582, 0.0702573, 0.0700223, 0.0750812, 0.0699759, 0.0737945, 0.0750655, 0.0758381, 0.072815, 0.0704952, 0.0712361, 0.0779347, 0.0762042, 0.0821648, 0.0847628, 0.077548, 0.0858927, 0.0755556, 0.0754716, 0.0718222, 0.0726705, 0.082665, 0.0844112, 0.0758227, 0.0717166, 0.0758917, 0.0777478, 0.0725651, 0.0720293, 0.0775783, 0.0784786, 0.0780898, 0.0749561, 0.076107, 0.074708, 0.0795886, 0.0747792, 0.0769627, 0.0803079, 0.0741758, 0.0765839, 0.0773766, 0.0864978, 0.0736693, 0.0763235, 0.0760622, 0.0751163, 0.0759073, 0.0783604, 0.0785644, 0.0660107, 0.0405451, 0.039513, 0.0409156, 0.039257, 0.0440669, 0.0419419, 0.0403363, 0.0395096, 0.0479605, 0.039714, 0.0370365, 0.039245, 0.0387049, 0.0380085, 0.039181, 0.0460186, 0.0431311, 0.046778, 0.0440661, 0.0384525, 0.0406509, 0.0384933, 0.0388873, 0.0413364, 0.0391542, 0.039036, 0.0388945, 0.0388088, 0.0403225, 0.0398177, 0.0431819, 0.0417834, 0.0508315, 0.0447285, 0.0401619, 0.0431931, 0.0425749, 0.0391143, 0.0401156, 0.0401153, 0.0399961, 0.0393476, 0.0394336, 0.0408767, 0.0415496, 0.0412664, 0.039348, 0.0399709, 0.0402044, 0.0462586, 0.0406274, 0.041561, 0.0398034, 0.0399118, 0.0417526, 0.040335, 0.0452037, 0.0439503, 0.0419566, 0.0432817, 0.0445183, 0.0430093, 0.0416696, 0.0419808, 0.0436265, 0.0419088, 0.0453464, 0.0440296, 0.0424192, 0.0418441, 0.041235, 0.0422794, 0.0450149, 0.0419015, 0.0535337, 0.0569562, 0.0472342, 0.0461127, 0.0453686, 0.0502894, 0.0490998, 0.0449671, 0.0466881, 0.0463889, 0.0462829, 0.0508525, 0.0487154, 0.0470666, 0.0464224, 0.0476204, 0.0448276, 0.0494002, 0.0474031, 0.044733, 0.0439198, 0.0435442, 0.0455668, 0.049211, 0.0461102, 0.0472248, 0.0525482, 0.0539073, 0.050996, 0.0484653, 0.0477119, 0.0445589, 0.0457872, 0.0449623, 0.0480063, 0.0462105, 0.0488877, 0.0459833, 0.0461472, 0.0457562, 0.0463808, 0.0464316, 0.0470659, 0.0470006, 0.0471008, 0.0461179, 0.0459644, 0.0520295, 0.0485826, 0.047169, 0.0467158, 0.0456405, 0.0468305, 0.0469053, 0.0477925, 0.0474659, 0.049805, 0.0482617, 0.050301, 0.0528174, 0.0482939, 0.0475984, 0.0456105, 0.0471828, 0.0492715, 0.0572987, 0.0550541, 0.0578473, 0.0595607, 0.0505665, 0.0503204, 0.0500529, 0.0472794, 0.0509886, 0.0507362, 0.048566, 0.0537012, 0.0490438, 0.0492393, 0.0485242, 0.0481478, 0.0525764, 0.0512306, 0.055146, 0.0500126, 0.0496092, 0.0531888, 0.0613099, 0.0583438, 0.0509896, 0.0510678, 0.0498737, 0.0498133, 0.0496001, 0.0500194, 0.0499713, 0.0490571, 0.0505172, 0.0500107, 0.0544811, 0.0495994, 0.0499513, 0.0495694, 0.0513926, 0.0515407, 0.0525085, 0.0565077, 0.0587987, 0.0658004, 0.0622314, 0.0545299, 0.0545649, 0.0514742, 0.0531484, 0.0534982, 0.0524878, 0.0483598, 0.0501653, 0.0532739, 0.0492975, 0.0485503, 0.0508417, 0.0522699, 0.0521935, 0.0506826, 0.0605368, 0.0565989, 0.0604093, 0.0530365, 0.064229, 0.0629535, 0.0624096, 0.066361, 0.0618925, 0.0583711, 0.0571752, 0.062502, 0.0548659, 0.0522817, 0.0557696, 0.052914, 0.0522001, 0.0600728, 0.0543017, 0.0532161, 0.0553463, 0.0565696, 0.0595538, 0.0520377, 0.0522839, 0.0523485, 0.0563499, 0.0609379, 0.0612478, 0.0575998, 0.0575029, 0.0561437, 0.0655699, 0.054431, 0.0523803, 0.0551981, 0.0523392, 0.0546566, 0.0542116, 0.0541699, 0.0544687, 0.0545416, 0.055942, 0.0559525, 0.0545977, 0.0536383, 0.0546254, 0.054316, 0.0557946, 0.05389, 0.0514351, 0.0550739, 0.0533242, 0.0646394, 0.0520651, 0.0542545, 0.0603806, 0.0581443, 0.0568814, 0.0543293, 0.0548077, 0.0805176, 0.0657295, 0.0646935, 0.0589276, 0.0551856, 0.0603163, 0.0605762, 0.064717, 0.0609224, 0.0625518, 0.0572685, 0.056946, 0.0581012, 0.0555207, 0.0542294, 0.0564566, 0.0550297, 0.0546578, 0.0580913, 0.0580568, 0.0617315, 0.0579516, 0.0603745, 0.0585286, 0.0609434, 0.0683426, 0.0732768, 0.0666987, 0.0568952, 0.0573148, 0.0580979, 0.0575629, 0.0599727, 0.0616112, 0.0593086, 0.0584075, 0.0574656, 0.0580244, 0.0590697, 0.0607608, 0.0623766, 0.0623745, 0.068389, 0.0609393, 0.0608429, 0.0577811, 0.0628812, 0.0610943, 0.0568408, 0.0583296, 0.0585439, 0.0595483, 0.0596767, 0.0606275, 0.0581862, 0.0595932, 0.0593933, 0.0587425, 0.0618889, 0.0694505, 0.0677476, 0.0631403, 0.0583485, 0.0616749, 0.0599815, 0.0621532, 0.0607646, 0.0604511, 0.0587148, 0.059066, 0.0587783, 0.0584401, 0.0569055, 0.0582229, 0.061676, 0.0667713, 0.0599334, 0.0611014, 0.0622944, 0.060729, 0.0603675, 0.057546, 0.061077, 0.0607897, 0.0636711, 0.0602081, 0.0598515, 0.0623742, 0.0620331, 0.0607267, 0.0594055, 0.0704705, 0.0630482, 0.0630267, 0.0613812, 0.0615533, 0.0616067, 0.0632971, 0.0627742, 0.0596687, 0.0601888, 0.0648484, 0.0636087, 0.065057, 0.0627491, 0.0654386, 0.0649758, 0.0678422, 0.0660876, 0.0607184, 0.0661711, 0.064417, 0.0686019, 0.07235, 0.0782594, 0.0733848, 0.0708524, 0.0649743, 0.0627133, 0.0659702, 0.0655011, 0.063743, 0.0727605, 0.0648036, 0.0624012, 0.0631326, 0.0635783, 0.0644976, 0.061628, 0.0636082, 0.062726, 0.0633133, 0.0655694, 0.070263, 0.0668848, 0.0659407, 0.0691758, 0.0841067, 0.0719216, 0.0653001, 0.0649174, 0.0649765, 0.0653888, 0.065135, 0.0655023, 0.0660012, 0.063854, 0.0649523, 0.0650156, 0.0684334, 0.0704392, 0.0673033, 0.0713305, 0.0690728, 0.0701402, 0.0695365
+
int main()
{
- std::cout << bench(16) << "\n";
+ std::cout << "[";
+ for (int i = 1; i < 1300; i++) {
+ std::cout << bench(i) << ", ";
+ }
+ std::cout << "]";
// show_console_cursor(false);
// BlockProgressBar bar{
// option::BarWidth{80},
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -232,11 +232,12 @@ void Network::feedforward()
// std::cout << "\nGETTING SUM\n";
float sum = avx_exp(m).sum();
//std::cout << "\nFINAL ACTIVATION\n";
- for (int j = 0; j < layers[length-1].contents->cols(); j++) {
- m(0,j) = exp(m(0,j))/sum;
- // std::cout << "Calculating " << exp(m(0,j)) << "/" << sum << " to be " << (*layers[length-1].contents)(i,j) << "(aka " << test<<")\n";
- checknan(m(0,j), "output of Softmax operation");
- }
+ // for (int j = 0; j < layers[length-1].contents->cols(); j++) {
+ // m(0,j) = exp(m(0,j))/sum;
+ // // std::cout << "Calculating " << exp(m(0,j)) << "/" << sum << " to be " << (*layers[length-1].contents)(i,j) << "(aka " << test<<")\n";
+ // checknan(m(0,j), "output of Softmax operation");
+ // }
+ m = avx_cdiv(avx_exp(m), sum);
layers[length-1].contents->block(i,0,1,layers[length-1].contents->cols()) = m;
}
// std::cout << "\n\n";
@@ -543,7 +544,7 @@ void Network::train()
epoch_acc = 1.0/((float) instances/batch_size) * acc_sum;
epoch_cost = 1.0/((float) instances/batch_size) * cost_sum;
validate(VAL_PATH);
- printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n", epochs, epoch_cost, epoch_acc, val_cost, val_acc);
+ // printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n", epochs, epoch_cost, epoch_acc, val_cost, val_acc);
batches=1;
rewind(data);
decay();
diff --git a/src/pybind.cpp b/src/pybind.cpp
@@ -16,16 +16,16 @@ PYBIND11_MODULE(mrbpnn, m) {
py::class_<Network>(m, "Network")
.def(py::init<char*, int, float, float, int, float, float, bool, float>())
.def("add_layer", &Network::add_layer, py::arg("nodes"), py::arg("activation"))
- .def("add_prelu_layer", &Network::add_prelu_layer, py::arg("nodes"), py::arg("a"))
+ //.def("add_prelu_layer", &Network::add_prelu_layer, py::arg("nodes"), py::arg("a"))
.def("initialize", &Network::initialize)
- .def("init_decay", &Network::init_decay, py::arg("type"), py::arg("a_0"), py::arg("k"))
- .def("set_activation", &Network::set_activation)
+ //.def("init_decay", &Network::init_decay, py::arg("type"), py::arg("a_0"), py::arg("k"))
+ //.def("set_activation", &Network::set_activation)
.def("feedforward", &Network::feedforward)
.def("backpropagate", &Network::backpropagate)
.def("list_net", &Network::list_net)
.def("cost", &Network::cost)
.def("accuracy", &Network::accuracy)
- .def("update_layer", &Network::update_layer, py::arg("vals"), py::arg("len"), py::arg("index"))
+ //.def("update_layer", &Network::update_layer, py::arg("vals"), py::arg("len"), py::arg("index"))
.def("next_batch", &Network::next_batch)
.def("train", &Network::train)
.def("get_acc", &Network::get_acc)
diff --git a/src/utils.cpp b/src/utils.cpp
@@ -118,13 +118,13 @@ Eigen::MatrixXf avx_exp(Eigen::MatrixXf m)
return dst;
}
-Eigen::MatrixXf avx_div(Eigen::MatrixXf m, float denom)
+Eigen::MatrixXf avx_cdiv(Eigen::MatrixXf m, float denom)
{
int size = ((m.rows() * m.cols()) + 7) & (-8);
float arr1[size];
memcpy(arr1, m.data(), sizeof(float)*m.cols()*m.rows());
for (int i = 0; i < size/8; i++) {
- __m256 denom_vec = _mm256_broadcast_ss(denom);
+ __m256 denom_vec = _mm256_broadcast_ss(&denom);
_mm256_store_ps(arr1+i*8, _mm256_div_ps(_mm256_load_ps(arr1+i*8), denom_vec));
}
Eigen::Map<Eigen::MatrixXf> dst (arr1, m.rows(), m.cols());
@@ -143,13 +143,14 @@ Eigen::MatrixXf avx_log(Eigen::MatrixXf m)
return dst;
}
+
Eigen::MatrixXf avx_pow(Eigen::MatrixXf m, float exponent)
{
int size = ((m.rows() * m.cols()) + 7) & (-8);
float arr1[size];
memcpy(arr1, m.data(), sizeof(float)*m.cols()*m.rows());
for (int i = 0; i < size/8; i++) {
- __m256 exponent_vec = _mm256_broadcast_ss(exponent);
+ __m256 exponent_vec = _mm256_broadcast_ss(&exponent);
_mm256_store_ps(arr1+i*8, _mm256_pow_ps(_mm256_load_ps(arr1+i*8), exponent_vec));
}
Eigen::Map<Eigen::MatrixXf> dst (arr1, m.rows(), m.cols());
diff --git a/src/utils.hpp b/src/utils.hpp
@@ -40,6 +40,12 @@ std::function<float(float)> rectifier(float (*activation)(float));
Eigen::MatrixXf avx_product(Eigen::MatrixXf a, Eigen::MatrixXf b);
Eigen::MatrixXf avx_exp(Eigen::MatrixXf m);
+Eigen::MatrixXf avx_cdiv(Eigen::MatrixXf m, float denom);
+Eigen::MatrixXf avx_log(Eigen::MatrixXf m);
+Eigen::MatrixXf avx_cpow(Eigen::MatrixXf m, float exponent);
+Eigen::MatrixXf avx_tanh(Eigen::MatrixXf m);
+Eigen::MatrixXf avx_cosh(Eigen::MatrixXf m);
+
Eigen::MatrixXf strassen_mul(Eigen::MatrixXf a, Eigen::MatrixXf b);
