commit 95ab74a318efd0dbf671b824e6597d0eb5565fdb
parent 4996187a8c377823286306f9737691ebbdde3836
Author: David Freifeld <freifeld.david@gmail.com>
Date: Fri, 31 Jul 2020 13:39:44 -0700
Benchmark code update to include more metrics
Diffstat:
4 files changed, 44 insertions(+), 19 deletions(-)
diff --git a/bench/benchmark.py b/bench/benchmark.py
@@ -43,25 +43,47 @@ for i in range(1):
# plt.show()
# print(y)
-plt.plot([0.5025448077098257, 0.06346596189658764, 0.047917547554598754, 0.04105092616154024, 0.03523913263298199, 0.031739621835796136, 0.028315013611574093, 0.026019472083075448, 0.024406521083899426, 0.022933075354279908, 0.021600019709607246, 0.020935506796379722, 0.020341872339561994, 0.019887265494774282, 0.019298826768017932, 0.019021222173712722, 0.018694215646250782, 0.01869210122639975, 0.01835323961580634, 0.01805996506773795, 0.017826852610588563, 0.017619089276841214, 0.0173522467286016, 0.017464391304677935, 0.01741090491941103, 0.016986178006763254, 0.016966580251704732, 0.016847677484476468, 0.01688444716072269, 0.016816666155598632, 0.016728257525321137, 0.01650929572067639, 0.016523535019764574, 0.016360246686629634, 0.016583477585886453, 0.016515186448995516, 0.016167543447082024, 0.016142403505456843, 0.016143194300608676, 0.01618973151673178, 0.01584432253715314, 0.01614457198639892, 0.015996006825746038, 0.015867076906405406, 0.015746370663870562, 0.015635421002396035, 0.015604364669764566, 0.01580412467568558, 0.015422517508656697, 0.01570749596837556], label="Keras")
-plt.plot([0.22765946708964985, 0.05371553719734514, 0.03755848292286033, 0.031034886701316232, 0.029416448738894714, 0.028644813125990085, 0.027122890983788384, 0.025797686040006727, 0.025556786300380104, 0.025362928792686257, 0.025110247483267293, 0.024511196780086352, 0.025183815891478237, 0.024482555564073537, 0.02378877399131557, 0.024125701865020967, 0.02412329411861912, 0.022786135199350237, 0.022480409765085637, 0.021458697166071822, 0.019833367598545273, 0.02021988033199843, 0.01960400892236572, 0.019237394278116573, 0.019515914248845335, 0.018973920885966906, 0.019074104276072507, 0.019292046152524008, 0.01877620817772699, 0.018388098750312793, 0.019136795989971683, 0.01865537591056516, 0.018765945022999747, 0.018489238997267572, 0.018801562214387422, 0.018801799720918007, 0.018268155148376198, 0.019060883800566985, 0.018474303461944327, 0.018674987837030792, 0.01909543504778123, 0.018523710002715618, 0.018588384030547165, 0.017887256971702275, 0.01847935954667011, 0.018385809867809348, 0.018235951924190814, 0.018927972964227, 0.018075138759114685, 0.018163896239800564], label="Keras (set val split)")
-plt.plot([0.6356955872469517, 0.4474506605850878, 0.3890421118235231, 0.3383762977820027, 0.2570263108725408, 0.17083878727236804, 0.1206560030408328, 0.09028454783518607, 0.06834034661147965, 0.05482004367813022, 0.046493575456135125, 0.04067401763115656, 0.03674237276445186, 0.03355837383721336, 0.03091509716139769, 0.028653709084133477, 0.026661936578240484, 0.024985219962964603, 0.023432621666806992, 0.022137642019742405, 0.020931669067147784, 0.019861774528818207, 0.018922685342936688, 0.018049194997636995, 0.017238716312126513, 0.016458408508859233, 0.0158025484137779, 0.015156084850422274, 0.014549496406066243, 0.014012374438633365, 0.013499269858957372, 0.013027227670698194, 0.01258420495404383, 0.012184903993599385, 0.011772133709941639, 0.01141798619621345, 0.011055273510342334, 0.01075009333315004, 0.010450198537204518, 0.010153955363072054, 0.009865946080363197, 0.009610634188130004, 0.009366072731556546, 0.009125463513132877, 0.008900918802492984, 0.008680248000794285, 0.008473303210025152, 0.008258737640210677, 0.008091226337141558, 0.007901268149935282], label = "SciKit Learn")
-plt.plot([0.147248, 0.0459116, 0.0426555, 0.0399301, 0.038506, 0.0373933, 0.0364922, 0.0356698, 0.0350383, 0.0344831, 0.0340619, 0.0336462, 0.0332532, 0.0329713, 0.032643, 0.0323429, 0.0320908, 0.0318416, 0.0316404, 0.0315699, 0.0313803, 0.0311661, 0.0310349, 0.030904, 0.030829, 0.0307032, 0.0306185, 0.0305414, 0.0305478, 0.0304674, 0.0304535, 0.0303114, 0.0303842, 0.0302684, 0.0302227, 0.0301911, 0.0301535, 0.0301598, 0.0300992, 0.0301056, 0.0300929, 0.0301216, 0.0299648, 0.0300934, 0.0299637, 0.030046, 0.0299663, 0.0299561, 0.0299137, 0.0298976], label = "Jacobian")
-plt.legend()
-plt.title("Loss over epochs")
-plt.xlabel("Epoch")
-plt.ylabel("Loss")
+fig, axs = plt.subplots(2, 2)
+axs[0,0].plot([0.5025448077098257, 0.06346596189658764, 0.047917547554598754, 0.04105092616154024, 0.03523913263298199, 0.031739621835796136, 0.028315013611574093, 0.026019472083075448, 0.024406521083899426, 0.022933075354279908, 0.021600019709607246, 0.020935506796379722, 0.020341872339561994, 0.019887265494774282, 0.019298826768017932, 0.019021222173712722, 0.018694215646250782, 0.01869210122639975, 0.01835323961580634, 0.01805996506773795, 0.017826852610588563, 0.017619089276841214, 0.0173522467286016, 0.017464391304677935, 0.01741090491941103, 0.016986178006763254, 0.016966580251704732, 0.016847677484476468, 0.01688444716072269, 0.016816666155598632, 0.016728257525321137, 0.01650929572067639, 0.016523535019764574, 0.016360246686629634, 0.016583477585886453, 0.016515186448995516, 0.016167543447082024, 0.016142403505456843, 0.016143194300608676, 0.01618973151673178, 0.01584432253715314, 0.01614457198639892, 0.015996006825746038, 0.015867076906405406, 0.015746370663870562, 0.015635421002396035, 0.015604364669764566, 0.01580412467568558, 0.015422517508656697, 0.01570749596837556], label="Keras", color="r")
+axs[0,0].plot([0.7879068318088323, 0.09516164867688488, 0.07563164060478968, 0.06368552646692226, 0.054840093046821504, 0.04791283077060782, 0.04277797107467588, 0.03926807701809714, 0.03580813260983355, 0.03316434786118419, 0.03137411057504992, 0.029850296676158905, 0.028874377368499113, 0.02738272940234238, 0.026146136594331817, 0.025704378878932125, 0.02479495369638039, 0.02364667130646524, 0.023422482195279457, 0.02243726485908426, 0.021918368050890254, 0.021494656505174195, 0.021088606653278632, 0.020731150474868072, 0.020081241636703544, 0.01991242998347476, 0.019817821484133108, 0.01929077183186317, 0.019043239484238902, 0.01861191921503536, 0.01828094854309464, 0.017715507970247048, 0.01763268999311308, 0.017503339562885807, 0.017498996990751352, 0.016864338578249247, 0.016716289371005354, 0.01664047861866505, 0.016465328815732372, 0.016277130278419483, 0.016030110511312028, 0.015790291867582806, 0.015560780664202788, 0.015480223164233742, 0.015054066943017063, 0.015169570015879064, 0.014821364322294858, 0.014672116970100643, 0.01468882526869312, 0.014406936027306989], label="Keras (w/ val split)")
+axs[0,0].plot([0.6356955872469517, 0.4474506605850878, 0.3890421118235231, 0.3383762977820027, 0.2570263108725408, 0.17083878727236804, 0.1206560030408328, 0.09028454783518607, 0.06834034661147965, 0.05482004367813022, 0.046493575456135125, 0.04067401763115656, 0.03674237276445186, 0.03355837383721336, 0.03091509716139769, 0.028653709084133477, 0.026661936578240484, 0.024985219962964603, 0.023432621666806992, 0.022137642019742405, 0.020931669067147784, 0.019861774528818207, 0.018922685342936688, 0.018049194997636995, 0.017238716312126513, 0.016458408508859233, 0.0158025484137779, 0.015156084850422274, 0.014549496406066243, 0.014012374438633365, 0.013499269858957372, 0.013027227670698194, 0.01258420495404383, 0.012184903993599385, 0.011772133709941639, 0.01141798619621345, 0.011055273510342334, 0.01075009333315004, 0.010450198537204518, 0.010153955363072054, 0.009865946080363197, 0.009610634188130004, 0.009366072731556546, 0.009125463513132877, 0.008900918802492984, 0.008680248000794285, 0.008473303210025152, 0.008258737640210677, 0.008091226337141558, 0.007901268149935282], label = "SciKit Learn", color='orange')
+axs[0,0].plot([0.147248, 0.0459116, 0.0426555, 0.0399301, 0.038506, 0.0373933, 0.0364922, 0.0356698, 0.0350383, 0.0344831, 0.0340619, 0.0336462, 0.0332532, 0.0329713, 0.032643, 0.0323429, 0.0320908, 0.0318416, 0.0316404, 0.0315699, 0.0313803, 0.0311661, 0.0310349, 0.030904, 0.030829, 0.0307032, 0.0306185, 0.0305414, 0.0305478, 0.0304674, 0.0304535, 0.0303114, 0.0303842, 0.0302684, 0.0302227, 0.0301911, 0.0301535, 0.0301598, 0.0300992, 0.0301056, 0.0300929, 0.0301216, 0.0299648, 0.0300934, 0.0299637, 0.030046, 0.0299663, 0.0299561, 0.0299137, 0.0298976], label = "Jacobian", color="g")
+
+# axs[0,1].plot([0.6614238410596026, 0.8857615894039735, 0.9461920529801324, 0.9718543046357616, 0.9834437086092715, 0.9867549668874173, 0.9950331125827815, 0.9942052980132451, 0.9950331125827815, 0.9966887417218543, 0.9975165562913907, 0.9975165562913907, 0.9983443708609272, 0.9983443708609272, 0.9991721854304636, 0.9983443708609272, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 1.0, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 1.0, 0.9991721854304636, 1.0, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 1.0, 0.9991721854304636, 1.0, 0.9991721854304636, 0.9991721854304636, 0.9975165562913907, 0.9991721854304636, 0.9966887417218543, 0.9991721854304636], label="Keras", color="r")
+axs[0,1].plot([0.7425496688741722, 0.9130794701986755, 0.918046357615894, 0.9619205298013245, 0.9693708609271523, 0.9701986754966887, 0.9668874172185431, 0.9685430463576159, 0.9627483443708609, 0.9735099337748344, 0.9693708609271523, 0.9710264900662252, 0.9677152317880795, 0.9644039735099338, 0.9751655629139073, 0.9759933774834437, 0.9718543046357616, 0.9685430463576159, 0.9677152317880795, 0.9677152317880795, 0.9735099337748344, 0.9718543046357616, 0.9768211920529801, 0.972682119205298, 0.9735099337748344, 0.9743377483443708, 0.9718543046357616, 0.9751655629139073, 0.9759933774834437, 0.9710264900662252, 0.9743377483443708, 0.9759933774834437, 0.9743377483443708, 0.9759933774834437, 0.9751655629139073, 0.9735099337748344, 0.972682119205298, 0.9751655629139073, 0.9743377483443708, 0.9718543046357616, 0.9768211920529801, 0.9793046357615894, 0.9817880794701986, 0.9776490066225165, 0.9776490066225165, 0.9776490066225165, 0.9776490066225165, 0.9751655629139073, 0.9751655629139073, 0.9759933774834437], label="Keras", color="r")
+axs[0,1].plot([0.6614238410596026, 0.8857615894039735, 0.9461920529801324, 0.9718543046357616, 0.9834437086092715, 0.9867549668874173, 0.9950331125827815, 0.9942052980132451, 0.9950331125827815, 0.9966887417218543, 0.9975165562913907, 0.9975165562913907, 0.9983443708609272, 0.9983443708609272, 0.9991721854304636, 0.9983443708609272, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 1.0, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 1.0, 0.9991721854304636, 1.0, 0.9991721854304636, 0.9991721854304636, 0.9991721854304636, 1.0, 0.9991721854304636, 1.0, 0.9991721854304636, 0.9991721854304636, 0.9975165562913907, 0.9991721854304636, 0.9966887417218543, 0.9991721854304636], label="Keras (w/ val split)")
+axs[0,1].plot([0.942975, 0.980992, 0.983471, 0.98595, 0.98595, 0.985124, 0.98595, 0.984298, 0.985124, 0.985124, 0.985124, 0.98595, 0.98595, 0.98595, 0.986777, 0.98595, 0.98595, 0.98595, 0.98595, 0.98595, 0.986777, 0.986777, 0.986777, 0.98595, 0.986777, 0.986777, 0.986777, 0.98595, 0.98595, 0.98595, 0.986777, 0.98595, 0.986777, 0.98595, 0.98595, 0.98595, 0.98595, 0.98595, 0.98595, 0.98595, 0.98595, 0.985124, 0.98595, 0.984298, 0.985124, 0.98595, 0.984298, 0.985124, 0.984298, 0.984298], label="Jacobian", color="g")
+
+axs[1,0].plot([0.10888315538565317, 0.06680880729798917, 0.053077199889553915, 0.07688223214613067, 0.045733443758002035, 0.03640310698085361, 0.031724032787261186, 0.04391241666895372, 0.031088130314041066, 0.026419770469268165, 0.026483222897405977, 0.02286748499781997, 0.023777789181029355, 0.01819943821540585, 0.021010621030021596, 0.017266716283780556, 0.015340352886252934, 0.016395327476439653, 0.01578087834296403, 0.015752164343440973, 0.01598852108474131, 0.01501362330659672, 0.016887195642899585, 0.016618405779202778, 0.011775514552438702, 0.013678248254237352, 0.011577272056429475, 0.011427491268625967, 0.014759536873963143, 0.011453486385720748, 0.013112282615017007, 0.012316111875353036, 0.017063750271443967, 0.010874567663779965, 0.011459515726676694, 0.011196554424586119, 0.010264440970840278, 0.011569284523526827, 0.009628297457540477, 0.009635593286818928, 0.01166622194427031, 0.016940338909626006, 0.014556518707562376, 0.01091362305537418, 0.019304899429833447, 0.014454653472812088, 0.018006062010924022, 0.018680620772971047, 0.014109227254434867, 0.011193500597167899], label="Keras (w/ val split)")
+axs[1,0].plot([0.0234097, 0.0141612, 0.0114124, 0.00986921, 0.00872293, 0.00805279, 0.00776756, 0.00758032, 0.00742217, 0.00729374, 0.0071354, 0.00701655, 0.0068978, 0.00680974, 0.00674059, 0.00668204, 0.00665047, 0.00661398, 0.00658959, 0.00655535, 0.00652, 0.00649501, 0.00647428, 0.00644689, 0.00643667, 0.00639617, 0.00637101, 0.00636733, 0.00634465, 0.00632476, 0.00632126, 0.00630413, 0.00628944, 0.00627891, 0.00626959, 0.0062554, 0.00623879, 0.00623331, 0.00622584, 0.00621284, 0.00620501, 0.00619419, 0.00618274, 0.00617308, 0.00616499, 0.00615112, 0.00612101, 0.00609225, 0.00607023, 0.0060534], label="Jacobian", color="g")
+
+axs[1,1].plot([0.8518518527348836, 0.9481481481481482, 1.0, 0.9481481481481482, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], label="Keras (w/ val split)")
+axs[1,1].plot([0.954887, 0.954887, 0.954887, 0.954887, 0.954887, 0.954887, 0.954887, 0.954887, 0.954887, 0.954887, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406, 0.962406], label="Jacobian", color="g")
+
+axs[0,0].legend(loc="upper right")
+axs[0,1].legend(loc="lower right")
+axs[1,0].legend(loc="upper right")
+axs[1,1].legend(loc="lower right")
+axs[0,0].set_xlabel("Epoch")
+axs[0,0].set_ylabel("Loss")
+axs[0,1].set_xlabel("Epoch")
+axs[0,1].set_ylabel("Accuracy")
+axs[1,0].set_xlabel("Epoch")
+axs[1,0].set_ylabel("Validation loss")
+axs[1,1].set_xlabel("Epoch")
+axs[1,1].set_ylabel("Validation accuracy")
+fig.suptitle("Model metrics over epochs", fontsize=16)
plt.show()
-x = ['Keras', 'Scikit-Learn', 'Jacobian (Python)', 'Jacobian (C++)']
-speed = [2.87004685402, 2.074756145477295,0.05295228958129883, 0.0521543]
+# x = ['Keras', 'Scikit-Learn', 'Jacobian (Python)', 'Jacobian (C++)']
+# speed = [2.87004685402, 2.074756145477295,0.05295228958129883, 0.0521543]
-x_pos = [i for i, _ in enumerate(x)]
+# x_pos = [i for i, _ in enumerate(x)]
-plt.bar(x_pos, speed, color='green')
-plt.ylabel("Time (s)")
-plt.title("Runtime for Task")
+# plt.bar(x_pos, speed, color='green')
+# plt.ylabel("Time (s)")
+# plt.title("Runtime for Task")
-plt.xticks(x_pos, x)
+# plt.xticks(x_pos, x)
plt.show()
diff --git a/bench/kerasdemo.py b/bench/kerasdemo.py
@@ -38,9 +38,12 @@ def kerasbench(batch_sz, layers):
model.add(Dense(1, activation='linear'))
opt = keras.optimizers.SGD(lr=0.0155)
model.compile(loss='mse', optimizer=opt, metrics=['accuracy'])
- history = model.fit(X, y, epochs=50, batch_size=batch_sz)
+ history = model.fit(X, y, epochs=50, validation_split=0.1, batch_size=batch_sz)
end = time.time()
print(history.history['loss'])
+ print(history.history['acc'])
+ print(history.history['val_loss'])
+ print(history.history['val_acc'])
return (end-init)
# sum = 0
diff --git a/example.cpp b/example.cpp
@@ -14,7 +14,7 @@
double bench(int batch_sz)
{
auto start = std::chrono::high_resolution_clock::now();
- Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 2, 0.1, 0.9);
+ Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, 2, 0.01, 0.9);
net.add_layer(4, "linear");
net.add_layer(5, "relu");
net.add_layer(2, "linear");
@@ -22,7 +22,7 @@ double bench(int batch_sz)
std::vector<float> vals;
for (int i = 0; i < 50; i++) {
net.train();
- vals.push_back(net.get_cost());
+ vals.push_back(net.get_val_acc());
}
std::cout << "[";
for (int i = 0; i < vals.size(); i++) {
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -57,7 +57,7 @@ Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, in
test_instances = split_file(SHUFFLED_PATH, total_instances, ratio);
instances = total_instances - test_instances;
data = fopen(TRAIN_PATH, "r");
- test_data = fopen(path, "r");
+ test_data = fopen(TEST_PATH, "r");
decay = [](float lr, float t) -> float {
return lr;
};
