commit c7de52be83359e68e16abaeacba067a3392f73b5
parent 6fd978b1f4a97ae789a13e0c2f20638672848aa5
Author: quantumish <freifeld.david@gmail.com>
Date: Sat, 25 Jul 2026 15:59:41 -0700
Clean up files for easier reading, remove GSL dependency
Diffstat:
13 files changed, 649 insertions(+), 684 deletions(-)
diff --git a/UNLICENSE b/UNLICENSE
@@ -1,24 +0,0 @@
-This is free and unencumbered software released into the public domain.
-
-Anyone is free to copy, modify, publish, use, compile, sell, or
-distribute this software, either in source code form or as a compiled
-binary, for any purpose, commercial or non-commercial, and by any
-means.
-
-In jurisdictions that recognize copyright laws, the author or authors
-of this software dedicate any and all copyright interest in the
-software to the public domain. We make this dedication for the benefit
-of the public at large and to the detriment of our heirs and
-successors. We intend this dedication to be an overt act of
-relinquishment in perpetuity of all present and future rights to this
-software under copyright law.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
-
-For more information, please refer to <https://unlicense.org>
diff --git a/checks.cpp b/checks.cpp
@@ -2,8 +2,6 @@
// checks.cpp
// Jacobian
//
-// Created by David Freifeld
-//
#include "./src/utils.hpp"
#include "./src/bpnn.hpp"
diff --git a/example.cpp b/example.cpp
@@ -1,15 +1,3 @@
-//
-// example.cpp
-// Jacobian
-//
-// Created by David Freifeld
-//
-
-// #include <indicators/cursor_control.hpp>
-// #include <indicators/progress_bar.hpp>
-// #include <indicators/block_progress_bar.hpp>
-// using namespace indicators;
-
#include "src/bpnn.hpp"
#include "src/utils.hpp"
#include "unistd.h"
@@ -18,28 +6,29 @@
double bench(int batch_sz, int epochs)
{
- auto start = std::chrono::high_resolution_clock::now();
- Jacobian::Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155, 0.03, Jacobian::Regularization::L2, 0, 0.9);
- net.add_layer(4, Jacobian::activations::linear, Jacobian::activations::linear_deriv);
- net.add_layer(5, Jacobian::activations::lecun_tanh, Jacobian::activations::lecun_tanh_deriv);
- net.add_layer(2, Jacobian::activations::linear, Jacobian::activations::linear_deriv);
- net.init_optimizer(Jacobian::optimizers::momentum(0.1));
- net.initialize();
- for (int i = 0; i < epochs; i++) {
- net.train();
- }
- auto end = std::chrono::high_resolution_clock::now();
- return std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count() / pow(10,9);
+ auto start = std::chrono::high_resolution_clock::now();
+ Jacobian::Network net ("./data_banknote_authentication.txt", batch_sz, 0.0155,
+ 0.03, Jacobian::Regularization::L2, 0, 0.9);
+ net.add_layer(4, Jacobian::activations::linear, Jacobian::activations::linear_deriv);
+ net.add_layer(5, Jacobian::activations::lecun_tanh, Jacobian::activations::lecun_tanh_deriv);
+ net.add_layer(2, Jacobian::activations::linear, Jacobian::activations::linear_deriv);
+ net.init_optimizer(Jacobian::optimizers::momentum(0.1));
+ net.initialize();
+ for (int i = 0; i < epochs; i++) {
+ net.train();
+ }
+ auto end = std::chrono::high_resolution_clock::now();
+ return std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count() / pow(10,9);
}
int main(int argc, char** argv)
{
- if (argc < 2) {
- std::cout << "Invalid command! Either pass a special option or pass two integers - batch_size and epochs (in that order)." << "\n";
- exit(1);
- }
- else {
- sleep(strtol(argv[3], NULL, 10));
- std::cout << bench(strtol(argv[1], NULL, 10), strtol(argv[2], NULL, 10)) << "\n";
- }
+ if (argc < 2) {
+ std::cout << "usage: jacobian_cli [batch_size] [epoch] [time to sleep before starting]" << "\n";
+ exit(1);
+ }
+ else {
+ sleep(strtol(argv[3], NULL, 10));
+ std::cout << bench(strtol(argv[1], NULL, 10), strtol(argv[2], NULL, 10)) << "\n";
+ }
}
diff --git a/readme.md b/readme.md
@@ -1,17 +1,24 @@
<!-- readme.md -->
<!-- Jacobian -->
- <!-- Created by David Freifeld -->
- <!-- Markdown has the worst comment syntax I've ever seen. Seriously, what is this. TODO: Migrate this README to Org Mode. -->
+ <!-- Markdown has the worst comment syntax I've ever seen. Seriously, what is this. -->
+ <!-- TODO: Migrate this README to Org Mode. -->
## About
-Jacobian is a work-in-progress machine learning library written in C++ designed to run as fast as possible while still being simple to use. Jacobian is accessible via Python and enables you to write models that train faster with the same amount of code. As of now, Jacobian supports feedforward neural networks and has partial support for convolutional neural networks.
+Jacobian is a work-in-progress machine learning library written in C++
+designed to run as fast as possible while still being simple to
+use. Jacobian is accessible via Python and enables you to write models that
+train faster with the same amount of code. As of now, Jacobian supports
+feedforward neural networks and has partial support for convolutional
+neural networks.
## Usage
-Initializing and training a neural network with Jacobian takes just 8 lines of code!
+Initializing and training a neural network with Jacobian takes just 8 lines
+of code!
+
```python
import jacobian as jcb
net = jcb.Network("./data_banknote_authentication.txt", 10, 0.0155, 0.03, jcb.L2, 1, 0.9)
@@ -26,39 +33,40 @@ for i in range(50):
```
## Examples
-See `example.cpp` for an example of using Jacobian from C++, and `example.py` for an example of using Jacobian from Python.
+See `example.cpp` for an example of using Jacobian from C++, and
+`example.py` for an example of using Jacobian from Python.
## Building
### Dependencies
-Eigen 3 is the only dependency, although building the python library requires `pybind11`.
-
-<!-- ### Main Steps
-Note: This is all ideally the process for manual building, but it's so confusing that I'm not sure. You're better off manually copying the .so file!
-1. Install the C++ library Eigen. Ensure the that the path eigen3/Eigen/Dense is within one of your include directories.
-2. Install the C++ and python ends of the pybind11 library.
-3. Run `make` with the configuration of your choosing.
-4. Run `python setup.py bdist_wheel`
-5. `cd` into `dist` and run `python3 -m pip install --upgrade Jacobian-1.0-cp37-cp37m-macosx_10_13_x86_64.whl`
-6. Be unhappy when it doesn't work out and resort to just copying .so files. -->
+Eigen 3 is the only dependency, although building the python library
+requires `pybind11`.
### Building with CMake
-**Don't forget to delete `CMakeCache.txt` after each compilation if you plan on switching things up!**
+**Don't forget to delete `CMakeCache.txt` after each compilation if you
+plan on switching things up!**
-There are two target languages, five main build configurations, and a number of toggleable build 'attributes'. The preferred target language can be specified by setting a CMake variable from the command-line: `-DCXX=ON` or `-DPYTHON=ON`.
+There are two target languages, five main build configurations, and a
+number of toggleable build 'attributes'. The preferred target language can
+be specified by setting a CMake variable from the command-line: `-DCXX=ON`
+or `-DPYTHON=ON`.
The five main configurations correspond to differing levels of optimization.
- `cmake .`: No compiler optimizations.
- `cmake . -DFAST=ON`: Enables the O3 optimization layer in the compiler.
- `cmake . -DFASTER=ON`: Enables O3 as well as extra individual flags.
-- `cmake . -DTRADEOFFS=ON`: All previous optimizations as well as ones that sacrifice precision.
-- `cmake . -DRECKLESS=ON`: Like `TRADEOFFS`, but defines the RECKLESS macro (and NDEBUG) which skips all checks within the code.
+- `cmake . -DTRADEOFFS=ON`: All previous optimizations as well as ones that
+ sacrifice precision.
+- `cmake . -DRECKLESS=ON`: Like `TRADEOFFS`, but defines the RECKLESS macro
+ (and NDEBUG) which skips all checks within the code.
-One you've selected a main optimization level, extra configurations can be passed in.
+One you've selected a main optimization level, extra configurations can be
+passed in.
-- `-DDEBUG=ON` enables debugging features in the compiler (and shows warnings).
+- `-DDEBUG=ON` enables debugging features in the compiler (and shows
+ warnings).
A sample build process would look like this:
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -1,10 +1,3 @@
-//
-// bpnn.cpp
-// Jacobian
-//
-// Created by David Freifeld
-//
-
#include "bpnn.hpp"
#include "utils.hpp"
#include <random>
@@ -12,323 +5,332 @@
namespace Jacobian {
Layer::Layer(int batch_sz, int nodes)
{
- contents = Eigen::MatrixXf(batch_sz, nodes);
- dZ = Eigen::MatrixXf(batch_sz, nodes);
- int datalen = batch_sz*nodes;
- for (int i = 0; i < datalen; i++) {
- contents(static_cast<int>(i / nodes),i%nodes) = 0;
- dZ(static_cast<int>(i / nodes),i%nodes) = 0;
- }
- bias = Eigen::MatrixXf (batch_sz, nodes);
- for (int i = 0; i < nodes; i++) {
- for (int j = 0; j < batch_sz; j++) bias(j, i) = 0;
- }
+ contents = Eigen::MatrixXf(batch_sz, nodes);
+ dZ = Eigen::MatrixXf(batch_sz, nodes);
+ int datalen = batch_sz*nodes;
+ for (int i = 0; i < datalen; i++) {
+ contents(static_cast<int>(i / nodes),i%nodes) = 0;
+ dZ(static_cast<int>(i / nodes),i%nodes) = 0;
+ }
+ bias = Eigen::MatrixXf (batch_sz, nodes);
+ for (int i = 0; i < nodes; i++) {
+ for (int j = 0; j < batch_sz; j++) bias(j, i) = 0;
+ }
}
void Layer::init_weights(Layer next)
{
- v = Eigen::MatrixXf (contents.cols(), next.contents.cols());
- m = Eigen::MatrixXf (contents.cols(), next.contents.cols());
- weights = Eigen::MatrixXf (contents.cols(), next.contents.cols());
- int nodes = weights.cols();
- int n = contents.cols() + next.contents.cols();
- std::normal_distribution<float> d(0,sqrt(1.0/n));
- for (int i = 0; i < (weights.rows()*weights.cols()); i++) {
- std::random_device rd;
- std::mt19937 gen(rd());
- weights(static_cast<int>(i / nodes), i%nodes) = d(gen);
- v(static_cast<int>(i / nodes), i%nodes) = 0;
- m(static_cast<int>(i / nodes), i%nodes) = 0;
- }
+ v = Eigen::MatrixXf (contents.cols(), next.contents.cols());
+ m = Eigen::MatrixXf (contents.cols(), next.contents.cols());
+ weights = Eigen::MatrixXf (contents.cols(), next.contents.cols());
+ int nodes = weights.cols();
+ int n = contents.cols() + next.contents.cols();
+ std::normal_distribution<float> d(0,sqrt(1.0/n));
+ for (int i = 0; i < (weights.rows()*weights.cols()); i++) {
+ std::random_device rd;
+ std::mt19937 gen(rd());
+ weights(static_cast<int>(i / nodes), i%nodes) = d(gen);
+ v(static_cast<int>(i / nodes), i%nodes) = 0;
+ m(static_cast<int>(i / nodes), i%nodes) = 0;
+ }
}
-Network::Network(const char* path, int batch_sz, float learn_rate, float bias_rate, Regularization regularization, float l, float ratio, bool early_exit, float cutoff)
- :batch_size(batch_sz), learning_rate(learn_rate), bias_lr(bias_rate), reg_type(regularization),
- lambda(l), early_stop(early_exit), threshold(cutoff)
+Network::Network(const char* path, int batch_sz, float learn_rate, float bias_rate,
+ Regularization regularization, float l, float ratio, bool early_exit, float cutoff)
+ :batch_size(batch_sz), learning_rate(learn_rate), bias_lr(bias_rate), reg_type(regularization),
+ lambda(l), early_stop(early_exit), threshold(cutoff)
{
- Expects(batch_size > 0 && learning_rate > 0 &&
- bias_rate > 0 && l >= 0 && ratio >= 0 && ratio <= 1);
- int total_instances = prep_file(path, SHUFFLED_PATH);
- val_instances = split_file(SHUFFLED_PATH, total_instances, ratio);
- prep(TRAIN_PATH, TRAIN_BIN_PATH);
- prep(VAL_PATH, VAL_BIN_PATH);
- data = open(TRAIN_BIN_PATH, O_RDONLY | O_NONBLOCK);
- val_data = open(VAL_BIN_PATH, O_RDONLY | O_NONBLOCK);
- instances = total_instances - val_instances;
- decay = [](float& learning_rate) -> void {};
- update = [](Layer& layer, const Eigen::MatrixXf delta, const float learning_rate) {
- layer.weights = (learning_rate * delta);
- };
- // File descriptors are nonnegative integers and open() returns -1 on failure.
- Ensures(batch_size < instances && data > 0 && val_data > 0);
+ Expects(batch_size > 0 && learning_rate > 0 &&
+ bias_rate > 0 && l >= 0 && ratio >= 0 && ratio <= 1);
+ int total_instances = prep_file(path, SHUFFLED_PATH);
+ val_instances = split_file(SHUFFLED_PATH, total_instances, ratio);
+ prep(TRAIN_PATH, TRAIN_BIN_PATH);
+ prep(VAL_PATH, VAL_BIN_PATH);
+ data = open(TRAIN_BIN_PATH, O_RDONLY | O_NONBLOCK);
+ val_data = open(VAL_BIN_PATH, O_RDONLY | O_NONBLOCK);
+ instances = total_instances - val_instances;
+ decay = [](float& learning_rate) -> void {};
+ update = [](Layer& layer, const Eigen::MatrixXf delta, const float learning_rate) {
+ layer.weights = (learning_rate * delta);
+ };
+ // File descriptors are nonnegative integers and open() returns -1 on failure.
+ Ensures(batch_size < instances && data > 0 && val_data > 0);
}
Network::~Network()
{
- close(data);
- close(val_data);
+ close(data);
+ close(val_data);
}
-void Network::add_layer(int nodes, std::function<float(float)> activation, std::function<float(float)> activation_deriv)
+void Network::add_layer(int nodes, std::function<float(float)> activation,
+ std::function<float(float)> activation_deriv)
{
- Expects(nodes > 0);
- length++;
- layers.emplace_back(batch_size, nodes);
- layers[length-1].activation = activation;
- layers[length-1].activation_deriv = activation_deriv;
+ Expects(nodes > 0);
+ length++;
+ layers.emplace_back(batch_size, nodes);
+ layers[length-1].activation = activation;
+ layers[length-1].activation_deriv = activation_deriv;
}
void Network::initialize()
{
- Expects(length > 1);
- labels = new Eigen::MatrixXf (batch_size,layers[length-1].contents.cols());
- for (int i = 0; i < length-1; i++) layers[i].init_weights(layers[i+1]);
+ Expects(length > 1);
+ labels = new Eigen::MatrixXf (batch_size,layers[length-1].contents.cols());
+ for (int i = 0; i < length-1; i++) layers[i].init_weights(layers[i+1]);
}
-void Network::set_activation(int index, std::function<float(float)> custom, std::function<float(float)> custom_deriv)
+void Network::set_activation(int index, std::function<float(float)> custom,
+ std::function<float(float)> custom_deriv)
{
- Expects(index >= 0 && index < length);
- layers[index].activation = custom;
- layers[index].activation_deriv = custom_deriv;
+ Expects(index >= 0 && index < length);
+ layers[index].activation = custom;
+ layers[index].activation_deriv = custom_deriv;
}
void Network::softmax()
{
- for (int i = 0; i < layers[length-1].contents.rows(); i++) {
- Eigen::MatrixXf m = layers[length-1].contents.block(i,0,1,layers[length-1].contents.cols());
- Eigen::MatrixXf::Index maxRow, maxCol;
- float max = m.maxCoeff(&maxRow, &maxCol);
- m = (m.array() - max).matrix();
- float sum = 0;
- for (int j = 0; j < layers[length-1].contents.cols(); j++) {
- sum += exp(m(0,j));
- }
- for (int j = 0; j < layers[length-1].contents.cols(); j++) {
- m(0,j) = exp(m(0,j))/sum;
- }
- layers[length-1].contents.block(i,0,1,layers[length-1].contents.cols()) = m;
- }
+ for (int i = 0; i < layers[length-1].contents.rows(); i++) {
+ Eigen::MatrixXf m = layers[length-1].contents.block(i,0,1,layers[length-1].contents.cols());
+ Eigen::MatrixXf::Index maxRow, maxCol;
+ float max = m.maxCoeff(&maxRow, &maxCol);
+ m = (m.array() - max).matrix();
+ float sum = 0;
+ for (int j = 0; j < layers[length-1].contents.cols(); j++) {
+ sum += exp(m(0,j));
+ }
+ for (int j = 0; j < layers[length-1].contents.cols(); j++) {
+ m(0,j) = exp(m(0,j))/sum;
+ }
+ layers[length-1].contents.block(i,0,1,layers[length-1].contents.cols()) = m;
+ }
}
void Network::feedforward()
{
- for (int i = 0; i < length-1; i++) {
- for (int j = 0; j < layers[i].contents.rows(); j++) {
- for (int k = 0; k < layers[i].contents.cols(); k++) {
- layers[i].dZ(j,k) = layers[i].activation_deriv(layers[i].contents(j,k));
- layers[i].contents(j,k) = layers[i].activation(layers[i].contents(j,k));
- }
- }
- layers[i+1].contents = layers[i].contents * layers[i].weights;
- layers[i+1].contents += layers[i+1].bias;
- }
- for (int j = 0; j < layers[length-1].contents.rows(); j++) {
- for (int k = 0; k < layers[length-1].contents.cols(); k++) {
- layers[length-1].dZ(j,k) = layers[length-1].activation_deriv(layers[length-1].contents(j,k));
- layers[length-1].contents(j,k) = layers[length-1].activation(layers[length-1].contents(j,k));
- }
- }
- softmax();
+ for (int i = 0; i < length-1; i++) {
+ for (int j = 0; j < layers[i].contents.rows(); j++) {
+ for (int k = 0; k < layers[i].contents.cols(); k++) {
+ layers[i].dZ(j,k) = layers[i].activation_deriv(layers[i].contents(j,k));
+ layers[i].contents(j,k) = layers[i].activation(layers[i].contents(j,k));
+ }
+ }
+ layers[i+1].contents = layers[i].contents * layers[i].weights;
+ layers[i+1].contents += layers[i+1].bias;
+ }
+ for (int j = 0; j < layers[length-1].contents.rows(); j++) {
+ for (int k = 0; k < layers[length-1].contents.cols(); k++) {
+ layers[length-1].dZ(j,k) = layers[length-1].activation_deriv(layers[length-1].contents(j,k));
+ layers[length-1].contents(j,k) = layers[length-1].activation(layers[length-1].contents(j,k));
+ }
+ }
+ softmax();
}
std::function<void(float&)> decays::step(float a_0, float k)
{
- return [a_0, k](float& learning_rate) -> void {
- learning_rate = a_0 * learning_rate/k;
- };
+ return [a_0, k](float& learning_rate) -> void {
+ learning_rate = a_0 * learning_rate/k;
+ };
}
std::function<void(float&)> decays::exponential(float a_0, float k)
{
- int epochs = 0;
- return [a_0, k, epochs](float& learning_rate) mutable -> void {
- learning_rate = a_0 * exp(-k * epochs);
- epochs++;
- };
+ int epochs = 0;
+ return [a_0, k, epochs](float& learning_rate) mutable -> void {
+ learning_rate = a_0 * exp(-k * epochs);
+ epochs++;
+ };
}
std::function<void(float&)> decays::fractional(float a_0, float k)
{
- int epochs = 0;
- return [a_0, k, epochs](float& learning_rate) mutable -> void {
- learning_rate = a_0 / (1+(k * epochs));
- epochs++;
- };
+ int epochs = 0;
+ return [a_0, k, epochs](float& learning_rate) mutable -> void {
+ learning_rate = a_0 / (1+(k * epochs));
+ epochs++;
+ };
}
std::function<void(float&)> decays::linear(int max_ep)
{
- int epochs = 0;
- return [max_ep, epochs](float& learning_rate) mutable -> void {
- learning_rate = 1 - epochs/max_ep;
- epochs++;
- };
+ int epochs = 0;
+ return [max_ep, epochs](float& learning_rate) mutable -> void {
+ learning_rate = 1 - epochs/max_ep;
+ epochs++;
+ };
}
void Network::init_decay(std::function<void(float&)> f)
{
- decay = f;
+ decay = f;
}
void Network::list_net()
{
- Expects(length > 1);
- std::cout << "-----------------------\nINPUT LAYER (LAYER 0)\n"
- << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << layers[0].contents
- << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << layers[0].weights
- << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << layers[0].bias << "\n\n\n";
- for (int i = 1; i < length-1; i++) {
- std::cout << "-----------------------\nLAYER " << i
- << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << layers[i].contents
- << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << layers[i].bias
- << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << layers[i].weights << "\n\n\n";
- }
- std::cout << "-----------------------\nOUTPUT LAYER (LAYER " << length-1
- <<"\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << layers[length-1].contents
- << "\n\n\u001b[31BIASES:\x1B[0;37m\n" << layers[length-1].bias << "\n\n\n";
+ Expects(length > 1);
+ std::cout << "-----------------------\nINPUT LAYER (LAYER 0)\n"
+ << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << layers[0].contents
+ << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << layers[0].weights
+ << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << layers[0].bias << "\n\n\n";
+ for (int i = 1; i < length-1; i++) {
+ std::cout << "-----------------------\nLAYER " << i
+ << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << layers[i].contents
+ << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << layers[i].bias
+ << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << layers[i].weights << "\n\n\n";
+ }
+ std::cout << "-----------------------\nOUTPUT LAYER (LAYER " << length-1
+ <<"\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << layers[length-1].contents
+ << "\n\n\u001b[31BIASES:\x1B[0;37m\n" << layers[length-1].bias << "\n\n\n";
}
float Network::cost()
{
- float sum = 0;
- float reg = 0; // Regularization term
- for (int i = 0; i < layers[length-1].contents.rows(); i++) {
- float tempsum = 0;
- for (int j = 0; j < layers[length-1].contents.cols(); j++) {
- float truth;
- if (j==(*labels)(i,0)) truth = 1;
- else truth = 0;
- if (layers[length-1].contents(i,j) == 0) layers[length-1].contents(i,j) += 0.00001;
- tempsum += truth * log(layers[length-1].contents(i,j));
- }
- sum-=tempsum;
- }
- for (unsigned long i = 0; i < layers.size()-1; i++) {
- if (reg_type == Regularization::L2) reg += layers[i].weights.cwiseProduct(layers[i].weights).sum();
- else if (reg_type == Regularization::L1) reg += (layers[i].weights.array().abs().matrix()).sum();
- }
- return ((1.0/batch_size) * sum) + (1/2*lambda*reg);
+ float sum = 0;
+ float reg = 0; // Regularization term
+ for (int i = 0; i < layers[length-1].contents.rows(); i++) {
+ float tempsum = 0;
+ for (int j = 0; j < layers[length-1].contents.cols(); j++) {
+ float truth;
+ if (j==(*labels)(i,0)) truth = 1;
+ else truth = 0;
+ if (layers[length-1].contents(i,j) == 0) layers[length-1].contents(i,j) += 0.00001;
+ tempsum += truth * log(layers[length-1].contents(i,j));
+ }
+ sum-=tempsum;
+ }
+ for (unsigned long i = 0; i < layers.size()-1; i++) {
+ if (reg_type == Regularization::L2) {
+ reg += layers[i].weights.cwiseProduct(layers[i].weights).sum();
+ } else if (reg_type == Regularization::L1) {
+ reg += (layers[i].weights.array().abs().matrix()).sum();
+ }
+ }
+ return ((1.0/batch_size) * sum) + (1/2*lambda*reg);
}
float Network::accuracy()
{
- float correct = 0;
- for (int i = 0; i < layers[length-1].contents.rows(); i++) {
- float ans = -INFINITY;
- float index = -1;
- for (int j = 0; j < layers[length-1].contents.cols(); j++) {
- if (layers[length-1].contents(i, j) > ans) {
- ans = layers[length-1].contents(i, j);
- index = j;
- }
- }
- if ((*labels)(i, 0) == index) correct += 1;
- }
- return (1.0/batch_size) * correct;
+ float correct = 0;
+ for (int i = 0; i < layers[length-1].contents.rows(); i++) {
+ float ans = -INFINITY;
+ float index = -1;
+ for (int j = 0; j < layers[length-1].contents.cols(); j++) {
+ if (layers[length-1].contents(i, j) > ans) {
+ ans = layers[length-1].contents(i, j);
+ index = j;
+ }
+ }
+ if ((*labels)(i, 0) == index) correct += 1;
+ }
+ return (1.0/batch_size) * correct;
}
Eigen::MatrixXf l1_deriv(Eigen::MatrixXf m)
{
- Eigen::MatrixXf r(m.rows(), m.cols());
- for (int i = 0; i < m.rows(); i++) {
- for (int j = 0; j < m.cols(); j++) {
- if (m(i,j) == 0) r(i,j) = 0;
- else r(i,j) = 1;
- }
- }
- return r;
+ Eigen::MatrixXf r(m.rows(), m.cols());
+ for (int i = 0; i < m.rows(); i++) {
+ for (int j = 0; j < m.cols(); j++) {
+ if (m(i,j) == 0) r(i,j) = 0;
+ else r(i,j) = 1;
+ }
+ }
+ return r;
}
Eigen::MatrixXf Network::backpropagate()
{
- std::vector<Eigen::MatrixXf> gradients;
- std::vector<Eigen::MatrixXf> deltas;
- Eigen::MatrixXf error (layers[length-1].contents.rows(), layers[length-1].contents.cols());
- for (int i = 0; i < error.rows(); i++) {
- for (int j = 0; j < error.cols(); j++) {
- float truth;
- if (j==(*labels)(i,0)) truth = 1;
- else truth = 0;
- error(i,j) = layers[length-1].contents(i,j) - truth;
- }
- }
- gradients.push_back(error);
- deltas.push_back(layers[length-2].contents.transpose() * gradients[0]);
- int counter = 1;
- for (int i = length-2; i >= 1; i--) {
- // TODO: Add nesterov momentum | -p B -t conundrum -t coding -m Without causing segmentation faults.
- // (*layers[i].weights-((learning_rate * *layers[i].weights) + (0.9 * *layers[i].v))).transpose()
- //grad_calc(gradients, counter, i)
- gradients.push_back((gradients[counter-1] * layers[i].weights.transpose()).cwiseProduct(layers[i].dZ));
- deltas.push_back(layers[i-1].contents.transpose() * gradients[counter]);
- counter++;
- }
- for (int i = 0; i < length-1; i++) {
- update(layers[length-2-i], deltas[i], learning_rate);
- if (reg_type == Regularization::L2) layers[length-2-i].weights -= ((lambda/batch_size) * (layers[length-2-i].weights));
- else if (reg_type == Regularization::L1) layers[length-2-i].weights -= ((lambda/(2*batch_size)) * l1_deriv(layers[length-2-i].weights));
- layers[length-1-i].bias -= bias_lr * gradients[i];
- }
- return gradients.back();
+ std::vector<Eigen::MatrixXf> gradients;
+ std::vector<Eigen::MatrixXf> deltas;
+ Eigen::MatrixXf error (layers[length-1].contents.rows(), layers[length-1].contents.cols());
+ for (int i = 0; i < error.rows(); i++) {
+ for (int j = 0; j < error.cols(); j++) {
+ float truth;
+ if (j==(*labels)(i,0)) truth = 1;
+ else truth = 0;
+ error(i,j) = layers[length-1].contents(i,j) - truth;
+ }
+ }
+ gradients.push_back(error);
+ deltas.push_back(layers[length-2].contents.transpose() * gradients[0]);
+ int counter = 1;
+ for (int i = length-2; i >= 1; i--) {
+ // TODO: Add nesterov momentum
+ // (*layers[i].weights-((learning_rate * *layers[i].weights) + (0.9 * *layers[i].v))).transpose()
+ //grad_calc(gradients, counter, i)
+ gradients.push_back((gradients[counter-1] * layers[i].weights.transpose()).cwiseProduct(layers[i].dZ));
+ deltas.push_back(layers[i-1].contents.transpose() * gradients[counter]);
+ counter++;
+ }
+ for (int i = 0; i < length-1; i++) {
+ update(layers[length-2-i], deltas[i], learning_rate);
+ if (reg_type == Regularization::L2) {
+ layers[length-2-i].weights -= ((lambda/batch_size) * (layers[length-2-i].weights));
+ } else if (reg_type == Regularization::L1) {
+ layers[length-2-i].weights -= ((lambda/(2*batch_size)) * l1_deriv(layers[length-2-i].weights));
+ }
+ layers[length-1-i].bias -= bias_lr * gradients[i];
+ }
+ return gradients.back();
}
#include "data.cpp"
void Network::validate(const char* path)
{
- if (val_instances == 0) return;
- float costsum = 0;
- float accsum = 0;
- for (int i = 0; i <= val_instances-batch_size; i+=batch_size) {
- next_batch(val_data);
- feedforward();
- costsum += cost();
- accsum += accuracy();
- }
- val_acc = 1.0/(static_cast<float>(val_instances/batch_size)) * accsum;
- val_cost = 1.0/(static_cast<float>(val_instances/batch_size)) * costsum;
- val_data = open(VAL_BIN_PATH, O_RDONLY | O_NONBLOCK);
- Ensures(lseek(val_data, 0, SEEK_CUR) == 0);
+ if (val_instances == 0) return;
+ float costsum = 0;
+ float accsum = 0;
+ for (int i = 0; i <= val_instances-batch_size; i+=batch_size) {
+ next_batch(val_data);
+ feedforward();
+ costsum += cost();
+ accsum += accuracy();
+ }
+ val_acc = 1.0/(static_cast<float>(val_instances/batch_size)) * accsum;
+ val_cost = 1.0/(static_cast<float>(val_instances/batch_size)) * costsum;
+ val_data = open(VAL_BIN_PATH, O_RDONLY | O_NONBLOCK);
+ Ensures(lseek(val_data, 0, SEEK_CUR) == 0);
}
void Network::interactive_next_batch()
{
- if (batches < instances/batch_size-batch_size) next_batch(data);
- else {
- batches = 0;
- data = open(TRAIN_BIN_PATH, O_RDONLY | O_NONBLOCK);
- decay(learning_rate);
- }
- batches++;
+ if (batches < instances/batch_size-batch_size) next_batch(data);
+ else {
+ batches = 0;
+ data = open(TRAIN_BIN_PATH, O_RDONLY | O_NONBLOCK);
+ decay(learning_rate);
+ }
+ batches++;
}
void Network::train()
{
- float cost_sum = 0;
- float acc_sum = 0;
- for (int i = 0; i <= instances - batch_size; i += batch_size) {
- if (i != instances - batch_size)
- next_batch(data);
- feedforward();
- backpropagate();
- cost_sum += cost();
- acc_sum += accuracy();
- batches++;
- }
- epoch_acc =
- 1.0 / (static_cast<float>(instances / batch_size)) * acc_sum;
- epoch_cost =
- 1.0 / (static_cast<float>(instances / batch_size)) * cost_sum;
- validate(VAL_PATH);
- if (silenced == false)
- printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n",
- epochs, epoch_cost, epoch_acc, val_cost, val_acc);
- batches = 1;
- data = open(TRAIN_BIN_PATH, O_RDONLY | O_NONBLOCK);
- decay(learning_rate);
- epochs++;
- Ensures(lseek(data, 0, SEEK_CUR) == 0);
+ float cost_sum = 0;
+ float acc_sum = 0;
+ for (int i = 0; i <= instances - batch_size; i += batch_size) {
+ if (i != instances - batch_size)
+ next_batch(data);
+ feedforward();
+ backpropagate();
+ cost_sum += cost();
+ acc_sum += accuracy();
+ batches++;
+ }
+ epoch_acc =
+ 1.0 / (static_cast<float>(instances / batch_size)) * acc_sum;
+ epoch_cost =
+ 1.0 / (static_cast<float>(instances / batch_size)) * cost_sum;
+ validate(VAL_PATH);
+ if (silenced == false)
+ printf("Epoch %i complete - cost %f - acc %f - val_cost %f - val_acc %f\n",
+ epochs, epoch_cost, epoch_acc, val_cost, val_acc);
+ batches = 1;
+ data = open(TRAIN_BIN_PATH, O_RDONLY | O_NONBLOCK);
+ decay(learning_rate);
+ epochs++;
+ Ensures(lseek(data, 0, SEEK_CUR) == 0);
}
}
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -12,7 +12,6 @@
#include <sys/types.h>
#include <fcntl.h>
#include <unistd.h>
-#include <gsl/gsl_assert>
namespace Jacobian {
#define BUFFER_SIZE 600*1024
@@ -21,82 +20,82 @@ enum class Regularization {L1, L2};
class Layer {
public:
- Eigen::MatrixXf contents;
- Eigen::MatrixXf weights;
- Eigen::MatrixXf bias;
- Eigen::MatrixXf dZ;
- Eigen::MatrixXf v;
- Eigen::MatrixXf m;
- std::function<float(float)> activation;
- std::function<float(float)> activation_deriv;
+ Eigen::MatrixXf contents;
+ Eigen::MatrixXf weights;
+ Eigen::MatrixXf bias;
+ Eigen::MatrixXf dZ;
+ Eigen::MatrixXf v;
+ Eigen::MatrixXf m;
+ std::function<float(float)> activation;
+ std::function<float(float)> activation_deriv;
- Layer(int rows, int columns);
- Layer(float* vals, int rows, int columns);
- void operator=(const Layer& that);
- void init_weights(Layer next);
+ Layer(int rows, int columns);
+ Layer(float* vals, int rows, int columns);
+ void operator=(const Layer& that);
+ void init_weights(Layer next);
};
class Network {
- char buf[BUFFER_SIZE];
- char* p;
+ char buf[BUFFER_SIZE];
+ char* p;
protected:
- int instances;
- float epoch_acc;
- float epoch_cost;
- float val_acc;
- float val_cost;
- std::function<void(float&)> decay;
- std::function<void(std::vector<Eigen::MatrixXf>, int, int)> grad_calc;
- std::function<void(Layer&, Eigen::MatrixXf, float)> update;
- void next_batch(int fd);
+ int instances;
+ float epoch_acc;
+ float epoch_cost;
+ float val_acc;
+ float val_cost;
+ std::function<void(float&)> decay;
+ std::function<void(std::vector<Eigen::MatrixXf>, int, int)> grad_calc;
+ std::function<void(Layer&, Eigen::MatrixXf, float)> update;
+ void next_batch(int fd);
public:
- int data;
- int val_data;
- int val_instances;
- int test_instances;
- std::vector<Layer> layers;
- int length = 0;
- int batch_size;
- float learning_rate;
- float bias_lr;
- Regularization reg_type;
- float lambda;
- bool early_stop;
- float threshold;
- bool silenced = false;
- int epochs = 0;
- int batches = 0;
- Eigen::MatrixXf* labels;
+ int data;
+ int val_data;
+ int val_instances;
+ int test_instances;
+ std::vector<Layer> layers;
+ int length = 0;
+ int batch_size;
+ float learning_rate;
+ float bias_lr;
+ Regularization reg_type;
+ float lambda;
+ bool early_stop;
+ float threshold;
+ bool silenced = false;
+ int epochs = 0;
+ int batches = 0;
+ Eigen::MatrixXf* labels;
- Network(const char* path, int batch_sz, float learn_rate,
- float bias_rate, Regularization regularization,
- float l, float ratio, bool early_exit=true, float cutoff=0);
- ~Network();
- void add_layer(int nodes, std::function<float(float)> activation, std::function<float(float)> activation_deriv);
- void initialize();
- void init_optimizer(std::function<void(Layer &, Eigen::MatrixXf, float)> f)
- {
- update = f;
- };
- void init_decay(std::function<void(float&)> f);
- void set_activation(int index, std::function<float(float)> custom, std::function<float(float)> custom_deriv);
- void feedforward();
- void softmax();
- void list_net();
- void interactive_next_batch();
- float cost();
- float accuracy();
- Eigen::MatrixXf backpropagate();
- void validate(const char* path);
- void train();
- float get_acc() {return epoch_acc;}
- float get_val_acc() {return val_acc;}
- float get_cost() {return epoch_cost;}
- float get_val_cost()
- {
- return val_cost;
- }
+ Network(const char* path, int batch_sz, float learn_rate,
+ float bias_rate, Regularization regularization,
+ float l, float ratio, bool early_exit=true, float cutoff=0);
+ ~Network();
+ void add_layer(int nodes, std::function<float(float)> activation, std::function<float(float)> activation_deriv);
+ void initialize();
+ void init_optimizer(std::function<void(Layer &, Eigen::MatrixXf, float)> f)
+ {
+ update = f;
+ };
+ void init_decay(std::function<void(float&)> f);
+ void set_activation(int index, std::function<float(float)> custom, std::function<float(float)> custom_deriv);
+ void feedforward();
+ void softmax();
+ void list_net();
+ void interactive_next_batch();
+ float cost();
+ float accuracy();
+ Eigen::MatrixXf backpropagate();
+ void validate(const char* path);
+ void train();
+ float get_acc() {return epoch_acc;}
+ float get_val_acc() {return val_acc;}
+ float get_cost() {return epoch_cost;}
+ float get_val_cost()
+ {
+ return val_cost;
+ }
};
int prep_file(const char *path, const char *out_path);
@@ -110,14 +109,14 @@ Eigen::MatrixXf l1_deriv(Eigen::MatrixXf m);
#if (!RECKLESS)
#define checknan(x, loc) \
- if (x == INFINITY || x == NAN || x == -INFINITY) \
- throw ValueError("Detected NaN in operation", loc)
+ if (x == INFINITY || x == NAN || x == -INFINITY) \
+ throw ValueError("Detected NaN in operation", loc)
#define Expects(cond) assert(cond);
#define Ensures(cond) assert(cond);
#else
#define checknan(x, loc)
-#define Expects(cond) GSL_ASSUME(cond);
-#define Ensures(cond) GSL_ASSUME(cond);
+#define Expects(cond)
+#define Ensures(cond)
#endif
#define SHUFFLED_PATH "./shuffled.txt"
diff --git a/src/cnn.cpp b/src/cnn.cpp
@@ -1,10 +1,3 @@
-//
-// cnn.cpp
-// Jacobian
-//
-// Created by David Freifeld
-//
-
#include "bpnn.hpp"
#include "utils.hpp"
@@ -93,7 +86,8 @@ unsigned char* read_mnist_labels(std::string full_path, int number_of_labels) {
}
}
-ConvLayer::ConvLayer(int x, int y, int stride, int kern_x, int kern_y, int pad, std::function<float(float)> activ, std::function<float(float)> activ_deriv)
+ConvLayer::ConvLayer(int x, int y, int stride, int kern_x, int kern_y, int pad,
+ std::function<float(float)> activ, std::function<float(float)> activ_deriv)
:stride_len(stride), padding(pad), activation(activ), activation_deriv(activ_deriv)
{
pad*=2;
@@ -218,7 +212,8 @@ void ConvNet::process()
//pool_layers[preprocess_length-1].input = conv_layers[preprocess_length-1].output;
//pool_layers[preprocess_length-1].pool();
// std::cout << "Output:\n" << *pool_layers[preprocess_length-1].output << "\n\n";
- Eigen::Map<Eigen::RowVectorXf> flattened (conv_layers[preprocess_length-1].output->data(), conv_layers[preprocess_length-1].output->size());
+ Eigen::Map<Eigen::RowVectorXf> flattened (conv_layers[preprocess_length-1].output->data(),
+ conv_layers[preprocess_length-1].output->size());
// std::cout << "Flattened:\n" << flattened << "\n\n";
for (int i = 0; i < flattened.cols(); i++) {
(*layers[0].contents)(0, i) = flattened[i];
@@ -233,14 +228,39 @@ void ConvNet::set_label(Eigen::MatrixXf newlabels)
void ConvNet::list_net()
{
for (int i = 0; i < preprocess_length; i++) {
- std::cout << "-----------------------\nCONVOLUTIONAL LAYER " << i << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nStride: " << conv_layers[i].stride_len << "\nPadding: " << conv_layers[i].padding << "\n\n\u001b[31mINPUT:\x1B[0;37m\n" << *conv_layers[i].input << "\n\n\u001b[31mKERNEL:\x1B[0;37m\n" << *conv_layers[i].kernel << "\n\n\u001b[31mOUTPUT:\x1B[0;37m\n" << *conv_layers[i].output << "\n\n\u001b[31mBIAS:\x1B[0;37m\n" << conv_layers[i].bias << "\n\n\n";
- //std::cout << "-----------------------\nPOOLING LAYER " << i << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nStride: " << pool_layers[i].stride_len << "\nPadding: " << conv_layers[i].padding << "\n\n\u001b[31mINPUT:\x1B[0;37m\n" << *pool_layers[i].input << "\n\n\u001b[31mKERNEL:\x1B[0;37m\n-" << *pool_layers[i].kernel << "\n\n\u001b[31mOUTPUT:\x1B[0;37m\n" << *pool_layers[i].output << "\n\n\n";
+ std::cout << "-----------------------\nCONVOLUTIONAL LAYER " << i
+ << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nStride: "
+ << conv_layers[i].stride_len << "\nPadding: " << conv_layers[i].padding
+ << "\n\n\u001b[31mINPUT:\x1B[0;37m\n" << *conv_layers[i].input
+ << "\n\n\u001b[31mKERNEL:\x1B[0;37m\n" << *conv_layers[i].kernel
+ << "\n\n\u001b[31mOUTPUT:\x1B[0;37m\n" << *conv_layers[i].output
+ << "\n\n\u001b[31mBIAS:\x1B[0;37m\n" << conv_layers[i].bias << "\n\n\n";
+ // std::cout << "-----------------------\nPOOLING LAYER " << i
+ // << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nStride: "
+ // << pool_layers[i].stride_len << "\nPadding: " << conv_layers[i].padding
+ // << "\n\n\u001b[31mINPUT:\x1B[0;37m\n" << *pool_layers[i].input
+ // << "\n\n\u001b[31mKERNEL:\x1B[0;37m\n-" << *pool_layers[i].kernel
+ // << "\n\n\u001b[31mOUTPUT:\x1B[0;37m\n" << *pool_layers[i].output << "\n\n\n";
}
- std::cout << "-----------------------\nINPUT LAYER (LAYER 0)\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[0].activation_str << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[0].contents << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << *layers[0].weights << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[0].bias << "\n\n\n";
+ std::cout << "-----------------------\nINPUT LAYER (LAYER 0)\n-----------------------"
+ << "\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: "
+ << layers[0].activation_str << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n"
+ << *layers[0].contents << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n"
+ << *layers[0].weights << "\n\n\u001b[31mBIASES:\x1B[0;37m\n"
+ << *layers[0].bias << "\n\n\n";
for (int i = 1; i < length-1; i++) {
- std::cout << "-----------------------\nLAYER " << i << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[i].activation_str << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[i].contents << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[i].bias << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << *layers[i].weights << "\n\n\n";
+ std::cout << "-----------------------\nLAYER " << i
+ << "\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m"
+ << "\nActivation Function: " << layers[i].activation_str
+ << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[i].contents
+ << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[i].bias
+ << "\n\n\u001b[31mWEIGHTS:\x1B[0;37m\n" << *layers[i].weights << "\n\n\n";
}
- std::cout << "-----------------------\nOUTPUT LAYER (LAYER " << length-1 << ")\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m\nActivation Function: " << layers[length-1].activation_str <<"\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[length-1].contents << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[length-1].bias << "\n\n\n";
+ std::cout << "-----------------------\nOUTPUT LAYER (LAYER " << length-1
+ << ")\n-----------------------\n\n\u001b[31mGENERAL INFO:\x1B[0;37m"
+ << "\nActivation Function: " << layers[length-1].activation_str
+ << "\n\n\u001b[31mACTIVATIONS:\x1B[0;37m\n" << *layers[length-1].contents
+ << "\n\n\u001b[31mBIASES:\x1B[0;37m\n" << *layers[length-1].bias << "\n\n\n";
}
void ConvNet::backpropagate()
@@ -261,7 +281,9 @@ void ConvNet::backpropagate()
std::cout << conv_layers[layer].input->cols() << " " << gradients.back().cols() << "\n";
for (int i = 0; i < conv_layers[layer].input->rows() - gradients.back().rows() + 1; i++) {
for (int j = 0; j < conv_layers[layer].input->cols() - gradients.back().cols() + 1; j++) {
- conv_deltas[conv_deltas.size()-1](i, j) = (gradients.back() * conv_layers[layer].input->block(i, j, gradients.back().rows(), gradients.back().cols())).sum();
+ conv_deltas[conv_deltas.size()-1](i, j) =
+ (gradients.back() * conv_layers[layer].input->block(i, j, gradients.back().rows(),
+ gradients.back().cols())).sum();
}
}
*conv_layers[layer].kernel -= conv_deltas.back();
@@ -269,12 +291,16 @@ void ConvNet::backpropagate()
Eigen::MatrixXf flipped_kernel =
Eigen::MatrixXf::Zero(conv_layers[layer].kernel->rows(), conv_layers[layer].kernel->cols());
flipped_kernel = conv_layers[layer].kernel->transpose().colwise().reverse().transpose().colwise().reverse();
- Eigen::MatrixXf padded_grad = Eigen::MatrixXf::Zero(gradients.back().rows() + ((flipped_kernel.rows() - 1)*2), gradients.back().cols() + ((flipped_kernel.cols() - 1)*2));
- padded_grad.block(flipped_kernel.rows() - 1, flipped_kernel.cols() - 1, gradients.back().rows(), gradients.back().cols()) = gradients.back();
- Eigen::MatrixXf final_grad (padded_grad.rows() - flipped_kernel.rows() + 1, padded_grad.cols() - flipped_kernel.cols() + 1);
+ Eigen::MatrixXf padded_grad = Eigen::MatrixXf::Zero(gradients.back().rows() + ((flipped_kernel.rows() - 1)*2),
+ gradients.back().cols() + ((flipped_kernel.cols() - 1)*2));
+ padded_grad.block(flipped_kernel.rows() - 1, flipped_kernel.cols() - 1,
+ gradients.back().rows(), gradients.back().cols()) = gradients.back();
+ Eigen::MatrixXf final_grad (padded_grad.rows() - flipped_kernel.rows() + 1,
+ padded_grad.cols() - flipped_kernel.cols() + 1);
for (int i = 0; i < padded_grad.rows() - flipped_kernel.rows() + 1; i++) {
for (int j = 0; j < padded_grad.cols() - gradients.back().cols() + 1; j++) {
- final_grad(i, j) = (flipped_kernel * padded_grad.block(i, j, flipped_kernel.rows(), flipped_kernel.cols())).sum();
+ final_grad(i, j) = (flipped_kernel * padded_grad.block(i, j, flipped_kernel.rows(),
+ flipped_kernel.cols())).sum();
}
}
gradients.push_back(final_grad.cwiseProduct(*conv_layers[layer].dZ));
diff --git a/src/data.cpp b/src/data.cpp
@@ -1,111 +1,111 @@
typedef float val_t;
inline float scan(char **p)
{
- float n;
- int neg = 1;
- while (!isdigit(**p) && **p != '-' && **p != '.') ++*p;
- if (**p == '-') neg = -1, ++*p;
- for (n=0; isdigit(**p); ++*p) (n *= 10) += (**p-'0');
- if (*(*p)++ != '.') return n*neg;
- float d = 1;
- for (; isdigit(**p); ++*p) n += (d /= 10) * (**p-'0');
- return n*neg;
+ float n;
+ int neg = 1;
+ while (!isdigit(**p) && **p != '-' && **p != '.') ++*p;
+ if (**p == '-') neg = -1, ++*p;
+ for (n=0; isdigit(**p); ++*p) (n *= 10) += (**p-'0');
+ if (*(*p)++ != '.') return n*neg;
+ float d = 1;
+ for (; isdigit(**p); ++*p) n += (d /= 10) * (**p-'0');
+ return n*neg;
}
void prep(const char* rname, const char* wname)
{
- FILE* wptr = fopen(wname, "wb");
- FILE* rptr = fopen(rname, "rb");
- if(!wptr) throw std::runtime_error{"prep() could not write to the output file."};
- if(!rptr) throw std::runtime_error{"prep() could not read file for binary translation."};
- float tmp;
- char buf[BUFFER_SIZE+1];
- while(fgets(buf, BUFFER_SIZE+1, rptr)) {
- char* p = buf;
- for (int i=0; i<5; ++i) {
- tmp = scan(&p);
- fwrite(static_cast<void*>(&tmp), sizeof(float), 1, wptr);
- }
- }
- fclose(wptr);
- fclose(rptr);
+ FILE* wptr = fopen(wname, "wb");
+ FILE* rptr = fopen(rname, "rb");
+ if(!wptr) throw std::runtime_error{"prep() could not write to the output file."};
+ if(!rptr) throw std::runtime_error{"prep() could not read file for binary translation."};
+ float tmp;
+ char buf[BUFFER_SIZE+1];
+ while(fgets(buf, BUFFER_SIZE+1, rptr)) {
+ char* p = buf;
+ for (int i=0; i<5; ++i) {
+ tmp = scan(&p);
+ fwrite(static_cast<void*>(&tmp), sizeof(float), 1, wptr);
+ }
+ }
+ fclose(wptr);
+ fclose(rptr);
}
void Network::next_batch(int fd)
{
- Expects(fd > 0); // File descriptor must be valid.
- uintmax_t lines = 0;
- while(size_t bytes_read = read(fd, buf, BUFFER_SIZE)) {
- if (!bytes_read) break;
- p = buf;
- while(p < buf+BUFFER_SIZE) {
- if (lines >= 10) return;
- for (int i=0; i<layers[0].contents.cols(); ++i) {
- layers[0].contents(lines,i) = *(reinterpret_cast<float*>(p));
- p += sizeof(float);
- }
- (*labels)(lines,0) = *(reinterpret_cast<float*>(p));
- p += sizeof(float);
- ++lines;
- }
- }
- if (p < buf+BUFFER_SIZE) {
- while(p < buf+BUFFER_SIZE) {
- if (lines >= 10) return;
- for (int i=0; i<layers[0].contents.cols(); ++i) {
- layers[0].contents(lines,i) = *(reinterpret_cast<float*>(p));
- p += sizeof(float);
- }
- (*labels)(lines,0) = *(reinterpret_cast<float*>(p));
- p += sizeof(float);
- ++lines;
- }
- }
+ Expects(fd > 0); // File descriptor must be valid.
+ uintmax_t lines = 0;
+ while(size_t bytes_read = read(fd, buf, BUFFER_SIZE)) {
+ if (!bytes_read) break;
+ p = buf;
+ while(p < buf+BUFFER_SIZE) {
+ if (lines >= 10) return;
+ for (int i=0; i<layers[0].contents.cols(); ++i) {
+ layers[0].contents(lines,i) = *(reinterpret_cast<float*>(p));
+ p += sizeof(float);
+ }
+ (*labels)(lines,0) = *(reinterpret_cast<float*>(p));
+ p += sizeof(float);
+ ++lines;
+ }
+ }
+ if (p < buf+BUFFER_SIZE) {
+ while(p < buf+BUFFER_SIZE) {
+ if (lines >= 10) return;
+ for (int i=0; i<layers[0].contents.cols(); ++i) {
+ layers[0].contents(lines,i) = *(reinterpret_cast<float*>(p));
+ p += sizeof(float);
+ }
+ (*labels)(lines,0) = *(reinterpret_cast<float*>(p));
+ p += sizeof(float);
+ ++lines;
+ }
+ }
}
int prep_file(const char* path, const char* out_path)
{
- FILE* rptr = fopen(path, "r");
- if (!rptr) throw std::runtime_error{"prep_file() could not open file for shuffle/read."};
- char line[MAXLINE];
- std::vector<std::string> lines;
- int count = 0;
- while (fgets(line, MAXLINE, rptr) != NULL) {
- lines.emplace_back(line);
- count++;
- }
- lines[lines.size()-1] = lines[lines.size()-1] + "\n";
- std::random_device rd;
- std::mt19937 g(rd());
- std::shuffle(lines.begin(), lines.end(), g);
- fclose(rptr);
- FILE* wptr = fopen(out_path, "w");
- for (std::string & i : lines) {
- const char* cstr = i.c_str();
- fprintf(wptr,"%s", cstr);
- }
- fclose(wptr);
- return count;
+ FILE* rptr = fopen(path, "r");
+ if (!rptr) throw std::runtime_error{"prep_file() could not open file for shuffle/read."};
+ char line[MAXLINE];
+ std::vector<std::string> lines;
+ int count = 0;
+ while (fgets(line, MAXLINE, rptr) != NULL) {
+ lines.emplace_back(line);
+ count++;
+ }
+ lines[lines.size()-1] = lines[lines.size()-1] + "\n";
+ std::random_device rd;
+ std::mt19937 g(rd());
+ std::shuffle(lines.begin(), lines.end(), g);
+ fclose(rptr);
+ FILE* wptr = fopen(out_path, "w");
+ for (std::string & i : lines) {
+ const char* cstr = i.c_str();
+ fprintf(wptr,"%s", cstr);
+ }
+ fclose(wptr);
+ return count;
}
int split_file(const char* path, int lines, float ratio)
{
- FILE* src = fopen(path, "r");
- if (!src) throw std::runtime_error{"split_file() could not open file to split."};
- FILE* test = fopen(VAL_PATH, "w");
- FILE* train = fopen(TRAIN_PATH, "w");
- int switch_line = round(ratio * lines);
- char line[MAXLINE];
- int tests = 0;
- for (int i = 0; fgets(line, MAXLINE, src) != NULL; i++) {
- if (i > switch_line) {
- fprintf(test, "%s", line);
- tests++;
- }
- else fprintf(train, "%s", line);
- }
- fclose(src);
- fclose(test);
- fclose(train);
- return tests;
+ FILE* src = fopen(path, "r");
+ if (!src) throw std::runtime_error{"split_file() could not open file to split."};
+ FILE* test = fopen(VAL_PATH, "w");
+ FILE* train = fopen(TRAIN_PATH, "w");
+ int switch_line = round(ratio * lines);
+ char line[MAXLINE];
+ int tests = 0;
+ for (int i = 0; fgets(line, MAXLINE, src) != NULL; i++) {
+ if (i > switch_line) {
+ fprintf(test, "%s", line);
+ tests++;
+ }
+ else fprintf(train, "%s", line);
+ }
+ fclose(src);
+ fclose(test);
+ fclose(train);
+ return tests;
}
diff --git a/src/experimental/fileread.cpp b/src/experimental/fileread.cpp
@@ -1,9 +1,3 @@
-//
-// fileread.cpp
-// Jacobian
-//
-// Created by David Freifeld
-//
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
diff --git a/src/experimental/strassen.cpp b/src/experimental/strassen.cpp
@@ -1,15 +1,4 @@
-//
-// strassen.cpp
-// Jacobian
-//
-// Created by David Freifeld
-// Copyright © 2020 David Freifeld. All rights reserved.
-//
-// Description:
// Experimental benchmarking of Strassen's Algorithm vs plain Eigen matrix multiply.
-// While naive multiplication is O(n^3), Strassen's Algorithm for matrix multiply is O(n^2.8074)
-// which means significant differences will begin to manifest for large matrices. All faster algorithms are galactic.
-//
#include <Eigen/Dense>
#include <ctime>
@@ -34,20 +23,25 @@ Eigen::MatrixXf strassen_mul(Eigen::MatrixXf x, Eigen::MatrixXf y)
b.block(0,0,y.rows(), y.cols()) = y;
std::cout << "\nINIT\n" << a << "\n\n" << b << "\n\n\n";
std::cout << "\nEIGEN_VER\n" << a*b << "\n\n\n";
- //Eigen::MatrixXf a ()
+
int block_len = largest/2;
Eigen::MatrixXf result ((int)pow(2,power), (int)pow(2,power));
- Eigen::MatrixXf m1 = ((a.block(0,0, block_len, block_len)) + a.block(a.rows()-block_len,a.cols()-block_len, block_len, block_len)) * (b.block(0,0, block_len, block_len) + b.block(b.rows()-block_len,b.cols()-block_len, block_len, block_len));
- Eigen::MatrixXf m2 = (a.block(a.rows()-block_len, 0, block_len, block_len) + a.block(a.rows()-block_len,a.cols()-block_len, block_len, block_len)) * (b.block(0,0, block_len, block_len));
- Eigen::MatrixXf m3 = a.block(0,0, block_len, block_len) * (b.block(0,b.cols()-block_len, block_len, block_len) - b.block(b.rows()-block_len,b.cols()-block_len, block_len, block_len));
- Eigen::MatrixXf m4 = a.block(a.rows()-block_len,a.cols()-block_len, block_len, block_len) * (b.block(b.rows()-block_len,0, block_len, block_len) - b.block(0,0, block_len, block_len));
- Eigen::MatrixXf m5 = (a.block(0, 0, block_len, block_len) + a.block(0,a.cols()-block_len, block_len, block_len)) * (b.block(b.rows()-block_len,b.cols()-block_len, block_len, block_len));
- Eigen::MatrixXf m6 = (a.block(a.rows()-block_len,0, block_len, block_len) - a.block(0,0, block_len, block_len)) * (b.block(0,0, block_len, block_len) + b.block(0,b.cols()-block_len, block_len, block_len));
- Eigen::MatrixXf m7 = (a.block(0,a.cols()-block_len, block_len, block_len) - a.block(a.rows()-block_len,a.cols()-block_len, block_len, block_len)) * (b.block(a.rows()-block_len,0, block_len, block_len) + b.block(b.rows()-block_len,b.cols()-block_len, block_len, block_len));
+ Eigen::MatrixXf m1 = ((a.block(0,0, block_len, block_len)) + a.block(a.rows()-block_len,a.cols()-block_len, block_len, block_len)) *
+ (b.block(0,0, block_len, block_len) + b.block(b.rows()-block_len,b.cols()-block_len, block_len, block_len));
+ Eigen::MatrixXf m2 = (a.block(a.rows()-block_len, 0, block_len, block_len) + a.block(a.rows()-block_len,a.cols()-block_len, block_len, block_len)) *
+ (b.block(0,0, block_len, block_len));
+ Eigen::MatrixXf m3 = a.block(0,0, block_len, block_len) *
+ (b.block(0,b.cols()-block_len, block_len, block_len) - b.block(b.rows()-block_len,b.cols()-block_len, block_len, block_len));
+ Eigen::MatrixXf m4 = a.block(a.rows()-block_len,a.cols()-block_len, block_len, block_len) *
+ (b.block(b.rows()-block_len,0, block_len, block_len) - b.block(0,0, block_len, block_len));
+ Eigen::MatrixXf m5 = (a.block(0, 0, block_len, block_len) + a.block(0,a.cols()-block_len, block_len, block_len)) *
+ (b.block(b.rows()-block_len,b.cols()-block_len, block_len, block_len));
+ Eigen::MatrixXf m6 = (a.block(a.rows()-block_len,0, block_len, block_len) - a.block(0,0, block_len, block_len)) *
+ (b.block(0,0, block_len, block_len) + b.block(0,b.cols()-block_len, block_len, block_len));
+ Eigen::MatrixXf m7 = (a.block(0,a.cols()-block_len, block_len, block_len) - a.block(a.rows()-block_len,a.cols()-block_len, block_len, block_len)) *
+ (b.block(a.rows()-block_len,0, block_len, block_len) + b.block(b.rows()-block_len,b.cols()-block_len, block_len, block_len));
- // std::cout << m1 + m4 - m5 + m7 << "\n\n" << m3+m5 << "\n\n" << m2+m4 << "\n\n" << m1-m2+m3+m6;
-
result.block(0,0, block_len, block_len) = m1 + m4 - m5 + m7;
result.block(0,result.cols()-block_len, block_len, block_len) = m3 + m5;
result.block(result.rows()-block_len,0, block_len, block_len) = m2 + m4;
@@ -72,6 +66,7 @@ int main()
Eigen::MatrixXf sproduct = strassen_mul(a, b);
auto strassen_end = std::chrono::high_resolution_clock::now();
- /* std::cout << "EIGEN: " << std::chrono::duration_cast<std::chrono::nanoseconds>(eigen_end - eigen_begin).count() / pow(10,9) << " STRASSEN: " << std::chrono::duration_cast<std::chrono::nanoseconds>(strassen_end - strassen_begin).count() / pow(10,9) << "\n"; */
- /* std::cout << "A:\n" << a << "\nB:\n" << b << "\nEigen:\n" << product << "\nStrassen:\n" << sproduct << "\n"; */
+ std::cout << "EIGEN: " << std::chrono::duration_cast<std::chrono::nanoseconds>(eigen_end - eigen_begin).count() / pow(10,9)
+ << " STRASSEN: " << std::chrono::duration_cast<std::chrono::nanoseconds>(strassen_end - strassen_begin).count() / pow(10,9) << "\n";
+ std::cout << "A:\n" << a << "\nB:\n" << b << "\nEigen:\n" << product << "\nStrassen:\n" << sproduct << "\n";
}
diff --git a/src/pybind.cpp b/src/pybind.cpp
@@ -1,10 +1,3 @@
-//
-// pybind.cpp
-// Jacobian
-//
-// Created by David Freifeld
-//
-
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/functional.h>
@@ -17,79 +10,79 @@ namespace py = pybind11;
PYBIND11_MODULE(_jacobian, m)
{
- m.doc() = "Fast machine learning in C++"; // optional module docstring
- py::enum_<Regularization>(m, "Regularization")
- .value("L1", Regularization::L1)
- .value("L2", Regularization::L2)
- .export_values();
- py::class_<Layer>(m, "Layer")
- .def(py::init<int, int>())
- .def_readonly("contents", &Layer::contents)
- .def_readonly("weights", &Layer::weights)
- .def_readonly("bias", &Layer::bias)
- .def_readonly("v", &Layer::v)
- .def_readonly("m", &Layer::m)
- .def_readonly("dZ", &Layer::dZ)
- .def_readonly("activation", &Layer::activation)
- .def_readonly("activation_deriv", &Layer::activation);
- py::class_<Network>(m, "Network")
- .def(py::init<char *, int, float, float, Regularization, float,
- float, bool, float>(),
- py::arg("path"), py::arg("batch"), py::arg("learn_rate"),
- py::arg("bias_rate"), py::arg("regularization"),
- py::arg("lambda"), py::arg("ratio"),
- py::arg("early_exit") = true, py::arg("cutoff") = 0)
- .def("add_layer", &Network::add_layer, py::arg("nodes"),
- py::arg("activation"), py::arg("activation_deriv"))
- .def("initialize", &Network::initialize)
- .def("init_optimizer", &Network::init_optimizer, py::arg("optimizer"))
- .def("init_decay", &Network::init_decay, py::arg("decay"))
- .def("set_activation", &Network::set_activation,
- py::arg("index"), py::arg("custom"),
- py::arg("custom_deriv"))
- .def("feedforward", &Network::feedforward)
- .def("backpropagate", &Network::backpropagate)
- .def("list_net", &Network::list_net)
- .def("next_batch", &Network::interactive_next_batch)
- .def("cost", &Network::cost)
- .def("accuracy", &Network::accuracy)
- .def("train", &Network::train)
- .def("get_cost", &Network::get_cost)
- .def("get_acc", &Network::get_acc)
- .def("get_val_cost", &Network::get_val_cost)
- .def("get_val_acc", &Network::get_val_acc)
- .def_readonly("layers", &Network::layers);
- auto a = m.def_submodule("activations", "Submodule supplying built-in activation functions.");
- a.def("linear", &activations::linear, py::arg("x"));
- a.def("linear_deriv", &activations::linear_deriv, py::arg("x"));
- a.def("sigmoid", &activations::sigmoid, py::arg("x"));
- a.def("sigmoid_deriv", &activations::sigmoid_deriv, py::arg("x"));
- a.def("lecun_tanh", &activations::lecun_tanh, py::arg("x"));
- a.def("lecun_tanh_deriv", &activations::lecun_tanh_deriv, py::arg("x"));
- a.def("softplus", &activations::softplus, py::arg("x"));
- a.def("softplus_deriv", &activations::softplus_deriv, py::arg("x"));
- a.def("inverse_logit", &activations::inverse_logit, py::arg("x"));
- a.def("inverse_logit_deriv", &activations::inverse_logit_deriv, py::arg("x"));
- a.def("cloglog", &activations::cloglog, py::arg("x"));
- a.def("cloglog_deriv", &activations::cloglog_deriv, py::arg("x"));
- a.def("bipolar", &activations::bipolar, py::arg("x"));
- a.def("bipolar_deriv", &activations::bipolar_deriv, py::arg("x"));
- a.def("step", &activations::step, py::arg("x"));
- a.def("step_deriv", &activations::step_deriv, py::arg("x"));
- a.def("hard_tanh", &activations::hard_tanh, py::arg("x"));
- a.def("hard_tanh_deriv", &activations::hard_tanh_deriv, py::arg("x"));
- a.def("leaky_relu", &activations::leaky_relu, py::arg("x"));
- a.def("leaky_relu_deriv", &activations::leaky_relu_deriv, py::arg("x"));
- a.def("relu", (activations::rectifier(activations::linear)), py::arg("x"));
- a.def("relu_deriv", (activations::rectifier(activations::linear_deriv)), py::arg("x"));
- auto o = m.def_submodule("optimizers", "Submodule supplying built-in gradient descent optimizers.");
- o.def("momentum", &optimizers::momentum, py::arg("beta"));
- o.def("demon", &optimizers::demon, py::arg("beta"), py::arg("max_ep"));
- o.def("adam", &optimizers::adam, py::arg("beta1"), py::arg("beta2"), py::arg("epsilon"));
- o.def("adamax", &optimizers::adamax, py::arg("beta1"), py::arg("beta2"), py::arg("epsilon"));
- auto d = m.def_submodule("decays", "Submodule supplying built-in learning rate annealing functions.");
- d.def("step", &decays::step, py::arg("a_0"), py::arg("k"));
- d.def("exponential", &decays::exponential, py::arg("a_0"), py::arg("k"));
- d.def("fractional", &decays::fractional, py::arg("a_0"), py::arg("k"));
- d.def("linear", &decays::linear, py::arg("max_ep"));
+ m.doc() = "Fast machine learning in C++"; // optional module docstring
+ py::enum_<Regularization>(m, "Regularization")
+ .value("L1", Regularization::L1)
+ .value("L2", Regularization::L2)
+ .export_values();
+ py::class_<Layer>(m, "Layer")
+ .def(py::init<int, int>())
+ .def_readonly("contents", &Layer::contents)
+ .def_readonly("weights", &Layer::weights)
+ .def_readonly("bias", &Layer::bias)
+ .def_readonly("v", &Layer::v)
+ .def_readonly("m", &Layer::m)
+ .def_readonly("dZ", &Layer::dZ)
+ .def_readonly("activation", &Layer::activation)
+ .def_readonly("activation_deriv", &Layer::activation);
+ py::class_<Network>(m, "Network")
+ .def(py::init<char *, int, float, float, Regularization, float,
+ float, bool, float>(),
+ py::arg("path"), py::arg("batch"), py::arg("learn_rate"),
+ py::arg("bias_rate"), py::arg("regularization"),
+ py::arg("lambda"), py::arg("ratio"),
+ py::arg("early_exit") = true, py::arg("cutoff") = 0)
+ .def("add_layer", &Network::add_layer, py::arg("nodes"),
+ py::arg("activation"), py::arg("activation_deriv"))
+ .def("initialize", &Network::initialize)
+ .def("init_optimizer", &Network::init_optimizer, py::arg("optimizer"))
+ .def("init_decay", &Network::init_decay, py::arg("decay"))
+ .def("set_activation", &Network::set_activation,
+ py::arg("index"), py::arg("custom"),
+ py::arg("custom_deriv"))
+ .def("feedforward", &Network::feedforward)
+ .def("backpropagate", &Network::backpropagate)
+ .def("list_net", &Network::list_net)
+ .def("next_batch", &Network::interactive_next_batch)
+ .def("cost", &Network::cost)
+ .def("accuracy", &Network::accuracy)
+ .def("train", &Network::train)
+ .def("get_cost", &Network::get_cost)
+ .def("get_acc", &Network::get_acc)
+ .def("get_val_cost", &Network::get_val_cost)
+ .def("get_val_acc", &Network::get_val_acc)
+ .def_readonly("layers", &Network::layers);
+ auto a = m.def_submodule("activations", "Submodule supplying built-in activation functions.");
+ a.def("linear", &activations::linear, py::arg("x"));
+ a.def("linear_deriv", &activations::linear_deriv, py::arg("x"));
+ a.def("sigmoid", &activations::sigmoid, py::arg("x"));
+ a.def("sigmoid_deriv", &activations::sigmoid_deriv, py::arg("x"));
+ a.def("lecun_tanh", &activations::lecun_tanh, py::arg("x"));
+ a.def("lecun_tanh_deriv", &activations::lecun_tanh_deriv, py::arg("x"));
+ a.def("softplus", &activations::softplus, py::arg("x"));
+ a.def("softplus_deriv", &activations::softplus_deriv, py::arg("x"));
+ a.def("inverse_logit", &activations::inverse_logit, py::arg("x"));
+ a.def("inverse_logit_deriv", &activations::inverse_logit_deriv, py::arg("x"));
+ a.def("cloglog", &activations::cloglog, py::arg("x"));
+ a.def("cloglog_deriv", &activations::cloglog_deriv, py::arg("x"));
+ a.def("bipolar", &activations::bipolar, py::arg("x"));
+ a.def("bipolar_deriv", &activations::bipolar_deriv, py::arg("x"));
+ a.def("step", &activations::step, py::arg("x"));
+ a.def("step_deriv", &activations::step_deriv, py::arg("x"));
+ a.def("hard_tanh", &activations::hard_tanh, py::arg("x"));
+ a.def("hard_tanh_deriv", &activations::hard_tanh_deriv, py::arg("x"));
+ a.def("leaky_relu", &activations::leaky_relu, py::arg("x"));
+ a.def("leaky_relu_deriv", &activations::leaky_relu_deriv, py::arg("x"));
+ a.def("relu", (activations::rectifier(activations::linear)), py::arg("x"));
+ a.def("relu_deriv", (activations::rectifier(activations::linear_deriv)), py::arg("x"));
+ auto o = m.def_submodule("optimizers", "Submodule supplying built-in gradient descent optimizers.");
+ o.def("momentum", &optimizers::momentum, py::arg("beta"));
+ o.def("demon", &optimizers::demon, py::arg("beta"), py::arg("max_ep"));
+ o.def("adam", &optimizers::adam, py::arg("beta1"), py::arg("beta2"), py::arg("epsilon"));
+ o.def("adamax", &optimizers::adamax, py::arg("beta1"), py::arg("beta2"), py::arg("epsilon"));
+ auto d = m.def_submodule("decays", "Submodule supplying built-in learning rate annealing functions.");
+ d.def("step", &decays::step, py::arg("a_0"), py::arg("k"));
+ d.def("exponential", &decays::exponential, py::arg("a_0"), py::arg("k"));
+ d.def("fractional", &decays::fractional, py::arg("a_0"), py::arg("k"));
+ d.def("linear", &decays::linear, py::arg("max_ep"));
}
diff --git a/src/utils.cpp b/src/utils.cpp
@@ -1,11 +1,3 @@
-//
-// utils.cpp
-// Jacobian
-//
-// Created by David Freifeld
-// Copyright © 2020 David Freifeld. All rights reserved.
-//
-
#include <iostream>
#include <fstream>
#include <algorithm>
@@ -27,14 +19,14 @@ inline float sgn(float val) {return (0.0f < val) - (val < 0.0f);}
double fexp(double val)
{
- long tmp = static_cast<long>(1512775 * val + 1072632447) << 32;
- return *reinterpret_cast<double*>(&tmp);
+ long tmp = static_cast<long>(1512775 * val + 1072632447) << 32;
+ return *reinterpret_cast<double*>(&tmp);
}
float ftanh(float x)
{
- return (x*(10+pow(x,2))*(60+pow(x,2)))/
- (600+(270*pow(x,2))+(11*pow(x,4))+(pow(x,6)/24));
+ return (x*(10+pow(x,2))*(60+pow(x,2)))/
+ (600+(270*pow(x,2))+(11*pow(x,4))+(pow(x,6)/24));
}
//float ftanh(float val) {return sgn(val) * (1 - 2/(fexp(2*abs(val))+1));}
@@ -51,8 +43,8 @@ float linear(float x) {return x;}
float linear_deriv(float x) {return 1;}
float lecun_tanh(float x) {
- //std::cout << ftanh(x) << " vs " << tanh(x) << "\n";
- return 1.7159 * ftanh(0.66f * x);}
+ //std::cout << ftanh(x) << " vs " << tanh(x) << "\n";
+ return 1.7159 * ftanh(0.66f * x);}
float lecun_tanh_deriv(float x) {return 1.14393 * pow(1.0/fcosh(0.66f * x), 2);}
float inverse_logit(float x) {return (fexp(x)/(fexp(x)+1));}
@@ -66,16 +58,16 @@ float cloglog_deriv(float x) {return fexp(x-fexp(x));}
float step(float x)
{
- if (x > 0) return 1;
- else return 0;
+ if (x > 0) return 1;
+ else return 0;
}
float step_deriv(float x) {return 0;}
float bipolar(float x)
{
- if (x > 0) return 1;
- else if (x == 0) return 0;
- else return -1;
+ if (x > 0) return 1;
+ else if (x == 0) return 0;
+ else return -1;
}
float bipolar_deriv(float x) {return 0;}
@@ -85,77 +77,77 @@ float bipolar_sigmoid_deriv(float x) {return (2*fexp(x))/(pow(fexp(x)+1,2));}
float hard_tanh(float x) {return fmax(-1, fmin(1,x));}
float hard_tanh_deriv(float x)
{
- if (-1 < x && x < 1) return 1;
- else return 0;
+ if (-1 < x && x < 1) return 1;
+ else return 0;
}
float leaky_relu(float x)
{
- if (x > 0) return x;
- else return 0.01 * x;
+ if (x > 0) return x;
+ else return 0.01 * x;
}
float leaky_relu_deriv(float x)
{
- if (x > 0) return 1;
- else return 0.01;
+ if (x > 0) return 1;
+ else return 0.01;
}
std::function<float(float)> rectifier(float (*activation)(float))
{
- auto rectified = [activation](float x) -> float {
- if (x > 0)
- return (*activation)(x);
- else
- return 0;
- };
- return rectified;
+ auto rectified = [activation](float x) -> float {
+ if (x > 0)
+ return (*activation)(x);
+ else
+ return 0;
+ };
+ return rectified;
}
} // namespace activations
namespace optimizers {
std::function<void(Layer&, Eigen::MatrixXf, float)> momentum(float beta) {
- return [beta](Layer& layer, const Eigen::MatrixXf delta, const float learning_rate) {
- layer.weights -= (beta * layer.m) + (learning_rate * delta);
- layer.m = (learning_rate * delta);
- };
+ return [beta](Layer& layer, const Eigen::MatrixXf delta, const float learning_rate) {
+ layer.weights -= (beta * layer.m) + (learning_rate * delta);
+ layer.m = (learning_rate * delta);
+ };
}
std::function<void(Layer&, Eigen::MatrixXf, float)> demon(float beta, int max_ep) {
- float beta_init = beta;
- float prev_epoch = -1;
- float epochs = 0;
- return [max_ep, epochs, beta_init, beta](Layer& layer, const Eigen::MatrixXf delta, const float learning_rate) mutable {
- beta = beta_init * (1-(epochs/max_ep)) / ((beta_init * (1-(epochs/max_ep))) + (1-beta_init));
- layer.weights -= (beta * layer.m) + (learning_rate * delta);
- layer.m = (learning_rate * delta);
- epochs++;
- };
+ float beta_init = beta;
+ float prev_epoch = -1;
+ float epochs = 0;
+ return [max_ep, epochs, beta_init, beta](Layer& layer, const Eigen::MatrixXf delta, const float learning_rate) mutable {
+ beta = beta_init * (1-(epochs/max_ep)) / ((beta_init * (1-(epochs/max_ep))) + (1-beta_init));
+ layer.weights -= (beta * layer.m) + (learning_rate * delta);
+ layer.m = (learning_rate * delta);
+ epochs++;
+ };
}
std::function<void(Layer&, Eigen::MatrixXf, float)> adam(float beta1, float beta2, float epsilon) {
- return [beta1, beta2, epsilon](Layer& layer, const Eigen::MatrixXf delta, const float learning_rate) {
- layer.m = (beta1 * layer.m) + ((1-beta1)*delta);
- layer.v = (beta2 * layer.v) + (1-beta2)*(delta.cwiseProduct(delta));
- layer.weights -= learning_rate *
- ((layer.v.cwiseSqrt()).array()+epsilon).pow(-1).cwiseProduct(layer.m.array()).matrix();
- };
+ return [beta1, beta2, epsilon](Layer& layer, const Eigen::MatrixXf delta, const float learning_rate) {
+ layer.m = (beta1 * layer.m) + ((1-beta1)*delta);
+ layer.v = (beta2 * layer.v) + (1-beta2)*(delta.cwiseProduct(delta));
+ layer.weights -= learning_rate *
+ ((layer.v.cwiseSqrt()).array()+epsilon).pow(-1).cwiseProduct(layer.m.array()).matrix();
+ };
}
std::function<void(Layer&, Eigen::MatrixXf, float)> adamax(float beta1, float beta2, float epsilon) {
- return [beta1, beta2, epsilon](Layer &layer,
- const Eigen::MatrixXf delta,
- const float learning_rate) {
- layer.m = (beta1 * layer.m) + ((1 - beta1) * delta);
- if ((beta2 * layer.v).sum() > delta.array().abs().sum())
- layer.v = (beta2 * layer.v);
- else
- layer.v = delta.array().abs().matrix();
- layer.weights -=
- learning_rate *
- (layer.v.array().pow(-1).cwiseProduct(layer.m.array()))
- .matrix();
- };
+ return [beta1, beta2, epsilon](Layer &layer,
+ const Eigen::MatrixXf delta,
+ const float learning_rate) {
+ layer.m = (beta1 * layer.m) + ((1 - beta1) * delta);
+ if ((beta2 * layer.v).sum() > delta.array().abs().sum())
+ layer.v = (beta2 * layer.v);
+ else
+ layer.v = delta.array().abs().matrix();
+ layer.weights -=
+ learning_rate *
+ (layer.v.array().pow(-1).cwiseProduct(layer.m.array()))
+ .matrix();
+ };
}
}
}
diff --git a/src/utils.hpp b/src/utils.hpp
@@ -1,10 +1,3 @@
-//
-// utils.hpp
-// Jacobian
-//
-// Created by David Freifeld
-//
-
#ifndef UTILS_H
#define UTILS_H
