commit 6e347b335b4ffd2c958d9dc2f611daf36fc8b97d
parent e9d03fd360944cd588d5f1b0d6da5ae86e3fd32f
Author: David Freifeld <freifeld.david@gmail.com>
Date: Sun, 6 Sep 2020 21:13:06 -0700
Cleaning up ANN + more setup for RNN
Diffstat:
3 files changed, 19 insertions(+), 9 deletions(-)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -69,10 +69,9 @@ void Layer::init_weights(Layer next)
}
}
-Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, int regularization, float l, float ratio, bool early_exit, float cutoff)
+Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, Regularization regularization, float l, float ratio, bool early_exit, float cutoff)
:lambda(l), learning_rate(learn_rate), bias_lr(bias_rate), batch_size(batch_sz), reg_type(regularization), early_stop(early_exit), threshold(cutoff)
{
- assert(reg_type == 1 || reg_type == 2); // L1 and L2 are only relevant regularizations
int total_instances = prep_file(path, SHUFFLED_PATH);
val_instances = split_file(SHUFFLED_PATH, total_instances, ratio);
data = fopen(TRAIN_PATH, "r");
@@ -232,8 +231,8 @@ float Network::cost()
checknan(tempsum, "total summation inside cost calculation");
}
for (int i = 0; i < layers.size()-1; i++) {
- if (reg_type == 2) reg += cwise_product(*layers[i].weights,*layers[i].weights).sum();
- else if (reg_type == 1) reg += (layers[i].weights->array().abs().matrix()).sum();
+ if (reg_type == L2) reg += cwise_product(*layers[i].weights,*layers[i].weights).sum();
+ else if (reg_type == L1) reg += (layers[i].weights->array().abs().matrix()).sum();
}
return ((1.0/batch_size) * sum) + (1/2*lambda*reg);
}
@@ -294,8 +293,8 @@ void Network::backpropagate()
}
for (int i = 0; i < length-1; i++) {
update(deltas, i);
- if (reg_type == 2) *layers[length-2-i].weights -= ((lambda/batch_size) * (*layers[length-2-i].weights));
- else if (reg_type == 1) *layers[length-2-i].weights -= ((lambda/(2*batch_size)) * l1_deriv(*layers[length-2-i].weights));
+ if (reg_type == L2) *layers[length-2-i].weights -= ((lambda/batch_size) * (*layers[length-2-i].weights));
+ else if (reg_type == L1) *layers[length-2-i].weights -= ((lambda/(2*batch_size)) * l1_deriv(*layers[length-2-i].weights));
*layers[length-1-i].bias -= bias_lr * gradients[i];
if (strcmp(layers[length-2-i].activation_str, "prelu") == 0) {
float sum = 0;
@@ -341,6 +340,7 @@ float Network::validate(char* path)
float batch[datalen];
int label = -1;
for (int i = 0; i < batch_size; i++) {
+
fgets(line, MAXLINE, val_data);
char *p;
p = strtok(line,",");
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -15,6 +15,8 @@
#include <random>
#include <algorithm>
+enum Regularization {L1, L2};
+
class Layer {
public:
Eigen::MatrixXf* contents;
@@ -57,7 +59,7 @@ public:
float learning_rate;
float bias_lr;
float lambda;
- int reg_type;
+ Regularization reg_type;
int batch_size;
bool silenced = false;
@@ -69,7 +71,7 @@ public:
std::function<void(std::vector<Eigen::MatrixXf>, int, int)> grad_calc;
std::function<void(std::vector<Eigen::MatrixXf>, int)> update;
- Network(char* path, int batch_sz, float learn_rate, float bias_rate, int regularization, float l, float ratio, bool early_exit=true, float cutoff=0);
+ Network(char* path, int batch_sz, float learn_rate, float bias_rate, Regularization regularization, float l, float ratio, bool early_exit=true, float cutoff=0);
void add_layer(int nodes, char* name, std::function<float(float)> activation, std::function<float(float)> activation_deriv);
void add_prelu_layer(int nodes, float a);
void init_decay(char* type, ...);
diff --git a/src/rnn.cpp b/src/rnn.cpp
@@ -37,7 +37,15 @@ void init_weights(RecurrentLayer next)
class RNN : public Network {
public:
+ std::vector<RecurrentLayer> layers;
void feedforward();
void backpropagate();
- RNN();
+ RNN(char* path, int batch_sz, float learn_rate, float bias_rate, Regularization regularization, float l, float ratio, bool early_exit=true, float cutoff=0);
+
};
+
+RNN::RNN(char* path, int batch_sz, float learn_rate, float bias_rate, Regularization regularization, float l, float ratio, bool early_exit=true, float cutoff=0)
+ :Network(path, batch_sz, learn_rate, bias_rate, regularization, l, ratio, early_exit, cutoff)
+{}
+
+
