jacobian

commit 9590b0665d1cf578d2f1a81278f8acccccb528a0
parent c729908839f7c62f782633c37f24435740d65f3e
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sun, 11 Oct 2020 12:46:45 -0700

Compression better off experimental for now

Diffstat:
M
src/bpnn.cpp
 | 
7
++-----
M
src/bpnn.hpp
 | 
3
+--
M
src/data.cpp
 | 
31
++++---------------------------

3 files changed, 7 insertions(+), 34 deletions(-)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -71,7 +71,6 @@ void Layer::init_weights(Layer next)
         (*v)(static_cast<int>(i / nodes), i%nodes) = 0;
         (*m)(static_cast<int>(i / nodes), i%nodes) = 0;
     }
-    
 }
 
 Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, Regularization regularization, float l, float ratio, bool early_exit, float cutoff)
@@ -83,10 +82,8 @@ Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, Re
     val_instances = split_file(SHUFFLED_PATH, total_instances, ratio);
     prep(TRAIN_PATH, TRAIN_BIN_PATH);
     prep(VAL_PATH, VAL_BIN_PATH);
-    compress(TRAIN_BIN_PATH, TRAIN_LZ4_PATH);
-    compress(VAL_BIN_PATH, VAL_LZ4_PATH);
-    data = open(TRAIN_LZ4_PATH, O_RDONLY | O_NONBLOCK);
-    val_data = open(VAL_LZ4_PATH, O_RDONLY | O_NONBLOCK);
+    data = open(TRAIN_BIN_PATH, O_RDONLY | O_NONBLOCK);
+    val_data = open(VAL_BIN_PATH, O_RDONLY | O_NONBLOCK);
     instances = total_instances - val_instances;
     decay = [this]() -> void {};
     update = [this](std::vector<Eigen::MatrixXf> deltas, int i) {
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -48,9 +48,8 @@ public:
     int data;
     int val_data;
     int instances;
-    char buf[LARGE_BUF];
+    char buf[BUFFER_SIZE];
     char* p;
-    int read_len;
     int val_instances;
     int test_instances;
     Eigen::MatrixXf numerical_grad(int i, float epsilon);
diff --git a/src/data.cpp b/src/data.cpp
@@ -37,37 +37,14 @@ void prep(char* rname, char* wname)
     fclose(wptr);
 }
 
-void compress(char* rname, char* wname)
-{
-    FILE* wptr = fopen(wname, "wb");
-    int fd = open(rname, O_RDONLY | O_NONBLOCK);
-    fcntl(fd, F_RDADVISE);
-    char buf[BUFFER_SIZE];
-    char wbuf[BUFFER_SIZE];
-    while(size_t bytes_read = read(fd, buf, BUFFER_SIZE))
-    {
-        if(bytes_read == (size_t)-1) {
-            printf("bytes_read == (size_t)-1\n");
-            exit(1);
-        }
-        if (!bytes_read) break;
-        size_t len = LZ4_compress_default(buf, wbuf, BUFFER_SIZE, BUFFER_SIZE);
-        fwrite((void*)wbuf, len, 1, wptr); 
-    }
-    close(fd);
-    fclose(wptr);
-}
-
 int Network::next_batch(int fd)
 {
     Expects(fd > 0); // File descriptor must be valid.
     uintmax_t lines = 0;
-    char rbuf[BUFFER_SIZE];
-    while(size_t bytes_read = read(fd, rbuf, BUFFER_SIZE)) {        
+    while(size_t bytes_read = read(fd, buf, BUFFER_SIZE)) {        
         if (!bytes_read) break;
-        read_len = LZ4_decompress_safe(rbuf, buf, BUFFER_SIZE, LARGE_BUF);
         p = buf;
-        while(p < buf+read_len) {
+        while(p < buf+BUFFER_SIZE) {
             if (lines >= 10) return 0;
             for (int i=0; i<layers[0].contents->cols(); ++i) {
                 (*layers[0].contents)(lines,i) = *(reinterpret_cast<float*>(p));
@@ -78,8 +55,8 @@ int Network::next_batch(int fd)
             ++lines;
         }
     }
-    if (p < buf+read_len) {
-        while(p < buf+read_len) {
+    if (p < buf+BUFFER_SIZE) {
+        while(p < buf+BUFFER_SIZE) {
             if (lines >= 10) return 0;
             for (int i=0; i<layers[0].contents->cols(); ++i) {
                 (*layers[0].contents)(lines,i) = *(reinterpret_cast<float*>(p));