jacobian

commit c729908839f7c62f782633c37f24435740d65f3e
parent 7b168b0f7f4853961d3b982cc43a396223b6f865
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sun, 11 Oct 2020 12:15:54 -0700

Segfault on function call

Diffstat:
M
CMakeLists.txt
 | 
2
+-
M
src/bpnn.cpp
 | 
8
++++++--
M
src/bpnn.hpp
 | 
9
+++++++--
M
src/data.cpp
 | 
35
+++++++++++++++++++++++++++++------

4 files changed, 43 insertions(+), 11 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -26,7 +26,7 @@ if (AVX)
   set(COMPILE_FLAGS "${COMPILE_FLAGS} -mavx -D AVX")
 endif()
 
-set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} ${COMPILE_FLAGS} -w")
+set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} ${COMPILE_FLAGS} -w -llz4")
 
 # if (PYTHON)
 #   project(jacobian)
diff --git a/src/bpnn.cpp b/src/bpnn.cpp
@@ -17,6 +17,8 @@
 #define TRAIN_PATH "./train.txt"
 #define VAL_BIN_PATH "./test.bin"
 #define TRAIN_BIN_PATH "./train.bin"
+#define VAL_LZ4_PATH "./test.lz4"
+#define TRAIN_LZ4_PATH "./train.lz4"
 
 #if (AVX)
 #define cwise_product(a,b) avx_product(a, b)
@@ -81,8 +83,10 @@ Network::Network(char* path, int batch_sz, float learn_rate, float bias_rate, Re
     val_instances = split_file(SHUFFLED_PATH, total_instances, ratio);
     prep(TRAIN_PATH, TRAIN_BIN_PATH);
     prep(VAL_PATH, VAL_BIN_PATH);
-    data = open(TRAIN_BIN_PATH, O_RDONLY | O_NONBLOCK);
-    val_data = open(VAL_BIN_PATH, O_RDONLY | O_NONBLOCK);
+    compress(TRAIN_BIN_PATH, TRAIN_LZ4_PATH);
+    compress(VAL_BIN_PATH, VAL_LZ4_PATH);
+    data = open(TRAIN_LZ4_PATH, O_RDONLY | O_NONBLOCK);
+    val_data = open(VAL_LZ4_PATH, O_RDONLY | O_NONBLOCK);
     instances = total_instances - val_instances;
     decay = [this]() -> void {};
     update = [this](std::vector<Eigen::MatrixXf> deltas, int i) {
diff --git a/src/bpnn.hpp b/src/bpnn.hpp
@@ -17,8 +17,10 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <gsl/gsl_assert>
+#include <lz4.h>
 
 #define BUFFER_SIZE 600*1024
+#define LARGE_BUF 600*1024*15
 enum Regularization {L1, L2};
 
 class Layer {
@@ -46,8 +48,9 @@ public:
     int data;
     int val_data;
     int instances;
-    std::byte buf[BUFFER_SIZE];
-    std::byte* p;
+    char buf[LARGE_BUF];
+    char* p;
+    int read_len;
     int val_instances;
     int test_instances;
     Eigen::MatrixXf numerical_grad(int i, float epsilon);
@@ -127,6 +130,8 @@ struct ValueError : public std::exception
     }
 };
 void prep(char* rname, char* wname);
+void compress(char* rname, char* wname);
+
 #define MAXLINE 1024
 
 #if (!RECKLESS)
diff --git a/src/data.cpp b/src/data.cpp
@@ -37,16 +37,39 @@ void prep(char* rname, char* wname)
     fclose(wptr);
 }
 
+void compress(char* rname, char* wname)
+{
+    FILE* wptr = fopen(wname, "wb");
+    int fd = open(rname, O_RDONLY | O_NONBLOCK);
+    fcntl(fd, F_RDADVISE);
+    char buf[BUFFER_SIZE];
+    char wbuf[BUFFER_SIZE];
+    while(size_t bytes_read = read(fd, buf, BUFFER_SIZE))
+    {
+        if(bytes_read == (size_t)-1) {
+            printf("bytes_read == (size_t)-1\n");
+            exit(1);
+        }
+        if (!bytes_read) break;
+        size_t len = LZ4_compress_default(buf, wbuf, BUFFER_SIZE, BUFFER_SIZE);
+        fwrite((void*)wbuf, len, 1, wptr); 
+    }
+    close(fd);
+    fclose(wptr);
+}
+
 int Network::next_batch(int fd)
 {
     Expects(fd > 0); // File descriptor must be valid.
     uintmax_t lines = 0;
-    while(size_t bytes_read = read(fd, buf, BUFFER_SIZE)) {
-        p = buf;
+    char rbuf[BUFFER_SIZE];
+    while(size_t bytes_read = read(fd, rbuf, BUFFER_SIZE)) {        
         if (!bytes_read) break;
-        while(p < buf+BUFFER_SIZE) {
+        read_len = LZ4_decompress_safe(rbuf, buf, BUFFER_SIZE, LARGE_BUF);
+        p = buf;
+        while(p < buf+read_len) {
             if (lines >= 10) return 0;
-            for (int i=0; i<layers[0].contents->cols(); ++i) {b
+            for (int i=0; i<layers[0].contents->cols(); ++i) {
                 (*layers[0].contents)(lines,i) = *(reinterpret_cast<float*>(p));
                 p += sizeof(float);
             }
@@ -55,8 +78,8 @@ int Network::next_batch(int fd)
             ++lines;
         }
     }
-    if (p < buf+BUFFER_SIZE) {
-        while(p < buf+BUFFER_SIZE) {
+    if (p < buf+read_len) {
+        while(p < buf+read_len) {
             if (lines >= 10) return 0;
             for (int i=0; i<layers[0].contents->cols(); ++i) {
                 (*layers[0].contents)(lines,i) = *(reinterpret_cast<float*>(p));