jacobian

commit bb02aa54ba29c6f3687759dab6593dcf9e909999
parent df40d575343e8f46956b57f319ad16981a37e785
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Sat, 10 Oct 2020 10:55:45 -0700

Ludicrous file read speed achieved

Diffstat:
M
src/experimental/fileread.cpp
 | 
71
++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------

1 file changed, 52 insertions(+), 19 deletions(-)
diff --git a/src/experimental/fileread.cpp b/src/experimental/fileread.cpp
@@ -18,14 +18,8 @@
 #include <stdint.h>
 #include <sys/mman.h>
 #include <sys/uio.h>
-
-// No safety checks whatsoever. Use at your own risk.
-// Assumes solely numeric input. No NaN, no inf.
-// float strtof_but_bad(char* p)
-// {
-//     float flt;
-//     for (int i = 0; 
-// }
+#include <thread>
+#include <lz4.h>
 
 // Courtesy of github.com/exr0n
 typedef float val_t;
@@ -100,6 +94,27 @@ void prep()
     }
 }
 
+void compress()
+{
+    FILE* wptr = fopen("../../data_banknote_authentication.lz4", "wb");
+    int BUFFER_SIZE = 512*1024;
+    int fd = open("../../data_banknote_authentication.big.bin", O_RDONLY | O_NONBLOCK);
+    fcntl(fd, F_RDADVISE);
+    char buf[BUFFER_SIZE];
+    char wbuf[BUFFER_SIZE];
+    while(size_t bytes_read = read(fd, buf, BUFFER_SIZE))
+    {
+        if(bytes_read == (size_t)-1) {
+            printf("bytes_read == (size_t)-1\n");
+            exit(1);
+        }
+        if (!bytes_read) break;
+        size_t len = LZ4_compress_default(buf, wbuf, BUFFER_SIZE, BUFFER_SIZE);
+        fwrite((void*)wbuf, len, 1, wptr); 
+    }
+    close(fd);
+}
+
 float mmap_read()
 {
     static const auto BUFFER_SIZE = 600*1024;
@@ -151,10 +166,11 @@ float vec_read()
     return 0;
 }
 
-float std_read()
+
+float std_read(char* path)
 {
     int BUFFER_SIZE = 512*1024;
-    int fd = open("../../data_banknote_authentication.big.bin", O_RDONLY | O_NONBLOCK);
+    int fd = open(path, O_RDONLY | O_NONBLOCK);
     fcntl(fd, F_RDADVISE);
     char buf[BUFFER_SIZE];
     float tmp;
@@ -164,17 +180,24 @@ float std_read()
             printf("bytes_read == (size_t)-1\n");
             exit(1);
         }
-        if (!bytes_read) break;
+        // if (!bytes_read) break;
+        // for (char* p = buf; p<buf+BUFFER_SIZE;) {
+        //     for (int i = 0; i < 5; ++i) {
+        //         tmp = *(float*)p;
+        //         p += sizeof(float);
+        //     }
+        // }
     }
     return tmp;
 }
 
-float byte_read()
+float decomp_read(char* path)
 {
     int BUFFER_SIZE = 512*1024;
-    int fd = open("../../data_banknote_authentication.big.bin", O_RDONLY | O_NONBLOCK);
+    int fd = open(path, O_RDONLY | O_NONBLOCK);
     fcntl(fd, F_RDADVISE);
-    std::byte buf[BUFFER_SIZE];
+    char buf[BUFFER_SIZE];
+    char rbuf[BUFFER_SIZE*10];
     float tmp;
     while(size_t bytes_read = read(fd, buf, BUFFER_SIZE))
     {
@@ -183,6 +206,14 @@ float byte_read()
             exit(1);
         }
         if (!bytes_read) break;
+        size_t len = LZ4_decompress_safe(buf, rbuf, BUFFER_SIZE, BUFFER_SIZE*10);
+        // for (char* p = rbuf; p<buf+len;) {
+        //     for (int i = 0; i < 5; ++i) {
+        //         tmp = *(float*)p;
+        //         printf("%f\n", tmp);
+        //         p += sizeof(float);
+        //     }
+        // }
     }
     return tmp;
 }
@@ -192,23 +223,25 @@ int main () {
     float tmp;
     auto start = std::chrono::high_resolution_clock::now();
     //prep();
+    // compress();
     auto prep_end = std::chrono::high_resolution_clock::now();
     for (int i = 0; i < epochs; i++) {
-        tmp = std_read();
+        tmp = decomp_read("../../data_banknote_authentication.lz4");
     }
     auto end = std::chrono::high_resolution_clock::now();
-    double time = std::chrono::duration_cast<std::chrono::nanoseconds>(end - prep_end + prep_end-start).count() / pow(10,9);
+    double time = std::chrono::duration_cast<std::chrono::nanoseconds>(end - prep_end + prep_end-start).count() / pow(10,9) / epochs;
     double runtime = std::chrono::duration_cast<std::chrono::nanoseconds>(end - prep_end).count() / pow(10,9);
     float ftmp;
     auto f_start = std::chrono::high_resolution_clock::now();
     for (int i = 0; i < epochs; i++) {
-        ftmp = byte_read();
+        ftmp = std_read("../../data_banknote_authentication.big.bin");
     }
     auto f_end = std::chrono::high_resolution_clock::now();
     // if (flines == lines) std::cout << "Linecount is valid!" << "\n";
     // else std::cout << "WARN: INVALID LINECOUNT " << lines << " vs. " << flines << "\n";
     if (ftmp == tmp) std::cout << "Final value read is valid!" << "\n";
     else std::cout << "WARN: INVALID LAST READ VAL " << tmp << " vs. " << ftmp << "\n";
-    double ftime = std::chrono::duration_cast<std::chrono::nanoseconds>(f_end - f_start).count() / pow(10,9);
-    std::cout << ftime << " " << time << " so " << 9/ftime << " GB/s vs.  " << 9/time << " GB/s " << runtime << " (" << ftime/time * 100 << "% speedup overall, " << ftime/runtime * 100 << "% speedup runtime)\n";
+    double ftime = std::chrono::duration_cast<std::chrono::nanoseconds>(f_end - f_start).count() / pow(10,9) / epochs;
+    std::cout << ftime << " " << time << ". 15.25 GB input parsed so " << 15.25/ftime << " GB/s (really " << 9.02/ftime << " GB/s) vs. " << 15.25/time << " GB/s (really " << 1.04/time << " GB/s)\n";
+    std::cout << runtime << " (" << ftime/time * 100 << "% speedup overall, " << ftime/runtime * 100 << "% speedup runtime)\n";
 }