jacobian

commit 7028688481848a806470cdf949b01ab6a87bb8c2
parent 74c8a4630056abd6ee6df40935fe391bece52f4f
Author: David Freifeld <freifeld.david@gmail.com>
Date:   Mon, 21 Sep 2020 20:12:46 -0700

Decided file read is not fast enough

Diffstat:
M
src/experimental/fileread.cpp
 | 
101
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------

1 file changed, 74 insertions(+), 27 deletions(-)
diff --git a/src/experimental/fileread.cpp b/src/experimental/fileread.cpp
@@ -41,22 +41,48 @@ inline float scan(char **p)
     return n*neg;
 }
 
-int main () {
-    auto start = std::chrono::high_resolution_clock::now();
+float current()
+{
     static const auto BUFFER_SIZE = 16*1024;
     int fd = open("../../data_banknote_authentication.txt", O_RDONLY & O_NONBLOCK);
     if(fd == -1) {
         printf("fd == -1\n");
         return 1;
     }
-
-    /* Advise the kernel of our access pattern.  */
-    // posix_fadvise(fd, 0, 0, 1);  // FDADVICE_SEQUENTIAL
     
     char buf[BUFFER_SIZE + 1];
     uintmax_t lines = 0;
     float tmp;
-    int bufs = 0;
+    while(size_t bytes_read = read(fd, buf, BUFFER_SIZE))
+    {
+        if(bytes_read == (size_t)-1) {
+            printf("bytes_read == (size_t)-1\n");
+            return 1;
+        }
+        if (!bytes_read) break;
+        for(char *p = buf;;) {
+            char* bound = (char*) memchr(p, '\n', (buf + bytes_read) - p);
+            if (bound - p < 0) break; // Stop.
+            for (int i=0; i<5; ++i) tmp = scan(&p);
+            p = bound + 1;
+            ++lines;
+        }
+    }
+    return tmp;
+}
+
+void prep()
+{
+    FILE* wptr = fopen("../../data_banknote_authentication.bin", "wb");
+    static const auto BUFFER_SIZE = 16*1024;
+    int fd = open("../../data_banknote_authentication.txt", O_RDONLY & O_NONBLOCK);
+    if(fd == -1) {
+        printf("fd == -1\n");
+        return 1;
+    }
+    
+    char buf[BUFFER_SIZE + 1];
+    uintmax_t lines = 0;
     while(size_t bytes_read = read(fd, buf, BUFFER_SIZE))
     {
         if(bytes_read == (size_t)-1) {
@@ -68,37 +94,58 @@ int main () {
             char* bound = (char*) memchr(p, '\n', (buf + bytes_read) - p);
             if (bound - p < 0) break; // Stop.
             for (int i=0; i<5; ++i) {
-                tmp = scan(&p);
+                fwrite((void*)&scan(&p), sizeof(float), 1, wptr);
             }
             p = bound + 1;
             ++lines;
         }
-        bufs++;
     }
-    printf("Final %f, %i bufs\n", tmp, bufs);
+    return tmp;
+}
+
+float newer()
+{
+    static const auto BUFFER_SIZE = 16*1024;
+    int fd = open("../../data_banknote_authentication.txt", O_RDONLY & O_NONBLOCK);
+    if(fd == -1) {
+        printf("fd == -1\n");
+        return 1;
+    }
+    
+    char buf[BUFFER_SIZE + 1];
+    uintmax_t lines = 0;
+    float tmp;
+    while(size_t bytes_read = read(fd, buf, BUFFER_SIZE))
+    {
+        if(bytes_read == (size_t)-1) {
+            printf("bytes_read == (size_t)-1\n");
+            return 1;
+        }
+        if (!bytes_read) break;
+        for(char *p = buf;;) {
+            char* bound = (char*) memchr(p, '\n', (buf + bytes_read) - p);
+            if (bound - p < 0) break; // Stop.
+            for (int i=0; i<5; ++i) tmp = scan(&p);
+            p = bound + 1;
+            ++lines;
+        }
+    }
+    return tmp;
+}
+
+int main () {
+    auto start = std::chrono::high_resolution_clock::now();
+    float tmp = newer();
     auto end = std::chrono::high_resolution_clock::now();
     double time = std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count() / pow(10,9);
     
     auto f_start = std::chrono::high_resolution_clock::now();
-    int flines = 0;
-    FILE* fptr = fopen("../../data_banknote_authentication.txt", "r");
-    char fbuf[1024];
-    float ftmp;
-    while (fgets(fbuf, 1024, fptr) != NULL) {
-        char *p;
-        p = strtok(fbuf,",");
-        for (int j = 0; j < 4; j++) {
-            ftmp = strtod(p, NULL);
-            p = strtok(NULL,",");
-        }
-        ftmp = strtod(p, NULL);
-        flines++;
-    }
+    float ftmp = current();
     auto f_end = std::chrono::high_resolution_clock::now();
-    if (flines == lines) std::cout << "Linecount is valid!" << "\n";
-    else std::cout << "WARN: INVALID LINECOUNT " << lines << " vs. " << flines << "\n";
-    if (ftmp == tmp) std::cout << "Final value read is valid!" << "\n";
-    else std::cout << "WARN: INVALID LAST READ VAL " << tmp << " vs. " << ftmp << "\n";
+    // if (flines == lines) std::cout << "Linecount is valid!" << "\n";
+    // else std::cout << "WARN: INVALID LINECOUNT " << lines << " vs. " << flines << "\n";
+    // if (ftmp == tmp) std::cout << "Final value read is valid!" << "\n";
+    // else std::cout << "WARN: INVALID LAST READ VAL " << tmp << " vs. " << ftmp << "\n";
     double ftime = std::chrono::duration_cast<std::chrono::nanoseconds>(f_end - f_start).count() / pow(10,9);
     std::cout << ftime << " " << time << " so " << ftime/time * 100 << "% speedup\n";
 }