fileread.cpp (7620B)
1 #include <stdlib.h> 2 #include <stdio.h> 3 #include <sys/types.h> 4 #include <sys/stat.h> 5 #include <unistd.h> 6 #include <string.h> 7 #include <fcntl.h> 8 #include <ctime> 9 #include <chrono> 10 #include <iostream> 11 #include <cmath> 12 #include <stdint.h> 13 #include <sys/mman.h> 14 #include <sys/uio.h> 15 #include <thread> 16 #include <lz4.h> 17 18 // Courtesy of github.com/exr0n 19 typedef float val_t; 20 inline float scan(char **p) 21 { 22 float n; 23 int neg = 1; 24 while (!isdigit(**p) && **p != '-' && **p != '.') ++*p; 25 if (**p == '-') neg = -1, ++*p; 26 for (n=0; isdigit(**p); ++*p) (n *= 10) += (**p-'0'); 27 if (*(*p)++ != '.') return n*neg; 28 float d = 1; 29 for (; isdigit(**p); ++*p) n += (d /= 10) * (**p-'0'); 30 return n*neg; 31 } 32 33 float current() 34 { 35 static const auto BUFFER_SIZE = 16*1024; 36 int fd = open("../../data_banknote_authentication.txt", O_RDONLY & O_NONBLOCK); 37 if(fd == -1) { 38 printf("fd == -1\n"); 39 return 1; 40 } 41 42 char buf[BUFFER_SIZE + 1]; 43 uintmax_t lines = 0; 44 float tmp; 45 while(size_t bytes_read = read(fd, buf, BUFFER_SIZE)) 46 { 47 if(bytes_read == (size_t)-1) { 48 printf("bytes_read == (size_t)-1\n"); 49 return 1; 50 } 51 if (!bytes_read) break; 52 for(char *p = buf;;) { 53 char* bound = (char*) memchr(p, '\n', (buf + bytes_read) - p); 54 if (bound - p < 0) break; // Stop. 55 for (int i=0; i<5; ++i) tmp = scan(&p); 56 p = bound + 1; 57 ++lines; 58 } 59 } 60 return tmp; 61 } 62 63 void prep() 64 { 65 FILE* wptr = fopen("../../data_banknote_authentication.bin", "wb"); 66 static const auto BUFFER_SIZE = 512*1024; 67 int fd = open("../../data_banknote_authentication.txt", O_RDONLY | O_NONBLOCK); 68 if(fd == -1) { 69 printf("fd == -1\n"); 70 } 71 float tmp; 72 char buf[BUFFER_SIZE + 1]; 73 while(size_t bytes_read = read(fd, buf, BUFFER_SIZE)) 74 { 75 if(bytes_read == (size_t)-1) { 76 printf("bytes_read == (size_t)-1\n"); 77 } 78 if (!bytes_read) break; 79 for(char *p = buf;;) { 80 char* bound = (char*) memchr(p, '\n', (buf + bytes_read) - p); 81 if (bound - p < 0) break; // Stop. 82 for (int i=0; i<5; ++i) { 83 tmp = scan(&p); 84 fwrite((void*)&tmp, sizeof(float), 1, wptr); 85 } 86 p = bound + 1; 87 } 88 } 89 } 90 91 void compress() 92 { 93 FILE* wptr = fopen("../../data_banknote_authentication.lz4", "wb"); 94 int BUFFER_SIZE = 500*1024; 95 int fd = open("../../data_banknote_authentication.big.bin", O_RDONLY | O_NONBLOCK); 96 fcntl(fd, F_RDADVISE); 97 char buf[BUFFER_SIZE]; 98 char wbuf[BUFFER_SIZE]; 99 while(size_t bytes_read = read(fd, buf, BUFFER_SIZE)) 100 { 101 if(bytes_read == (size_t)-1) { 102 printf("bytes_read == (size_t)-1\n"); 103 exit(1); 104 } 105 if (!bytes_read) break; 106 int len = LZ4_compress_default(buf, wbuf, BUFFER_SIZE, BUFFER_SIZE); 107 fwrite((void*)wbuf, (size_t)len, 1, wptr); 108 } 109 close(fd); 110 } 111 112 float mmap_read() 113 { 114 static const auto BUFFER_SIZE = 600*1024; 115 int fd = open("../../data_banknote_authentication.big.bin", O_RDONLY | O_NONBLOCK); 116 if(fd == -1) { 117 printf("Cannot open file."); 118 exit(1); 119 } 120 int rc, ii; 121 struct stat st; 122 size_t size; 123 rc = fstat(fd, &st); 124 size=st.st_size; 125 float* ptr = (float*) mmap(0, size, PROT_READ, MAP_SHARED, fd, 0); 126 madvise(ptr, size, POSIX_MADV_SEQUENTIAL); 127 char buf[BUFFER_SIZE]; 128 uintmax_t lines = 0; 129 float tmp; 130 for (ii=0; ii < size/sizeof *ptr; ii++) { 131 // Nothin. 132 } 133 rc = munmap(ptr, size); 134 close(fd); 135 return tmp; 136 } 137 138 float vec_read() 139 { 140 #define CHUNK_SZ (200*1024) 141 #define BUFFER_SZ (600*1024) 142 #define NUM_CHUNKS (BUFFER_SZ/CHUNK_SZ) 143 int fd = open("../../data_banknote_authentication.big.bin", O_RDONLY | O_NONBLOCK); 144 char rawbuf[BUFFER_SZ]; 145 char* buf = (char*) rawbuf; 146 iovec iovecs[NUM_CHUNKS]; 147 for (int i = 0; i < BUFFER_SZ; i+=CHUNK_SZ) { 148 iovecs[i/CHUNK_SZ].iov_base = buf + i; 149 iovecs[i/CHUNK_SZ].iov_len = CHUNK_SZ; 150 } 151 while(size_t bytes_read = readv(fd, iovecs, NUM_CHUNKS)) 152 { 153 if(bytes_read == (size_t)-1) { 154 printf("\n%zu\n", bytes_read); 155 printf("bytes_read == (size_t)-1\n"); 156 exit(1); 157 } 158 if (!bytes_read) break; 159 } 160 return 0; 161 } 162 163 164 float std_read(char* path) 165 { 166 int BUFFER_SIZE = 512*1024; 167 int fd = open(path, O_RDONLY | O_NONBLOCK); 168 fcntl(fd, F_RDADVISE); 169 char buf[BUFFER_SIZE]; 170 float tmp; 171 while(size_t bytes_read = read(fd, buf, BUFFER_SIZE)) 172 { 173 if(bytes_read == (size_t)-1) { 174 printf("bytes_read == (size_t)-1\n"); 175 exit(1); 176 } 177 // if (!bytes_read) break; 178 // for (char* p = buf; p<buf+BUFFER_SIZE;) { 179 // for (int i = 0; i < 5; ++i) { 180 // tmp = *(float*)p; 181 // p += sizeof(float); 182 // } 183 // } 184 } 185 return tmp; 186 } 187 188 float decomp_read(const char* path) 189 { 190 int BUFFER_SIZE = 500*1024; 191 int fd = open(path, O_RDONLY | O_NONBLOCK); 192 fcntl(fd, F_RDADVISE); 193 char buf[BUFFER_SIZE]; 194 char* rbuf = (char*)malloc(BUFFER_SIZE*100); 195 float tmp; 196 while(size_t bytes_read = read(fd, buf, BUFFER_SIZE)) 197 { 198 if(bytes_read == (size_t)-1) { 199 printf("bytes_read == (size_t)-1\n"); 200 exit(1); 201 } 202 if (!bytes_read) break; 203 int len = LZ4_decompress_safe(buf, rbuf, BUFFER_SIZE, BUFFER_SIZE*100); 204 printf("Decompressed into %d bytes\n", len); 205 for (char* p = rbuf; p<buf+(-len);) { 206 for (int i = 0; i < 5; ++i) { 207 tmp = *(float*)p; 208 printf("%f ", tmp); 209 p += sizeof(float); 210 } 211 printf("\n"); 212 } 213 if (len < 0) lseek(fd, -(BUFFER_SIZE+len), SEEK_CUR); 214 } 215 return tmp; 216 } 217 218 int main () { 219 int epochs = 1; 220 float tmp; 221 auto start = std::chrono::high_resolution_clock::now(); 222 //prep(); 223 //compress(); 224 auto prep_end = std::chrono::high_resolution_clock::now(); 225 for (int i = 0; i < epochs; i++) { 226 tmp = decomp_read("../../data_banknote_authentication.lz4"); 227 } 228 auto end = std::chrono::high_resolution_clock::now(); 229 double time = std::chrono::duration_cast<std::chrono::nanoseconds>(end - prep_end + prep_end-start).count() / pow(10,9) / epochs; 230 double runtime = std::chrono::duration_cast<std::chrono::nanoseconds>(end - prep_end).count() / pow(10,9); 231 float ftmp; 232 auto f_start = std::chrono::high_resolution_clock::now(); 233 // for (int i = 0; i < epochs; i++) { 234 // ftmp = std_read("../../data_banknote_authentication.big.bin"); 235 // } 236 auto f_end = std::chrono::high_resolution_clock::now(); 237 // if (flines == lines) std::cout << "Linecount is valid!" << "\n"; 238 // else std::cout << "WARN: INVALID LINECOUNT " << lines << " vs. " << flines << "\n"; 239 if (ftmp == tmp) std::cout << "Final value read is valid!" << "\n"; 240 else std::cout << "WARN: INVALID LAST READ VAL " << tmp << " vs. " << ftmp << "\n"; 241 double ftime = std::chrono::duration_cast<std::chrono::nanoseconds>(f_end - f_start).count() / pow(10,9) / epochs; 242 std::cout << ftime << " " << time << ". 15.25 GB input parsed so " << 15.25/ftime << " GB/s (really " << 243 9.02/ftime << " GB/s) vs. " << 15.25/time << " GB/s (really " << 1.04/time << " GB/s)\n"; 244 std::cout << runtime << " (" << ftime/time * 100 << "% speedup overall, " << ftime/runtime * 100 << "% speedup runtime)\n"; 245 }