jacobian

a basic keras-like neural network library for c++/python
Log | Files | Refs | README

data.cpp (3398B)

      1 typedef float val_t;
      2 inline float scan(char **p)
      3 {
      4     float n;
      5     int neg = 1;
      6     while (!isdigit(**p) && **p != '-' && **p != '.') ++*p;
      7     if (**p == '-') neg = -1, ++*p;
      8     for (n=0; isdigit(**p); ++*p) (n *= 10) += (**p-'0');
      9     if (*(*p)++ != '.') return n*neg;
     10     float d = 1;
     11     for (; isdigit(**p); ++*p) n += (d /= 10) * (**p-'0');
     12     return n*neg;
     13 }
     14 
     15 void prep(const char* rname, const char* wname)
     16 {
     17     FILE* wptr = fopen(wname, "wb");
     18     FILE* rptr = fopen(rname, "rb");
     19     if(!wptr) throw std::runtime_error{"prep() could not write to the output file."};
     20     if(!rptr) throw std::runtime_error{"prep() could not read file for binary translation."};
     21     float tmp;
     22     char buf[BUFFER_SIZE+1];
     23     while(fgets(buf, BUFFER_SIZE+1, rptr)) {
     24         char* p = buf;
     25         for (int i=0; i<5; ++i) {
     26             tmp = scan(&p);
     27             fwrite(static_cast<void*>(&tmp), sizeof(float), 1, wptr);
     28         }
     29     }
     30     fclose(wptr);
     31     fclose(rptr);
     32 }
     33 
     34 void Network::next_batch(int fd)
     35 {
     36     Expects(fd > 0); // File descriptor must be valid.
     37     uintmax_t lines = 0;
     38     while(size_t bytes_read = read(fd, buf, BUFFER_SIZE)) {
     39         if (!bytes_read) break;
     40         p = buf;
     41         while(p < buf+BUFFER_SIZE) {
     42             if (lines >= 10) return;
     43             for (int i=0; i<layers[0].contents.cols(); ++i) {
     44                 layers[0].contents(lines,i) = *(reinterpret_cast<float*>(p));
     45                 p += sizeof(float);
     46             }
     47             (*labels)(lines,0) = *(reinterpret_cast<float*>(p));
     48             p += sizeof(float);
     49             ++lines;
     50         }
     51     }
     52     if (p < buf+BUFFER_SIZE) {
     53         while(p < buf+BUFFER_SIZE) {
     54             if (lines >= 10) return;
     55             for (int i=0; i<layers[0].contents.cols(); ++i) {
     56                 layers[0].contents(lines,i) = *(reinterpret_cast<float*>(p));
     57                 p += sizeof(float);
     58             }
     59             (*labels)(lines,0) = *(reinterpret_cast<float*>(p));
     60             p += sizeof(float);
     61             ++lines;
     62         }
     63     }
     64 }
     65 
     66 int prep_file(const char* path, const char* out_path)
     67 {
     68     FILE* rptr = fopen(path, "r");
     69     if (!rptr) throw std::runtime_error{"prep_file() could not open file for shuffle/read."};
     70     char line[MAXLINE];
     71     std::vector<std::string> lines;
     72     int count = 0;
     73     while (fgets(line, MAXLINE, rptr) != NULL) {
     74         lines.emplace_back(line);
     75         count++;
     76     }
     77     lines[lines.size()-1] = lines[lines.size()-1] + "\n";
     78     std::random_device rd;
     79     std::mt19937 g(rd());
     80     std::shuffle(lines.begin(), lines.end(), g);
     81     fclose(rptr);
     82     FILE* wptr = fopen(out_path, "w");
     83     for (std::string & i : lines) {
     84         const char* cstr = i.c_str();
     85         fprintf(wptr,"%s", cstr);
     86     }
     87     fclose(wptr);
     88     return count;
     89 }
     90 
     91 int split_file(const char* path, int lines, float ratio)
     92 {
     93     FILE* src = fopen(path, "r");
     94     if (!src) throw std::runtime_error{"split_file() could not open file to split."};
     95     FILE* test = fopen(VAL_PATH, "w");
     96     FILE* train = fopen(TRAIN_PATH, "w");
     97     int switch_line = round(ratio * lines);
     98     char line[MAXLINE];
     99     int tests = 0;
    100     for (int i = 0; fgets(line, MAXLINE, src) != NULL; i++) {
    101         if (i > switch_line) {
    102             fprintf(test, "%s", line);
    103             tests++;
    104         }
    105         else fprintf(train, "%s", line);
    106     }
    107     fclose(src);
    108     fclose(test);
    109     fclose(train);
    110     return tests;
    111 }