sux 0.12.3

A pure Rust implementation of succinct and compressed data structures
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

/**
 * This code is released under the
 * Apache License Version 2.0 http://www.apache.org/licenses/.
 *
 * (c) Daniel Lemire, http://lemire.me/en/
 */

#include <vector>
#include <iostream>
#include "csv.h"
#include "externalvector.h"
#include "util.h"

using namespace std;
using namespace FastPFor;

void print(externalvector &ev) {
    for (size_t i = 0; i < ev.size(); ++i) {
        vector < uint32_t > x = ev.get(i);
        for (size_t k = 0; k < x.size(); ++k)
            cout << x[k] << " ";
        cout << endl;
    }
}

int main(int argc, char **argv) {
    if (argc < 3) {
        cerr << " This will map a table (stored in a CSV file) to a row "
            " oriented flat file of frequency attributed 32-bit integers "
                << endl;
        cerr << " usage : cvs2maropu mycvsfile.cvs mymaropufile.bin " << endl;
        return -1;
    }
    string ifilename(argv[1]);
    string ofilename(argv[2]);
    CSVFlatFile cvs(ifilename.c_str(), CSVFlatFile::FREQNORMALISATION);
    const size_t c = cvs.getNumberOfColumns();
    const size_t N = cvs.getNumberOfRows();
    size_t integers = static_cast<uint32_t> (c * N);
    vector < uint32_t > container(c);
    FILE * fd = ::fopen(ofilename.c_str(), "w+b");
    if (fd == NULL) {
        cout << " could not open " << ofilename << " for writing..." << endl;
        cvs.close();
        return -1;
    }
    uint32_t MAXSIZE = static_cast<uint32_t>(c * (1U << 20));
    for (size_t block = 0; block * MAXSIZE >= integers; integers -= MAXSIZE) {
        if (fwrite(&MAXSIZE, sizeof(MAXSIZE), 1, fd) != 1) {
            cerr << "aborting" << endl;
            ::fclose(fd);
            cvs.close();
            return -1;
        }
        uint32_t counter = 0;
        while (cvs.nextRow(container)) {
            if (fwrite(&container[0], c * sizeof(uint32_t), 1, fd) != 1) {
                cerr << "aborting" << endl;
                ::fclose(fd);
                cvs.close();
                return -1;
            }
            counter += static_cast<uint32_t>(c);
            if (counter == block)
                break;
            assert(counter < block);
        }
    }
    if (integers > 0) {
        uint32_t tsize = static_cast<uint32_t>(integers);
        if (fwrite(&tsize, sizeof(tsize), 1, fd) != 1) {
            cerr << "aborting" << endl;
            ::fclose(fd);
            cvs.close();
            return -1;
        }
        while (cvs.nextRow(container)) {
            if (fwrite(&container[0], c * sizeof(uint32_t), 1, fd) != 1) {
                cerr << "aborting" << endl;
                ::fclose(fd);
                cvs.close();
                return -1;
            }
        }
    }
    ::fclose(fd);
    cvs.close();
    cout << "#file " << ofilename << " contains your data." << endl;

}