sigilyx 0.3.2

Reader and writer for Alteryx .yxdb (E1, with experimental E2 read) files.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230

//! Memory benchmark harness (not part of the public API).
//!
//! Generates a large synthetic YXDB file, then reads it back via the eager
//! `into_dataframe()` path. Run under `/usr/bin/time -v` to observe peak RSS:
//!
//!   cargo run --release --example mem_bench -- generate /tmp/bench.yxdb 4000000
//!   /usr/bin/time -v cargo run --release --example mem_bench -- read /tmp/bench.yxdb
use polars::prelude::*;
use sigilyx::{write_yxdb, YxdbReader};
use std::env;
use std::time::Instant;

/// Peak resident set size (VmHWM), in bytes, from /proc/self/status.
fn peak_rss_bytes() -> Option<u64> {
    let status = std::fs::read_to_string("/proc/self/status").ok()?;
    for line in status.lines() {
        if let Some(rest) = line.strip_prefix("VmHWM:") {
            let kb: u64 = rest.trim().trim_end_matches(" kB").trim().parse().ok()?;
            return Some(kb * 1024);
        }
    }
    None
}

fn report_peak_rss() {
    match peak_rss_bytes() {
        Some(b) => eprintln!("peak RSS (VmHWM): {:.1} MiB", b as f64 / (1024.0 * 1024.0)),
        None => eprintln!("peak RSS: unavailable (not Linux?)"),
    }
}

fn make_fixed_df(n: usize) -> DataFrame {
    let ids: Vec<i64> = (0..n as i64).collect();
    let a: Vec<i32> = (0..n as i32).map(|i| i.wrapping_mul(7)).collect();
    let b: Vec<f64> = (0..n).map(|i| (i as f64) * 2.5).collect();
    df!("id" => ids, "a" => a, "b" => b).unwrap()
}

fn make_df(n: usize) -> DataFrame {
    let ids: Vec<i64> = (0..n as i64).collect();
    let values: Vec<f64> = (0..n).map(|i| (i as f64) * 1.0001).collect();
    let names: Vec<String> = (0..n)
        .map(|i| format!("record-name-{i:08}-payload"))
        .collect();
    let notes: Vec<String> = (0..n)
        .map(|i| format!("some longer descriptive text field for row {i} used to simulate a realistic wide string column with enough bytes to matter"))
        .collect();
    df!(
        "id" => ids,
        "value" => values,
        "name" => names,
        "notes" => notes,
    )
    .unwrap()
}

fn main() {
    let args: Vec<String> = env::args().collect();
    match args.get(1).map(|s| s.as_str()) {
        Some("generate") => {
            let path = args.get(2).expect("path");
            let n: usize = args.get(3).expect("n").parse().expect("n is usize");
            let t0 = Instant::now();
            let df = make_df(n);
            eprintln!("generated {n} rows in {:?}", t0.elapsed());
            let t1 = Instant::now();
            write_yxdb(path, &df, &[]).unwrap();
            eprintln!("wrote {path} in {:?}", t1.elapsed());
        }
        Some("read") => {
            let path = args.get(2).expect("path");
            let t0 = Instant::now();
            let reader = YxdbReader::open(path).unwrap();
            let n = reader.header.num_records;
            let df = reader.into_dataframe().unwrap();
            eprintln!(
                "read {} records ({} cols) in {:?}",
                n,
                df.width(),
                t0.elapsed()
            );
            report_peak_rss();

            // Correctness check: recompute every expected value independently
            // (not by re-deriving from the DataFrame itself) and compare. This
            // is what actually exercises the chunk carry-over logic across
            // internal chunk boundaries for a file this size.
            let n = n as usize;
            let id_col = df.column("id").unwrap().i64().unwrap();
            let value_col = df.column("value").unwrap().f64().unwrap();
            let name_col = df.column("name").unwrap().str().unwrap();
            let notes_col = df.column("notes").unwrap().str().unwrap();
            let mut id_sum: i128 = 0;
            for i in 0..n {
                let expected_id = i as i64;
                assert_eq!(id_col.get(i), Some(expected_id), "id mismatch at row {i}");
                id_sum += expected_id as i128;
                let expected_value = expected_id as f64 * 1.0001;
                assert!(
                    (value_col.get(i).unwrap() - expected_value).abs() < 1e-6,
                    "value mismatch at row {i}"
                );
                let expected_name = format!("record-name-{i:08}-payload");
                assert_eq!(
                    name_col.get(i),
                    Some(expected_name.as_str()),
                    "name mismatch at row {i}"
                );
                let expected_notes = format!("some longer descriptive text field for row {i} used to simulate a realistic wide string column with enough bytes to matter");
                assert_eq!(
                    notes_col.get(i),
                    Some(expected_notes.as_str()),
                    "notes mismatch at row {i}"
                );
            }
            let expected_sum: i128 = (0..n as i128).sum();
            assert_eq!(id_sum, expected_sum, "id checksum mismatch");
            eprintln!("verified {n} rows byte-for-byte against independently recomputed expected values - OK");
        }
        Some("generate-fixed") => {
            let path = args.get(2).expect("path");
            let n: usize = args.get(3).expect("n").parse().expect("n is usize");
            let df = make_fixed_df(n);
            write_yxdb(path, &df, &[]).unwrap();
            eprintln!("wrote fixed-size {path} ({n} rows)");
        }
        Some("read-fixed-verify") => {
            let path = args.get(2).expect("path");
            let reader = YxdbReader::open(path).unwrap();
            let n = reader.header.num_records as usize;
            let df = reader.into_dataframe().unwrap();
            let id_col = df.column("id").unwrap().i64().unwrap();
            let a_col = df.column("a").unwrap().i32().unwrap();
            let b_col = df.column("b").unwrap().f64().unwrap();
            for i in 0..n {
                assert_eq!(id_col.get(i), Some(i as i64), "id mismatch at {i}");
                assert_eq!(
                    a_col.get(i),
                    Some((i as i32).wrapping_mul(7)),
                    "a mismatch at {i}"
                );
                assert!(
                    (b_col.get(i).unwrap() - (i as f64) * 2.5).abs() < 1e-9,
                    "b mismatch at {i}"
                );
            }
            eprintln!("verified {n} fixed-size rows - OK");
            report_peak_rss();
        }
        Some("read-projected-verify") => {
            let path = args.get(2).expect("path");
            let reader = YxdbReader::open(path).unwrap();
            let n = reader.header.num_records as usize;
            let df = reader
                .into_dataframe_projected(Some(&["id", "notes"]))
                .unwrap();
            assert_eq!(df.width(), 2, "expected only 2 projected columns");
            let id_col = df.column("id").unwrap().i64().unwrap();
            let notes_col = df.column("notes").unwrap().str().unwrap();
            for i in 0..n {
                assert_eq!(id_col.get(i), Some(i as i64), "id mismatch at {i}");
                let expected_notes = format!("some longer descriptive text field for row {i} used to simulate a realistic wide string column with enough bytes to matter");
                assert_eq!(
                    notes_col.get(i),
                    Some(expected_notes.as_str()),
                    "notes mismatch at {i}"
                );
            }
            eprintln!("verified {n} rows with column projection (id, notes only) - OK");
            report_peak_rss();
        }
        Some("read-batches") => {
            let path = args.get(2).expect("path");
            let batch_size: usize = args.get(3).map(|s| s.parse().unwrap()).unwrap_or(65_536);
            let t0 = Instant::now();
            let mut reader = YxdbReader::open(path).unwrap();
            let mut total = 0usize;
            while let Some(batch) = reader.next_batch(batch_size, None).unwrap() {
                total += batch.height();
            }
            eprintln!("streamed {total} records in {:?}", t0.elapsed());
            report_peak_rss();
        }
        Some("read-batches-verify") => {
            let path = args.get(2).expect("path");
            let batch_size: usize = args.get(3).map(|s| s.parse().unwrap()).unwrap_or(65_536);
            let t0 = Instant::now();
            let mut reader = YxdbReader::open(path).unwrap();
            let mut i: usize = 0;
            while let Some(batch) = reader.next_batch(batch_size, None).unwrap() {
                let id_col = batch.column("id").unwrap().i64().unwrap();
                let value_col = batch.column("value").unwrap().f64().unwrap();
                let name_col = batch.column("name").unwrap().str().unwrap();
                let notes_col = batch.column("notes").unwrap().str().unwrap();
                for row in 0..batch.height() {
                    assert_eq!(id_col.get(row), Some(i as i64), "id mismatch at row {i}");
                    let expected_value = i as f64 * 1.0001;
                    assert!(
                        (value_col.get(row).unwrap() - expected_value).abs() < 1e-6,
                        "value mismatch at row {i}"
                    );
                    let expected_name = format!("record-name-{i:08}-payload");
                    assert_eq!(
                        name_col.get(row),
                        Some(expected_name.as_str()),
                        "name mismatch at row {i}"
                    );
                    let expected_notes = format!("some longer descriptive text field for row {i} used to simulate a realistic wide string column with enough bytes to matter");
                    assert_eq!(
                        notes_col.get(row),
                        Some(expected_notes.as_str()),
                        "notes mismatch at row {i}"
                    );
                    i += 1;
                }
            }
            eprintln!(
                "verified {i} streamed rows byte-for-byte in {:?} - OK",
                t0.elapsed()
            );
            report_peak_rss();
        }
        _ => {
            eprintln!(
                "usage: mem_bench <generate|read|read-batches|read-batches-verify> <path> [n|batch_size]"
            );
            std::process::exit(1);
        }
    }
}