riegeli 0.1.0

Rust implementation of the Riegeli/records file format
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

//! Integration tests for file metadata (RecordsMetadata) read and write.

// ---------------------------------------------------------------------------
// Re-implement make_records() logic from the bench (we can't import bench code)
// ---------------------------------------------------------------------------

const NUM_RECORDS: usize = 10_000;
const RECORD_SIZE: usize = 1024;
const TOTAL_BYTES: u64 = (NUM_RECORDS * RECORD_SIZE) as u64;

fn make_records() -> Vec<Vec<u8>> {
    (0..NUM_RECORDS)
        .map(|i| {
            let mut rec = Vec::with_capacity(RECORD_SIZE);
            rec.push(0x08);
            let mut v = i as u64;
            loop {
                if v < 0x80 {
                    rec.push(v as u8);
                    break;
                }
                rec.push((v as u8 & 0x7f) | 0x80);
                v >>= 7;
            }
            rec.push(0x12);
            let remaining = RECORD_SIZE.saturating_sub(rec.len() + 2);
            let len = remaining;
            if len < 0x80 {
                rec.push(len as u8);
            } else {
                rec.push((len as u8 & 0x7f) | 0x80);
                rec.push((len >> 7) as u8);
            }
            let pattern: [u8; 8] = [
                (i & 0xff) as u8,
                ((i >> 8) & 0xff) as u8,
                0xAA,
                0x55,
                (i & 0xff) as u8,
                0xBB,
                0x33,
                ((i >> 4) & 0xff) as u8,
            ];
            let fill_len = RECORD_SIZE.saturating_sub(rec.len());
            for j in 0..fill_len {
                rec.push(pattern[j % pattern.len()]);
            }
            rec.truncate(RECORD_SIZE);
            while rec.len() < RECORD_SIZE {
                rec.push(0);
            }
            rec
        })
        .collect()
}

// ---------------------------------------------------------------------------
// Adversarial 16.1: make_records() produces exactly the right count and size
// ---------------------------------------------------------------------------

/// Each generated record must be exactly RECORD_SIZE bytes.
/// A size mismatch would invalidate the throughput calculation (TOTAL_BYTES
/// would not match actual payload, making MB/s numbers misleading).
#[test]
fn adv_16_make_records_size_invariant() {
    let records = make_records();
    assert_eq!(
        records.len(),
        NUM_RECORDS,
        "expected {NUM_RECORDS} records, got {}",
        records.len()
    );
    for (i, rec) in records.iter().enumerate() {
        assert_eq!(
            rec.len(),
            RECORD_SIZE,
            "record[{i}] has length {} != {RECORD_SIZE}",
            rec.len()
        );
    }
    // Sanity: total bytes equals TOTAL_BYTES constant
    let actual_total: u64 = records.iter().map(|r| r.len() as u64).sum();
    assert_eq!(actual_total, TOTAL_BYTES);
}

/// Records must not all be identical — distinct records exercise the compressor
/// more realistically and exercise transpose field splitting.
#[test]
fn adv_16_make_records_are_distinct() {
    let records = make_records();
    // Check first vs last — they should differ (different index varint)
    assert_ne!(
        records[0],
        records[NUM_RECORDS - 1],
        "first and last records should differ"
    );
    // Spot-check a handful of pairs
    for i in [0usize, 1, 100, 999, 9999] {
        if i + 1 < NUM_RECORDS {
            assert_ne!(
                records[i],
                records[i + 1],
                "records[{i}] and records[{i}+1] are identical"
            );
        }
    }
}