riegeli 0.1.0

Rust implementation of the Riegeli/records file format
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224

//! Conformance tests: Rust-written files are readable by C++ and vice versa.

use riegeli::{ReaderOptions, RecordReader, RecordWriter, WriterOptions};
use std::io::Cursor;

/// Number of records in each golden file.
const NUM_RECORDS: usize = 100;

/// Generate the expected record content for index `i`.
fn expected_record(i: usize) -> String {
    format!("record_{:06}", i)
}

/// Read all records from a golden file and verify they match expectations.
fn verify_golden_file(data: &[u8], label: &str) {
    let cursor = Cursor::new(data);
    let mut reader = RecordReader::new(cursor, ReaderOptions::new())
        .unwrap_or_else(|e| panic!("{label}: failed to open: {e:?}"));

    let mut count = 0;
    while let Some(record) = reader
        .read_record()
        .unwrap_or_else(|e| panic!("{label}: read_record failed at index {count}: {e:?}"))
    {
        let expected = expected_record(count);
        assert_eq!(
            record,
            expected.as_bytes(),
            "{label}: record {count} mismatch: expected {:?}, got {:?}",
            expected,
            String::from_utf8_lossy(&record)
        );
        count += 1;
    }

    assert_eq!(
        count, NUM_RECORDS,
        "{label}: expected {NUM_RECORDS} records, got {count}"
    );
}

// ---------------------------------------------------------------------------
// Criterion 14.1: simple_none.riegeli decoded correctly
// ---------------------------------------------------------------------------
#[test]
fn criterion_14_1_simple_none() {
    let data = include_bytes!("../testdata/golden/simple_none.riegeli");
    verify_golden_file(data, "simple_none");
}

// ---------------------------------------------------------------------------
// Criterion 14.2: simple_brotli.riegeli decoded correctly
// Also activates criterion 4.5
// ---------------------------------------------------------------------------
#[test]
#[cfg(feature = "brotli")]
fn criterion_14_2_simple_brotli() {
    let data = include_bytes!("../testdata/golden/simple_brotli.riegeli");
    verify_golden_file(data, "simple_brotli");
}

// ---------------------------------------------------------------------------
// Criterion 14.3: simple_zstd.riegeli decoded correctly
// Also activates criterion 4.6
// ---------------------------------------------------------------------------
#[test]
#[cfg(feature = "zstd")]
fn criterion_14_3_simple_zstd() {
    let data = include_bytes!("../testdata/golden/simple_zstd.riegeli");
    verify_golden_file(data, "simple_zstd");
}

// ---------------------------------------------------------------------------
// Criterion 14.4: transpose_none.riegeli decoded correctly
// Activates criterion 9.4
// ---------------------------------------------------------------------------
#[test]
fn criterion_14_4_transpose_none() {
    let data = include_bytes!("../testdata/golden/transpose_none.riegeli");
    verify_golden_file(data, "transpose_none");
}

// ---------------------------------------------------------------------------
// Criterion 14.5: transpose_brotli.riegeli decoded correctly
// Activates criterion 9.5
// ---------------------------------------------------------------------------
#[test]
#[cfg(feature = "brotli")]
fn criterion_14_5_transpose_brotli() {
    let data = include_bytes!("../testdata/golden/transpose_brotli.riegeli");
    verify_golden_file(data, "transpose_brotli");
}

// ---------------------------------------------------------------------------
// Criterion 14.6: transpose_zstd.riegeli decoded correctly
// ---------------------------------------------------------------------------
#[test]
#[cfg(feature = "zstd")]
fn criterion_14_6_transpose_zstd() {
    let data = include_bytes!("../testdata/golden/transpose_zstd.riegeli");
    verify_golden_file(data, "transpose_zstd");
}

// ---------------------------------------------------------------------------
// Criterion 14.7: Rust-written simple+Brotli file verified by C++ verifier
// Activates criterion 12.6
// ---------------------------------------------------------------------------
#[test]
#[cfg(feature = "brotli")]
fn criterion_14_7_rust_simple_brotli_verified_by_cpp() {
    let verifier_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join("cpp_verifier_bin");

    // Skip if the C++ verifier binary is not available
    if !verifier_path.exists() {
        // Try the bazel-built binary location
        let bazel_path =
            std::path::Path::new("/home/mike.danese/code/riegeli/bazel-bin/tools/verify_rust_file");
        if !bazel_path.exists() {
            eprintln!(
                "Skipping: C++ verifier not found at {:?} or {:?}",
                verifier_path, bazel_path
            );
            return;
        }
        run_cpp_verification(bazel_path, false);
        return;
    }
    run_cpp_verification(&verifier_path, false);
}

// ---------------------------------------------------------------------------
// Criterion 14.8: Rust-written transpose+Brotli file verified by C++ verifier
// ---------------------------------------------------------------------------
#[test]
#[cfg(feature = "brotli")]
fn criterion_14_8_rust_transpose_brotli_verified_by_cpp() {
    let bazel_path =
        std::path::Path::new("/home/mike.danese/code/riegeli/bazel-bin/tools/verify_rust_file");
    if !bazel_path.exists() {
        eprintln!("Skipping: C++ verifier not found at {:?}", bazel_path);
        return;
    }
    run_cpp_verification(bazel_path, true);
}

/// Write a Rust riegeli file with 100 records, then invoke the C++ verifier.
#[cfg(feature = "brotli")]
fn run_cpp_verification(verifier: &std::path::Path, transpose: bool) {
    use riegeli::CompressionType;

    // Write the file to a temp location
    let tmp_dir = std::env::temp_dir();
    let suffix = if transpose {
        "transpose_brotli"
    } else {
        "simple_brotli"
    };
    let tmp_file = tmp_dir.join(format!("riegeli_rs_test_{}.riegeli", suffix));

    {
        let file = std::fs::File::create(&tmp_file).expect("create temp file");
        let opts = WriterOptions::new()
            .compression(CompressionType::Brotli)
            .transpose(transpose);
        let mut writer = RecordWriter::new(file, opts).expect("create writer");
        for i in 0..NUM_RECORDS {
            let record = expected_record(i);
            writer
                .write_record(record.as_bytes())
                .expect("write record");
        }
        writer.close().expect("close writer");
    }

    // Invoke the C++ verifier
    let output = std::process::Command::new(verifier)
        .arg(format!("--file={}", tmp_file.display()))
        .arg(format!("--num_records={}", NUM_RECORDS))
        .output()
        .expect("failed to execute C++ verifier");

    let stdout = String::from_utf8_lossy(&output.stdout);
    let stderr = String::from_utf8_lossy(&output.stderr);

    assert!(
        output.status.success(),
        "C++ verifier failed for {suffix}:\nstdout: {stdout}\nstderr: {stderr}"
    );

    assert!(
        stdout.contains("OK"),
        "C++ verifier did not print OK for {suffix}:\nstdout: {stdout}\nstderr: {stderr}"
    );

    // Clean up
    let _ = std::fs::remove_file(&tmp_file);
}

// ---------------------------------------------------------------------------
// Additional test: Verify C++ golden file with RecordReader + seek
// Activates criterion 6.2 (reading C++ golden file)
// ---------------------------------------------------------------------------
#[test]
fn criterion_6_2_read_cpp_golden_file() {
    // Use the simple_none golden file (no compression needed)
    let data = include_bytes!("../testdata/golden/simple_none.riegeli");
    let cursor = Cursor::new(data.as_slice());
    let mut reader = RecordReader::new(cursor, ReaderOptions::new()).expect("open");

    // Read all records
    let mut records = Vec::new();
    while let Some(rec) = reader.read_record().expect("read") {
        records.push(rec);
    }
    assert_eq!(records.len(), NUM_RECORDS);

    // Verify each record
    for (i, rec) in records.iter().enumerate() {
        assert_eq!(rec.as_slice(), expected_record(i).as_bytes());
    }
}