znippy-compress 0.9.9

Compression logic for Znippy, a parallel chunked compression system.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

//! Tiny release timing harness for the `compress_stream` path (gate 2) + a
//! round-trip byte-identical check (gate 1). NOT part of the published crate;
//! an example only. Run: `cargo run -p znippy-compress --release --example bench_stream`.

use std::time::Instant;

use znippy_compress::{compress_stream, ArchiveEntry};

const MB: usize = 1024 * 1024;

fn gen_text(size: usize) -> Vec<u8> {
    // Mildly/highly compressible English-ish filler.
    let word = b"the quick brown fox jumps over the lazy dog ";
    let mut v = Vec::with_capacity(size);
    while v.len() < size {
        v.extend_from_slice(word);
    }
    v.truncate(size);
    v
}

fn gen_random(size: usize) -> Vec<u8> {
    // Deterministic xorshift => incompressible.
    let mut s = 0x9E37_79B9_7F4A_7C15u64;
    (0..size)
        .map(|_| {
            s ^= s >> 12;
            s ^= s << 25;
            s ^= s >> 27;
            (s.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 33) as u8
        })
        .collect()
}

fn build_entries(kind: &str) -> Vec<ArchiveEntry> {
    match kind {
        "text" => vec![ArchiveEntry::new("text.txt", gen_text(512 * MB))],
        "random" => vec![ArchiveEntry::new("random.bin", gen_random(512 * MB))],
        "mixed" => vec![
            ArchiveEntry::new("pom.xml", gen_text(64 * 1024)),
            ArchiveEntry::new("app.jar", gen_random(180 * MB)),
            ArchiveEntry::new("sources.jar", gen_text(120 * MB)),
            ArchiveEntry::new("javadoc.jar", gen_text(90 * MB)),
            ArchiveEntry::new("deps.tar.gz", gen_random(140 * MB)),
        ],
        "small" => (0..2000)
            .map(|i| ArchiveEntry::new(format!("f_{i:04}.txt"), gen_text(4096 + (i % 97))))
            .collect(),
        _ => panic!("unknown kind"),
    }
}

fn run_once(kind: &str, entries: Vec<ArchiveEntry>, archive: &std::path::PathBuf) -> (f64, u64, u64) {
    let input_bytes: u64 = entries.iter().map(|e| e.data.len() as u64).sum();
    let t0 = Instant::now();
    let compressor = compress_stream(archive, false).expect("compress_stream");
    for e in entries {
        compressor.sender().send(e).expect("send");
    }
    let report = compressor.finish().expect("finish");
    let secs = t0.elapsed().as_secs_f64();
    let _ = kind;
    (secs, input_bytes, report.chunks)
}

fn main() {
    let dir = std::env::var("BENCH_DIR").unwrap_or_else(|_| "/run/media/rickard/T9/tmp/bench_stream".into());
    std::fs::create_dir_all(&dir).unwrap();

    for kind in ["text", "random", "mixed", "small"] {
        let archive = std::path::PathBuf::from(&dir).join(format!("{kind}.znippy"));
        let mut best = f64::INFINITY;
        let mut in_bytes = 0u64;
        let mut chunks = 0u64;
        for _ in 0..3 {
            let entries = build_entries(kind);
            let (secs, ib, ch) = run_once(kind, entries, &archive);
            in_bytes = ib;
            chunks = ch;
            if secs < best {
                best = secs;
            }
        }
        let in_mb = in_bytes as f64 / MB as f64;
        let mbps = in_mb / best;

        // Round-trip verify (gate 1): decompress and compare bytes.
        let arc = archive.with_extension("znippy");
        let dec = std::path::PathBuf::from(&dir).join(format!("{kind}_dec"));
        let _ = std::fs::remove_dir_all(&dec);
        std::fs::create_dir_all(&dec).unwrap();
        let v = znippy_common::decompress_archive(&arc, true, &dec).expect("decompress");
        let orig = build_entries(kind);
        let mut roundtrip_ok = v.corrupt_files == 0;
        for e in &orig {
            let p = dec.join(&e.relative_path);
            match std::fs::read(&p) {
                Ok(b) => {
                    if b != e.data {
                        roundtrip_ok = false;
                        eprintln!("MISMATCH {kind} {}", e.relative_path);
                    }
                }
                Err(err) => {
                    roundtrip_ok = false;
                    eprintln!("MISSING {kind} {}: {err}", e.relative_path);
                }
            }
        }

        println!(
            "BENCH kind={kind} best_s={best:.4} in_mb={in_mb:.1} mbps={mbps:.1} chunks={chunks} corrupt={} roundtrip_ok={roundtrip_ok}",
            v.corrupt_files
        );
    }
}