gzippy 0.8.0

The fastest parallel gzip. Drop-in replacement for gzip and pigz, and a Rust library.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

//! Allocation budget tests.
//!
//! Asserts that hot decompression paths don't acquire more heap allocations
//! than a committed baseline. Any PR that adds a `Vec::new()`, `.collect()`,
//! `String::from()`, etc. to a hot path will fail here with a specific count.
//!
//! Baseline: benchmarks/baselines.json "alloc_budget.*"
//! Update with: RECORD_BASELINES=1 cargo test --release alloc_budget -- --nocapture
//!
//! Note: counts include reallocs (Vec growth). The budget has 20% headroom
//! over the measured baseline so minor allocator differences don't flake.

#[cfg(test)]
mod tests {
    use crate::tests::alloc_counter::CountingAllocator;

    // ── baseline reading ─────────────────────────────────────────────────────

    fn read_baseline_u64(key: &str, default: u64) -> u64 {
        for prefix in &["", "../", "../../"] {
            let path = format!("{}benchmarks/baselines.json", prefix);
            if let Ok(json) = std::fs::read_to_string(&path) {
                if let Some(v) = extract_u64(&json, key) {
                    return v;
                }
            }
        }
        default
    }

    fn extract_u64(json: &str, key: &str) -> Option<u64> {
        let pattern = format!("\"{}\":", key);
        let start = json.find(&pattern)? + pattern.len();
        let rest = json[start..].trim_start();
        let end = rest.find(|c: char| !c.is_ascii_digit())?;
        rest[..end].parse().ok()
    }

    // ── tests ─────────────────────────────────────────────────────────────────

    /// Single-member 1MB: count allocations on the default decompression path.
    /// This exercises ISA-L (x86_64) or libdeflate (arm64).
    #[test]
    fn alloc_budget_single_member_1mb() {
        let fixture = crate::tests::fixtures::text_1mb();

        // Warmup — ensures lazy statics are initialized before we count
        let _ = crate::decompress::decompress_gzip_to_vec(&fixture.single_member_gz, 1);

        CountingAllocator::reset();
        let _ = crate::decompress::decompress_gzip_to_vec(&fixture.single_member_gz, 1).unwrap();
        let count = CountingAllocator::count();
        let bytes = CountingAllocator::bytes();

        let baseline = read_baseline_u64("decompress_1mb_max_allocs", 999999);

        eprintln!(
            "alloc_budget_single_member_1mb: {} allocs, {} bytes (baseline <= {})",
            count, bytes, baseline
        );

        if std::env::var("RECORD_BASELINES").is_ok() {
            println!(
                "baseline: alloc_budget.decompress_1mb_max_allocs = {}",
                (count as f64 * 1.20) as u64
            );
            return;
        }

        assert!(
            count <= baseline,
            "alloc budget exceeded: {} allocations > baseline {}\n\
             Check for new Vec/String/collect() on the single-member hot path.",
            count,
            baseline
        );
    }

    /// gzippy-parallel 10MB T4: count allocations on the bgzf parallel path.
    #[test]
    fn alloc_budget_bgzf_10mb() {
        let fixture = crate::tests::fixtures::text_10mb();

        // Warmup
        let _ = crate::decompress::bgzf::decompress_bgzf_parallel_to_vec(&fixture.bgzf_gz, 4);

        CountingAllocator::reset();
        let _ =
            crate::decompress::bgzf::decompress_bgzf_parallel_to_vec(&fixture.bgzf_gz, 4).unwrap();
        let count = CountingAllocator::count();
        let bytes = CountingAllocator::bytes();

        let baseline = read_baseline_u64("decompress_bgzf_10mb_max_allocs", 999999);

        eprintln!(
            "alloc_budget_bgzf_10mb: {} allocs, {} bytes (baseline <= {})",
            count, bytes, baseline
        );

        if std::env::var("RECORD_BASELINES").is_ok() {
            println!(
                "baseline: alloc_budget.decompress_bgzf_10mb_max_allocs = {}",
                (count as f64 * 1.20) as u64
            );
            return;
        }

        assert!(
            count <= baseline,
            "alloc budget exceeded: {} allocations > baseline {}\n\
             Check for new Vec/String/collect() on the bgzf parallel hot path.",
            count,
            baseline
        );
    }
}