haagenti-zstd 0.1.1

Zstandard-inspired compression for Haagenti (internal format, not Zstd-compatible)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

//! Debug the RLE-like pattern issue - isolate the problem

use haagenti_core::CompressionLevel;
use haagenti_zstd::compress::CompressContext;

fn test_pattern(name: &str, data: &[u8]) {
    let mut ctx = CompressContext::new(CompressionLevel::Fast);
    let compressed = ctx.compress(data).unwrap();

    match zstd::decode_all(std::io::Cursor::new(&compressed)) {
        Ok(decompressed) => {
            if decompressed == data {
                println!(
                    "{}: PASS ({} -> {} bytes, {:.1}x)",
                    name,
                    data.len(),
                    compressed.len(),
                    data.len() as f64 / compressed.len() as f64
                );
            } else {
                println!(
                    "{}: MISMATCH ({} -> {} bytes)",
                    name,
                    data.len(),
                    compressed.len()
                );
            }
        }
        Err(e) => {
            println!(
                "{}: FAIL ({} -> {} bytes): {}",
                name,
                data.len(),
                compressed.len(),
                e
            );
        }
    }
}

fn main() {
    // Test various patterns to isolate the issue
    println!("=== Pattern Tests ===\n");

    // Simple repeats
    test_pattern("A*100", &[b'A'; 100]);
    test_pattern("A*200", &[b'A'; 200]);
    test_pattern("A*400", &vec![b'A'; 400]);

    // Two-phase patterns
    let mut xy = Vec::new();
    xy.extend(vec![b'X'; 100]);
    xy.extend(vec![b'Y'; 100]);
    test_pattern("X*100 Y*100", &xy);

    // Three-phase patterns
    let mut xyz = Vec::new();
    xyz.extend(vec![b'X'; 100]);
    xyz.extend(vec![b'Y'; 100]);
    xyz.extend(vec![b'Z'; 100]);
    test_pattern("X*100 Y*100 Z*100", &xyz);

    // Four-phase patterns (the failing case)
    let mut xyzx = Vec::new();
    xyzx.extend(vec![b'X'; 100]);
    xyzx.extend(vec![b'Y'; 100]);
    xyzx.extend(vec![b'Z'; 100]);
    xyzx.extend(vec![b'X'; 100]);
    test_pattern("X*100 Y*100 Z*100 X*100", &xyzx);

    // Variations to find the boundary
    let mut xyxy = Vec::new();
    xyxy.extend(vec![b'X'; 100]);
    xyxy.extend(vec![b'Y'; 100]);
    xyxy.extend(vec![b'X'; 100]);
    xyxy.extend(vec![b'Y'; 100]);
    test_pattern("X*100 Y*100 X*100 Y*100", &xyxy);

    // Check if it's the repeated X causing issues
    let mut xyx = Vec::new();
    xyx.extend(vec![b'X'; 100]);
    xyx.extend(vec![b'Y'; 100]);
    xyx.extend(vec![b'X'; 100]);
    test_pattern("X*100 Y*100 X*100", &xyx);

    // Check smaller sizes
    let mut xyzx_small = Vec::new();
    xyzx_small.extend(vec![b'X'; 50]);
    xyzx_small.extend(vec![b'Y'; 50]);
    xyzx_small.extend(vec![b'Z'; 50]);
    xyzx_small.extend(vec![b'X'; 50]);
    test_pattern("X*50 Y*50 Z*50 X*50", &xyzx_small);

    // Reference zstd for comparison
    println!("\n=== Reference zstd ===\n");
    let data = xyzx.clone();
    let ref_compressed = zstd::encode_all(std::io::Cursor::new(&data), 1).unwrap();
    println!(
        "Reference: {} -> {} bytes",
        data.len(),
        ref_compressed.len()
    );
}