zstd-complete 0.3.0

A pure Rust Zstandard compressor and decompressor with bounded streaming I/O
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

use super::{
    fast::{compress_block_fast_no_dict_with_state_and_loaded_dict, FastMatchState},
    fast_ext::compress_block_fast_ext_dict_with_state,
    params::{CParamMode, CompressionParameters},
    sequence_store::{OffBase, RepeatCode, RepeatOffsets},
};
use alloc::{vec, vec::Vec};

fn fast_params() -> CompressionParameters {
    CompressionParameters::for_level_with_mode(1, 32, 32, CParamMode::NoAttachDict)
}

#[test]
fn ext_dict_invalidates_repeat_equal_to_dictionary_size_like_c() {
    let dict = b"abcdefgh";
    let source = b"abcdefghabcdefgh";
    let mut combined = Vec::new();
    combined.extend_from_slice(dict);
    combined.extend_from_slice(source);

    let params = fast_params();
    let mut state = FastMatchState::new();
    state.load_prefix(&combined, dict.len(), params).expect("test storage admission");
    let output = compress_block_fast_ext_dict_with_state(
        &combined,
        dict.len()..combined.len(),
        dict.len(),
        params,
        RepeatOffsets::from_offsets(dict.len() as u32, 3, 7),
        &mut state,
        dict.len(),
    ).expect("test storage admission");

    assert!(!output
        .sequences
        .first()
        .is_some_and(|seq| matches!(seq.off_base(), OffBase::Repeat(RepeatCode::First))));
}

#[test]
fn ext_dict_finds_dictionary_offset_match() {
    let dict = b"prefix:shared-payload";
    let source = b"shared-payload shared-payload";
    let mut combined = Vec::new();
    combined.extend_from_slice(dict);
    combined.extend_from_slice(source);

    let params = fast_params();
    let mut state = FastMatchState::new();
    state.load_prefix(&combined, dict.len(), params).expect("test storage admission");
    let output = compress_block_fast_ext_dict_with_state(
        &combined,
        dict.len()..combined.len(),
        dict.len(),
        params,
        RepeatOffsets::new(),
        &mut state,
        dict.len(),
    ).expect("test storage admission");

    assert!(output
        .sequences
        .iter()
        .any(|seq| matches!(seq.off_base(), OffBase::Offset(_))));
}

#[test]
fn ext_dict_falls_back_to_regular_when_dictionary_window_expired_like_c() {
    let dict = vec![b'a'; 2048];
    let source = vec![b'a'; 2048];
    let mut combined = Vec::new();
    combined.extend_from_slice(&dict);
    combined.extend_from_slice(&source);

    let params = fast_params();
    let mut ext_state = FastMatchState::new();
    ext_state.load_prefix(&combined, dict.len(), params).expect("test storage admission");
    let mut regular_state = ext_state.clone();
    let block_range = dict.len()..combined.len();

    let ext = compress_block_fast_ext_dict_with_state(
        &combined,
        block_range.clone(),
        dict.len(),
        params,
        RepeatOffsets::new(),
        &mut ext_state,
        0,
    ).expect("test storage admission");
    let regular = compress_block_fast_no_dict_with_state_and_loaded_dict(
        &combined,
        block_range,
        params,
        RepeatOffsets::new(),
        &mut regular_state,
        0,
    ).expect("test storage admission");

    assert_eq!(ext, regular);
}