zstd-complete 0.3.0

A pure Rust Zstandard compressor and decompressor with bounded streaming I/O
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

use alloc::vec::Vec;

use crate::{bit_io::BitWriter, blocks::sequence_section::Sequence, fse::fse_encoder::FSETable};

use super::encode_sequences;

mod builder;
mod selection;

pub(in crate::encoding::blocks::compressed) use builder::{build_sequence_table, TableBuilder};
#[cfg(test)]
pub(super) use selection::choose_table;
pub(super) use selection::{
    choose_c_dfast_compact_sequence_table_modes_from_prepared,
    choose_c_dfast_compact_sequence_table_modes_from_stored,
    choose_c_dfast_compact_sequence_table_modes_from_stored_with_final_history,
    choose_c_fast_sequence_table_modes_from_prepared,
    choose_c_fast_sequence_table_modes_from_stored,
    choose_c_fast_sequence_table_modes_from_stored_with_final_history,
    choose_c_sequence_table_modes_from_prepared, choose_c_sequence_table_modes_from_stored,
    choose_c_sequence_table_modes_from_stored_with_scratch, choose_sequence_table_modes,
    choose_sequence_table_modes_for_estimate,
    choose_sequence_table_modes_for_estimate_from_counts_with_scratch,
};

#[derive(Clone)]
#[allow(clippy::large_enum_variant)]
pub(super) enum FseTableMode<'a> {
    Predefined(&'a FSETable),
    Rle(u8),
    Encoded(FSETable),
    RepeatLast(&'a FSETable),
}

impl FseTableMode<'_> {
    pub(super) fn table(&self) -> Option<&FSETable> {
        match self {
            Self::Predefined(t) => Some(t),
            Self::RepeatLast(t) => Some(t),
            Self::Encoded(t) => Some(t),
            Self::Rle(_) => None,
        }
    }
}

pub(super) struct SequenceModeSearchConfig<'a> {
    pub(super) ll_previous: Option<&'a FSETable>,
    pub(super) ll_repeat_valid: bool,
    pub(super) ll_default: &'a FSETable,
    pub(super) ml_previous: Option<&'a FSETable>,
    pub(super) ml_repeat_valid: bool,
    pub(super) ml_default: &'a FSETable,
    pub(super) of_previous: Option<&'a FSETable>,
    pub(super) of_repeat_valid: bool,
    pub(super) of_default: &'a FSETable,
    pub(super) repeat_table_max_sequences: usize,
    pub(super) llml_predefined_max_sequences: usize,
    pub(super) of_predefined_max_sequences: usize,
    pub(super) of_max_log: u8,
    pub(super) exact_sequence_mode_search: bool,
    pub(super) c_fast_heuristics: bool,
    pub(super) c_cost_model: bool,
}

pub(super) fn exact_sequence_section_size(
    sequences: &[Sequence],
    ll_mode: &FseTableMode<'_>,
    ml_mode: &FseTableMode<'_>,
    of_mode: &FseTableMode<'_>,
) -> usize {
    let mut encoded = Vec::new();
    let mut writer = BitWriter::from(&mut encoded);
    writer.write_bits(encode_fse_table_modes(ll_mode, ml_mode, of_mode), 8).expect("owned legacy codec allocation failed");
    encode_table(ll_mode, &mut writer).expect("owned legacy codec allocation failed");
    encode_table(of_mode, &mut writer).expect("owned legacy codec allocation failed");
    encode_table(ml_mode, &mut writer).expect("owned legacy codec allocation failed");
    encode_sequences(sequences, &mut writer, ll_mode, ml_mode, of_mode).expect("owned legacy codec allocation failed");
    writer.flush().expect("owned legacy codec allocation failed");
    encoded.len()
}

pub(super) fn encode_table(mode: &FseTableMode<'_>, writer: &mut BitWriter<impl crate::workspace::ByteBufferOwner>) -> Result<(), crate::workspace::CapacityError> {
    match mode {
        FseTableMode::Predefined(_) => {}
        FseTableMode::Rle(symbol) => writer.write_bits(*symbol, 8)?,
        FseTableMode::RepeatLast(_) => {}
        FseTableMode::Encoded(table) => table.write_table(writer)?,
    }
    Ok(())
}

pub(super) fn encode_fse_table_modes(
    ll_mode: &FseTableMode<'_>,
    ml_mode: &FseTableMode<'_>,
    of_mode: &FseTableMode<'_>,
) -> u8 {
    fn mode_to_bits(mode: &FseTableMode<'_>) -> u8 {
        match mode {
            FseTableMode::Predefined(_) => 0,
            FseTableMode::Rle(_) => 1,
            FseTableMode::Encoded(_) => 2,
            FseTableMode::RepeatLast(_) => 3,
        }
    }
    mode_to_bits(ll_mode) << 6 | mode_to_bits(of_mode) << 4 | mode_to_bits(ml_mode) << 2
}