zstd_rs/encode/
mod.rs

1//! Compression: configuration, the reusable `Compressor`, and frame assembly.
2mod block;
3mod cost;
4mod dictionary;
5mod entropy;
6mod literals;
7mod matchers;
8mod params;
9mod seqstore;
10mod sequences;
11
12pub use dictionary::EncoderDictionary;
13pub use params::{MAX_LEVEL, MIN_LEVEL};
14
15use crate::error::{Error, Result};
16use crate::frame::{write_block_header, BlockType, FrameFormat, FrameHeader, BLOCK_MAX};
17use crate::xxh64::xxh64;
18use alloc::vec::Vec;
19use entropy::{EntropyState, PredefinedC};
20use matchers::{opt::OptState, BlockInput, Tables};
21use params::CParams;
22use seqstore::{Reps, SeqStore};
23use sequences::Codes;
24
25/// Every compression setting, stated explicitly by the caller. Use one of the
26/// named presets or build your own; there is no implicit default.
27#[derive(Clone, Copy, Debug, PartialEq, Eq)]
28pub struct CompressionConfig {
29    /// zstd level: `MIN_LEVEL..=MAX_LEVEL`, excluding 0. Levels mean what they mean in zstd.
30    pub level: i32,
31    /// Upper bound on the window (log2 bytes, 10..=30). The window also shrinks to fit the input.
32    pub window_log: u32,
33    /// Append the XXH64-based content checksum.
34    pub checksum: bool,
35    /// Record the content size in the frame header.
36    pub content_size: bool,
37    /// Record the dictionary ID in the frame header (when the dictionary has one).
38    pub dict_id: bool,
39    /// Standard frames, or magicless ones (4 bytes smaller; the decoder must be told).
40    pub format: FrameFormat,
41}
42
43impl CompressionConfig {
44    /// zstd level 1: fastest.
45    pub const FAST: CompressionConfig =
46        CompressionConfig { level: 1, window_log: 23, checksum: false, content_size: true, dict_id: true, format: FrameFormat::Standard };
47    /// zstd level 3: the reference default.
48    pub const DEFAULT: CompressionConfig = CompressionConfig { level: 3, ..Self::FAST };
49    /// zstd level 19: optimal parsing.
50    pub const HIGH: CompressionConfig = CompressionConfig { level: 19, ..Self::FAST };
51    /// Small-message profile: level 3, no dictionary ID (the ID is known out of band).
52    pub const MESSAGE: CompressionConfig = CompressionConfig { dict_id: false, ..Self::DEFAULT };
53
54    /// Checks every field's range.
55    pub fn validate(&self) -> Result<()> {
56        if self.level == 0 || !(MIN_LEVEL..=MAX_LEVEL).contains(&self.level) {
57            return Err(Error::Parameter("level must be in MIN_LEVEL..=MAX_LEVEL and not 0"));
58        }
59        if !(10..=30).contains(&self.window_log) {
60            return Err(Error::Parameter("window_log must be in 10..=30"));
61        }
62        Ok(())
63    }
64}
65
66/// Reusable per-frame state shared by the block compressor.
67pub(crate) struct Workspace {
68    pub params: CParams,
69    pub tables: Tables,
70    pub store: SeqStore,
71    pub opt: OptState,
72    pub entropy: EntropyState,
73    pub pre: PredefinedC,
74    pub codes: Codes,
75    pub reps: Reps,
76}
77
78/// A reusable compression context bound to one configuration.
79pub struct Compressor {
80    cfg: CompressionConfig,
81    w: Workspace,
82    prev_len: usize,
83}
84
85impl Compressor {
86    /// Creates a context for `cfg` (validated here).
87    pub fn new(cfg: CompressionConfig) -> Result<Compressor> {
88        cfg.validate()?;
89        let w = Workspace {
90            params: params::lookup(cfg.level, 0),
91            tables: Tables::new(),
92            store: SeqStore::new(),
93            opt: OptState::new(),
94            entropy: EntropyState::new(),
95            pre: PredefinedC::new()?,
96            codes: Codes::default(),
97            reps: [1, 4, 8],
98        };
99        Ok(Compressor { cfg, w, prev_len: 0 })
100    }
101
102    /// The configuration this context compresses with.
103    pub fn config(&self) -> &CompressionConfig {
104        &self.cfg
105    }
106
107    /// Compresses `src` into one frame appended to `out`. A dictionary must have
108    /// been prepared for the same level.
109    pub fn compress(&mut self, src: &[u8], dict: Option<&EncoderDictionary>, out: &mut Vec<u8>) -> Result<()> {
110        let p = self.frame_params(src.len(), dict)?;
111        write_header(&self.cfg, &p, src.len(), dict, out);
112        let whole = Job { start: 0, end: src.len(), prefix_start: 0, first: true, last: true };
113        self.blocks(src, &whole, dict, &p, out)?;
114        write_checksum(&self.cfg, src, out);
115        Ok(())
116    }
117
118    /// Compresses one job of a frame (see [`crate::par`]) into complete blocks,
119    /// without the frame header or checksum. The output depends only on
120    /// (`self.config()`, `src`, `job`, `dict`), never on which context runs it.
121    pub fn compress_job(&mut self, src: &[u8], job: &Job, dict: Option<&EncoderDictionary>, out: &mut Vec<u8>) -> Result<()> {
122        if job.start > job.end || job.end > src.len() || job.prefix_start > job.start {
123            return Err(Error::Parameter("job range outside the input"));
124        }
125        let p = self.frame_params(src.len(), dict)?;
126        let window_start = job.start.saturating_sub(1 << p.window_log);
127        let job = Job { prefix_start: job.prefix_start.max(window_start), ..*job };
128        self.blocks(src, &job, if job.first { dict } else { None }, &p, out)
129    }
130
131    fn frame_params(&self, len: usize, dict: Option<&EncoderDictionary>) -> Result<CParams> {
132        checked_params(&self.cfg, len, dict)
133    }
134
135    /// Emits the blocks covering `job`; the first job starts from the dictionary's
136    /// state, later ones from none (unknown repeat offsets, no reusable tables).
137    fn blocks(&mut self, src: &[u8], job: &Job, dict: Option<&EncoderDictionary>, p: &CParams, out: &mut Vec<u8>) -> Result<()> {
138        let w = &mut self.w;
139        w.params = *p;
140        let hash2_log = if p.strategy == params::Strategy::DFast { p.chain_log } else { 0 };
141        let chain_log = if p.strategy >= params::Strategy::Greedy { p.chain_log } else { 0 };
142        w.tables.start_frame(src.len(), p.hash_log, hash2_log, chain_log, self.prev_len);
143        self.prev_len = src.len();
144        matchers::load_prefix(&mut w.tables, src, job.prefix_start, job.start, p);
145        let dict_tables = dict.and_then(|d| d.tables.as_ref());
146        w.entropy.reset(dict_tables);
147        w.reps = match (job.first, dict) {
148            (true, Some(d)) => d.reps,
149            (true, None) => [1, 4, 8],
150            (false, _) => [0, 0, 0],
151        };
152        w.opt.first_block = true;
153        let window = 1usize << p.window_log;
154        if job.start == job.end {
155            if job.last {
156                write_block_header(true, BlockType::Raw, 0, out);
157            }
158            return Ok(());
159        }
160        let block = BLOCK_MAX.min(window);
161        let mut start = job.start;
162        while start < job.end {
163            let end = (start + block).min(job.end);
164            let attached = dict.filter(|_| end <= window && job.prefix_start == 0).map(|d| &d.matcher);
165            let b = BlockInput { src, start, end, low: job.prefix_start, max_dist: window, dict: attached };
166            block::compress(w, &b, dict_tables, job.last && end == job.end, start > 0, out)?;
167            start = end;
168        }
169        Ok(())
170    }
171}
172
173/// Validates a request and resolves its frame parameters.
174fn checked_params(cfg: &CompressionConfig, len: usize, dict: Option<&EncoderDictionary>) -> Result<CParams> {
175    cfg.validate()?;
176    if len > (u32::MAX >> 2) as usize {
177        return Err(Error::Parameter("input larger than 1 GiB; split it into frames"));
178    }
179    if dict.is_some_and(|d| d.level != cfg.level) {
180        return Err(Error::Parameter("dictionary prepared for a different level"));
181    }
182    Ok(frame_params(cfg, len, dict))
183}
184
185/// Writes the header of a frame holding `len` bytes (used by the job API).
186pub(crate) fn begin_frame(cfg: &CompressionConfig, len: usize, dict: Option<&EncoderDictionary>, out: &mut Vec<u8>) -> Result<()> {
187    let p = checked_params(cfg, len, dict)?;
188    write_header(cfg, &p, len, dict, out);
189    Ok(())
190}
191
192/// Writes the frame header for an input of `len` bytes.
193pub(crate) fn write_header(cfg: &CompressionConfig, p: &CParams, len: usize, dict: Option<&EncoderDictionary>, out: &mut Vec<u8>) {
194    let window = 1u64 << p.window_log;
195    let header = FrameHeader {
196        window_size: window,
197        content_size: cfg.content_size.then_some(len as u64),
198        dict_id: if cfg.dict_id { dict.map_or(0, |d| d.id) } else { 0 },
199        checksum: cfg.checksum,
200        single_segment: cfg.content_size && len as u64 <= window,
201        header_len: 0,
202    };
203    header.write_format(cfg.format, p.window_log, out);
204}
205
206/// Appends the content checksum if configured.
207pub(crate) fn write_checksum(cfg: &CompressionConfig, src: &[u8], out: &mut Vec<u8>) {
208    if cfg.checksum {
209        out.extend_from_slice(&(xxh64(src, 0) as u32).to_le_bytes());
210    }
211}
212
213/// One independently compressible slice of a frame's input: blocks for
214/// `src[start..end]`, which may reference `src[prefix_start..start]` as history.
215#[derive(Clone, Copy, Debug, PartialEq, Eq)]
216pub struct Job {
217    /// First input byte this job encodes.
218    pub start: usize,
219    /// One past the last input byte this job encodes.
220    pub end: usize,
221    /// Lowest input byte the job may reference (the overlap with earlier jobs).
222    pub prefix_start: usize,
223    /// True for the first job: it may use the dictionary and its repeat offsets.
224    pub first: bool,
225    /// True for the last job: its final block is marked last.
226    pub last: bool,
227}
228
229/// Frame parameters: the level's row for this input size, shrunk to fit; with a
230/// dictionary, the dictionary's search settings (its tables are shared) and frame-sized tables.
231fn frame_params(cfg: &CompressionConfig, len: usize, dict: Option<&EncoderDictionary>) -> CParams {
232    let mut p = params::adjust(params::lookup(cfg.level, len as u64), len as u64, 0);
233    if let Some(d) = dict {
234        let dp = d.matcher.params;
235        p.strategy = dp.strategy;
236        p.min_match = dp.min_match;
237        p.search_log = dp.search_log;
238        p.target_length = dp.target_length;
239        p.hash_log = p.hash_log.max(6);
240        p.chain_log = p.chain_log.max(6);
241    }
242    p.window_log = p.window_log.min(cfg.window_log);
243    p
244}
zstd_rs/encode/mod.rs

zstd_rs/encode/
mod.rs
