Skip to main content

zrip_encode/
streaming.rs

1#![forbid(unsafe_code)]
2
3use std::io::{self, Write};
4
5use crate::block_encoder::{self, BlockEncodeWorkspace};
6use crate::dfast;
7use crate::fast;
8#[cfg(feature = "ldm")]
9use crate::ldm::LdmState;
10use crate::rebase::reduce_hash_table;
11use crate::strategy::{self, LevelParams, Strategy};
12use crate::write_frame_header_without_content_size;
13use zrip_core::Sequence;
14use zrip_core::dict::Dictionary;
15use zrip_core::error::CompressError;
16use zrip_core::frame::MAX_BLOCK_SIZE;
17use zrip_core::xxhash::Xxh64State;
18
19#[derive(Clone, Copy)]
20enum EncoderState {
21    Ready,
22    Finished,
23    Failed,
24}
25
26/// Streaming zstd compressor implementing [`Write`].
27///
28/// Buffers input until a full block (128 KiB, or the window if smaller) is
29/// ready, then compresses and writes it to the underlying writer. Call [`finish`](Self::finish)
30/// to flush the final block, write the content checksum, and recover the
31/// writer.
32///
33/// After a write, flush, or finalization error, the encoder cannot be reused.
34/// Later writes, flushes, [`finish`](Self::finish), and [`reset`](Self::reset)
35/// return errors. Discard the encoder and its incomplete output.
36///
37/// Internal buffers (hash tables, sequence scratch, block encoder workspace)
38/// are allocated once and reused across blocks. To reuse them across
39/// multiple frames, call [`reset`](Self::reset) instead of `finish`:
40///
41/// ```
42/// use std::io::Write;
43///
44/// let mut encoder = zrip::FrameEncoder::new(Vec::new(), 1).unwrap();
45/// encoder.write_all(b"first frame").unwrap();
46/// let first = encoder.reset(Vec::new()).unwrap();   // reuses buffers
47/// encoder.write_all(b"second frame").unwrap();
48/// let second = encoder.finish().unwrap();
49/// ```
50pub struct FrameEncoder<W: Write> {
51    inner: W,
52    params: LevelParams,
53    buffer: Vec<u8>,
54    rep_offsets: [u32; 3],
55    hasher: Xxh64State,
56    header_written: bool,
57    state: EncoderState,
58    workspace: BlockEncodeWorkspace,
59    dict: Option<Dictionary>,
60    first_block: bool,
61    hash_table: Vec<u32>,
62    hash_long: Vec<u32>,
63    sequences: Vec<Sequence>,
64    block_out: Vec<u8>,
65    window_buf: Vec<u8>,
66    #[cfg(feature = "ldm")]
67    ldm_state: Option<LdmState>,
68}
69
70impl<W: Write> FrameEncoder<W> {
71    /// Creates a new streaming encoder at the given level (-7..=4; -8 runs as -7).
72    pub fn new(writer: W, level: i32) -> Result<Self, CompressError> {
73        let params = strategy::level_params(level).ok_or(CompressError::InvalidLevel(level))?;
74        Self::from_params(writer, params, None)
75    }
76
77    /// Creates a new streaming encoder with [`Options`](strategy::Options)
78    /// for large windows and/or LDM.
79    pub fn with_options(
80        writer: W,
81        level: i32,
82        opts: &strategy::Options,
83    ) -> Result<Self, CompressError> {
84        let mut params = strategy::level_params(level).ok_or(CompressError::InvalidLevel(level))?;
85        strategy::apply_options(&mut params, opts);
86        Self::from_params(writer, params, None)
87    }
88
89    /// Creates a new streaming encoder with a dictionary at the given level (-7..=4; -8 runs as -7).
90    pub fn with_dict(writer: W, level: i32, dict: Dictionary) -> Result<Self, CompressError> {
91        let params = strategy::level_params(level).ok_or(CompressError::InvalidLevel(level))?;
92        Self::from_params(writer, params, Some(dict))
93    }
94
95    #[allow(clippy::unnecessary_wraps)]
96    fn from_params(
97        writer: W,
98        params: LevelParams,
99        dict: Option<Dictionary>,
100    ) -> Result<Self, CompressError> {
101        let (hash_table, hash_long) = alloc_hash_tables(&params);
102        let (rep_offsets, first_block) = match &dict {
103            Some(d) => (*d.rep_offsets(), true),
104            None => ([1, 4, 8], false),
105        };
106        let mut window_buf = Vec::new();
107        if let Some(ref d) = dict {
108            window_buf.extend_from_slice(d.content());
109        }
110        #[cfg(feature = "ldm")]
111        let ldm_state = params.ldm_params.as_ref().map(LdmState::new);
112        Ok(Self {
113            inner: writer,
114            params,
115            buffer: Vec::new(),
116            rep_offsets,
117            hasher: Xxh64State::new(0),
118            header_written: false,
119            state: EncoderState::Ready,
120            workspace: BlockEncodeWorkspace::new(),
121            dict,
122            first_block,
123            hash_table,
124            hash_long,
125            sequences: Vec::new(),
126            block_out: Vec::new(),
127            window_buf,
128            #[cfg(feature = "ldm")]
129            ldm_state,
130        })
131    }
132
133    /// Flushes remaining data, writes the content checksum, and returns the inner writer.
134    pub fn finish(mut self) -> Result<W, io::Error> {
135        self.finish_frame()?;
136        Ok(self.inner)
137    }
138
139    /// Finishes the current frame and installs `new_writer` for the next one.
140    ///
141    /// Returns the previous writer containing the completed frame. All
142    /// internal buffers (hash tables, workspace, block scratch) stay
143    /// allocated and are reused for the next frame.
144    pub fn reset(&mut self, new_writer: W) -> Result<W, io::Error> {
145        self.finish_frame()?;
146        let old = core::mem::replace(&mut self.inner, new_writer);
147        self.header_written = false;
148        self.state = EncoderState::Ready;
149        self.first_block = self.dict.is_some();
150        self.rep_offsets = match &self.dict {
151            Some(d) => *d.rep_offsets(),
152            None => [1, 4, 8],
153        };
154        self.hasher = Xxh64State::new(0);
155        self.workspace.prev_huffman = None;
156        self.window_buf.clear();
157        if let Some(ref d) = self.dict {
158            self.window_buf.extend_from_slice(d.content());
159        }
160        #[cfg(feature = "ldm")]
161        if let Some(ref mut ldm) = self.ldm_state {
162            ldm.reset();
163        }
164        Ok(old)
165    }
166
167    fn finish_frame(&mut self) -> io::Result<()> {
168        if matches!(self.state, EncoderState::Failed) {
169            return Err(io::Error::other("encoder cannot be reused after an error"));
170        }
171        if matches!(self.state, EncoderState::Finished) {
172            return Ok(());
173        }
174        self.state = EncoderState::Failed;
175
176        if !self.header_written {
177            self.write_header()?;
178        }
179
180        self.flush_block(true)?;
181
182        let hash = self.hasher.finish();
183        let checksum = (hash & 0xFFFF_FFFF) as u32;
184        self.inner.write_all(&checksum.to_le_bytes())?;
185        self.state = EncoderState::Finished;
186        Ok(())
187    }
188
189    fn write_header(&mut self) -> io::Result<()> {
190        self.block_out.clear();
191        write_frame_header_without_content_size(
192            &mut self.block_out,
193            self.dict.as_ref().map(Dictionary::id),
194            self.params.window_log,
195        )
196        .map_err(io::Error::other)?;
197        self.inner.write_all(&self.block_out)?;
198        self.header_written = true;
199        Ok(())
200    }
201
202    fn flush_block(&mut self, last: bool) -> io::Result<()> {
203        if self.buffer.is_empty() && last {
204            self.block_out.clear();
205            block_encoder::encode_raw_block(&[], true, &mut self.block_out)
206                .map_err(io::Error::other)?;
207            self.inner.write_all(&self.block_out)?;
208            return Ok(());
209        }
210
211        if self.buffer.is_empty() {
212            return Ok(());
213        }
214
215        let chunk = core::mem::take(&mut self.buffer);
216        let seed_dict = self.first_block && self.dict.is_some();
217
218        let plen = self.window_buf.len();
219        self.window_buf.extend_from_slice(&chunk);
220
221        self.block_out.clear();
222        self.block_out.reserve(chunk.len() + 32);
223        if crate::skip_match_search(&self.params, chunk.len(), &chunk) {
224            block_encoder::encode_raw_block(&chunk, last, &mut self.block_out)
225                .map_err(io::Error::other)?;
226        } else {
227            if seed_dict {
228                match self.params.strategy {
229                    Strategy::Fast => {
230                        fast::prefill_hash_table(
231                            &self.window_buf,
232                            plen,
233                            self.params.hash_log,
234                            self.params.min_match,
235                            &mut self.hash_table,
236                        );
237                    }
238                    Strategy::DFast => {
239                        dfast::prefill_hash_tables(
240                            &self.window_buf,
241                            plen,
242                            self.params.hash_log,
243                            self.params.chain_log,
244                            self.params.min_match,
245                            &mut self.hash_table,
246                            &mut self.hash_long,
247                        );
248                    }
249                }
250            } else if plen == 0 {
251                self.hash_table.fill(0);
252                if !self.hash_long.is_empty() {
253                    self.hash_long.fill(0);
254                }
255            }
256
257            #[cfg(feature = "ldm")]
258            let used_ldm = if let Some(ref mut ldm) = self.ldm_state {
259                ldm.compress_block(
260                    &self.window_buf,
261                    plen,
262                    self.window_buf.len(),
263                    1usize << self.params.window_log,
264                    &self.params,
265                    &self.rep_offsets,
266                    &mut self.hash_table,
267                    &mut self.hash_long,
268                    &mut self.sequences,
269                );
270                true
271            } else {
272                false
273            };
274            #[cfg(not(feature = "ldm"))]
275            let used_ldm = false;
276
277            if !used_ldm {
278                match self.params.strategy {
279                    Strategy::Fast => {
280                        fast::compress_fast_block(
281                            &self.window_buf,
282                            plen,
283                            self.window_buf.len(),
284                            &self.params,
285                            &self.rep_offsets,
286                            &mut self.hash_table,
287                            &mut self.sequences,
288                        );
289                    }
290                    Strategy::DFast => {
291                        dfast::compress_dfast_block(
292                            &self.window_buf,
293                            plen,
294                            self.window_buf.len(),
295                            &self.params,
296                            &self.rep_offsets,
297                            &mut self.hash_table,
298                            &mut self.hash_long,
299                            &mut self.sequences,
300                        );
301                    }
302                }
303            }
304
305            if self.params.force_raw_literals {
306                block_encoder::encode_compressed_block_raw(
307                    &chunk,
308                    &self.sequences,
309                    &mut self.rep_offsets,
310                    last,
311                    &mut self.block_out,
312                    &mut self.workspace,
313                )
314                .map_err(io::Error::other)?;
315            } else {
316                block_encoder::encode_compressed_block(
317                    &chunk,
318                    &self.sequences,
319                    &mut self.rep_offsets,
320                    last,
321                    &mut self.block_out,
322                    &mut self.workspace,
323                    strategy::block_policy(&self.params, chunk.len()),
324                )
325                .map_err(io::Error::other)?;
326            }
327        }
328
329        let window_size = 1usize << self.params.window_log;
330        if self.window_buf.len() > window_size * 2 {
331            let shift = self.window_buf.len() - window_size;
332            reduce_hash_table(&mut self.hash_table, shift as u32);
333            if !self.hash_long.is_empty() {
334                reduce_hash_table(&mut self.hash_long, shift as u32);
335            }
336            #[cfg(feature = "ldm")]
337            if let Some(ref mut ldm) = self.ldm_state {
338                ldm.reduce_positions(shift as u32);
339            }
340            self.window_buf.copy_within(shift.., 0);
341            self.window_buf.truncate(window_size);
342        }
343
344        self.first_block = false;
345        self.inner.write_all(&self.block_out)?;
346        Ok(())
347    }
348}
349
350fn alloc_hash_tables(params: &LevelParams) -> (Vec<u32>, Vec<u32>) {
351    match params.strategy {
352        Strategy::Fast => (vec![0u32; 1usize << params.hash_log], Vec::new()),
353        Strategy::DFast => (
354            vec![0u32; 1usize << params.chain_log],
355            vec![0u32; 1usize << params.hash_log],
356        ),
357    }
358}
359
360impl<W: Write> Write for FrameEncoder<W> {
361    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
362        if matches!(self.state, EncoderState::Failed) {
363            return Err(io::Error::other("encoder cannot be reused after an error"));
364        }
365        if matches!(self.state, EncoderState::Finished) {
366            return Err(io::Error::other("encoder already finished"));
367        }
368        // Any early return leaves the encoder failed, including partial writes.
369        self.state = EncoderState::Failed;
370
371        if !self.header_written {
372            self.write_header()?;
373        }
374
375        self.hasher.update(buf);
376
377        let block_size = MAX_BLOCK_SIZE.min(1usize << self.params.window_log);
378        let mut consumed = 0;
379        while consumed < buf.len() {
380            let space = block_size - self.buffer.len();
381            let n = space.min(buf.len() - consumed);
382            self.buffer.extend_from_slice(&buf[consumed..consumed + n]);
383            consumed += n;
384
385            if self.buffer.len() >= block_size {
386                self.flush_block(false)?;
387            }
388        }
389
390        self.state = EncoderState::Ready;
391        Ok(consumed)
392    }
393
394    fn flush(&mut self) -> io::Result<()> {
395        if matches!(self.state, EncoderState::Failed) {
396            return Err(io::Error::other("encoder cannot be reused after an error"));
397        }
398        let state = self.state;
399        self.state = EncoderState::Failed;
400        if !self.buffer.is_empty() {
401            self.flush_block(false)?;
402        }
403        self.inner.flush()?;
404        self.state = state;
405        Ok(())
406    }
407}