Skip to main content

zrip_encode/
push.rs

1//! Push-style (sans-io) encoding: the caller feeds input chunks and drains
2//! frame bytes into its own buffers.
3#![forbid(unsafe_code)]
4
5use alloc::vec::Vec;
6use core::mem::MaybeUninit;
7
8use crate::frame_state::EncodeState;
9use crate::strategy::{self, LevelParams};
10use zrip_core::dict::Dictionary;
11use zrip_core::error::CompressError;
12pub use zrip_core::push::{Progress, Status};
13
14/// A buffer that frame bytes are copied into.
15trait Output {
16    /// Copies as much of `data` as fits from `pos` on and returns how much.
17    fn put(&mut self, pos: usize, data: &[u8]) -> usize;
18}
19
20impl Output for [u8] {
21    fn put(&mut self, pos: usize, data: &[u8]) -> usize {
22        let n = data.len().min(self.len() - pos);
23        self[pos..pos + n].copy_from_slice(&data[..n]);
24        n
25    }
26}
27
28impl Output for [MaybeUninit<u8>] {
29    fn put(&mut self, pos: usize, data: &[u8]) -> usize {
30        let n = data.len().min(self.len() - pos);
31        // Compiles to a memcpy. `write_copy_of_slice` needs Rust 1.93.
32        for (dst, &src) in self[pos..pos + n].iter_mut().zip(data) {
33            dst.write(src);
34        }
35        n
36    }
37}
38
39/// What [`PushEncoder::encode`] should do once it has consumed the input.
40#[derive(Debug, Clone, Copy, PartialEq, Eq)]
41pub enum EndOp {
42    /// Keep buffering; emit blocks only as they fill.
43    Continue,
44    /// End the current block so everything consumed so far can be decoded.
45    Flush,
46    /// End the frame: last block and content checksum.
47    End,
48}
49
50/// Push-style zstd compressor.
51///
52/// Feed input in chunks of any size with [`encode`](Self::encode) and drain
53/// frame bytes into buffers of any size. No I/O, no `std`; it needs only the
54/// `alloc` feature. `FrameEncoder` is the `std::io::Write` counterpart.
55///
56/// Frames carry a content checksum. A frame that ends before any of it was
57/// written, such as all input passed with [`EndOp::End`], is encoded in one
58/// pass like `zrip::compress`: same bytes, content size in the header, and
59/// tables sized to the input. Longer streams have no content size. Hash
60/// tables, window and block buffers are allocated on first use and reused
61/// for every block and every frame.
62///
63/// After an error, every call returns that error until
64/// [`reset`](Self::reset).
65///
66/// ```
67/// use zrip::push::{EndOp, PushEncoder, Status};
68///
69/// let data = b"hello, push world! ".repeat(1000);
70/// let mut enc = PushEncoder::new(1).unwrap();
71/// let mut frame = Vec::new();
72/// let mut buf = [0u8; 256];
73/// for chunk in data.chunks(100) {
74///     let mut input = chunk;
75///     while !input.is_empty() {
76///         let p = enc.encode(input, &mut buf, EndOp::Continue).unwrap();
77///         frame.extend_from_slice(&buf[..p.produced]);
78///         input = &input[p.consumed..];
79///     }
80/// }
81/// loop {
82///     let p = enc.encode(&[], &mut buf, EndOp::End).unwrap();
83///     frame.extend_from_slice(&buf[..p.produced]);
84///     if p.status == Status::Finished {
85///         break;
86///     }
87/// }
88/// assert_eq!(zrip::decompress(&frame).unwrap(), data);
89/// ```
90pub struct PushEncoder {
91    frame: EncodeState,
92    /// Level and options, for frames encoded in one pass.
93    level: i32,
94    opts: Option<strategy::Options>,
95    /// Frame bytes in `pending[pending_pos..]` are not delivered yet.
96    pending: Vec<u8>,
97    pending_pos: usize,
98    header_written: bool,
99    /// The frame's last bytes are in `pending`; report
100    /// [`Status::Finished`] once they are delivered.
101    frame_done: bool,
102    error: Option<CompressError>,
103}
104
105impl core::fmt::Debug for PushEncoder {
106    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
107        f.debug_struct("PushEncoder")
108            .field("pending_output", &(self.pending.len() - self.pending_pos))
109            .field("error", &self.error)
110            .finish_non_exhaustive()
111    }
112}
113
114impl PushEncoder {
115    /// Creates an encoder at the given level (-7..=4; -8 runs as -7).
116    pub fn new(level: i32) -> Result<Self, CompressError> {
117        let params = strategy::level_params(level).ok_or(CompressError::InvalidLevel(level))?;
118        Ok(Self::from_params(level, None, params, None))
119    }
120
121    /// Creates an encoder with [`Options`](strategy::Options) for large
122    /// windows and/or LDM.
123    pub fn with_options(level: i32, opts: &strategy::Options) -> Result<Self, CompressError> {
124        let mut params = strategy::level_params(level).ok_or(CompressError::InvalidLevel(level))?;
125        strategy::apply_options(&mut params, opts);
126        Ok(Self::from_params(level, Some(opts.clone()), params, None))
127    }
128
129    /// Creates an encoder with a dictionary.
130    pub fn with_dict(level: i32, dict: Dictionary) -> Result<Self, CompressError> {
131        let params = strategy::level_params(level).ok_or(CompressError::InvalidLevel(level))?;
132        Ok(Self::from_params(level, None, params, Some(dict)))
133    }
134
135    fn from_params(
136        level: i32,
137        opts: Option<strategy::Options>,
138        params: LevelParams,
139        dict: Option<Dictionary>,
140    ) -> Self {
141        Self {
142            frame: EncodeState::new(params, dict),
143            level,
144            opts,
145            pending: Vec::new(),
146            pending_pos: 0,
147            header_written: false,
148            frame_done: false,
149            error: None,
150        }
151    }
152
153    /// Consumes input and writes frame bytes into `output`.
154    ///
155    /// - [`EndOp::Continue`]: returns [`Status::NeedsInput`] once all input
156    ///   is consumed and every full block is delivered.
157    /// - [`EndOp::Flush`]: also ends the current block; returns
158    ///   [`Status::NeedsInput`] once it is delivered.
159    /// - [`EndOp::End`]: also ends the frame; returns [`Status::Finished`]
160    ///   once its last byte is delivered. The next call starts a new frame.
161    ///
162    /// [`Status::NeedsOutput`] means `output` filled up first: call again
163    /// with the rest of the input (`consumed` tells how much was taken) and
164    /// the same `op`.
165    pub fn encode(
166        &mut self,
167        input: &[u8],
168        output: &mut [u8],
169        op: EndOp,
170    ) -> Result<Progress, CompressError> {
171        self.encode_with(input, output, op)
172    }
173
174    /// Like [`encode`](Self::encode), but `output` may be uninitialized,
175    /// so callers need not zero their buffers. On return, the first
176    /// `produced` bytes of `output` are initialized. Bytes past them are left
177    /// as they were, and no byte is ever made uninitialized, so callers may
178    /// rely on both in unsafe code, as when they track initialized capacity.
179    pub fn encode_uninit(
180        &mut self,
181        input: &[u8],
182        output: &mut [MaybeUninit<u8>],
183        op: EndOp,
184    ) -> Result<Progress, CompressError> {
185        self.encode_with(input, output, op)
186    }
187
188    fn encode_with<O: Output + ?Sized>(
189        &mut self,
190        input: &[u8],
191        output: &mut O,
192        op: EndOp,
193    ) -> Result<Progress, CompressError> {
194        if let Some(e) = &self.error {
195            return Err(e.clone());
196        }
197        let mut consumed = 0;
198        let mut produced = 0;
199        let result = self.run(input, output, op, &mut consumed, &mut produced);
200        match result {
201            Ok(status) => Ok(Progress {
202                consumed,
203                produced,
204                status,
205            }),
206            Err(e) => {
207                self.error = Some(e.clone());
208                Err(e)
209            }
210        }
211    }
212
213    /// Encodes all of `input` and appends the frame bytes to `output`.
214    /// With [`EndOp::End`], `output` then ends with a complete frame.
215    ///
216    /// Bytes still pending from [`encode`](Self::encode) go out first. If
217    /// they end a frame, `input` starts the next one.
218    pub fn encode_to_vec(
219        &mut self,
220        input: &[u8],
221        output: &mut Vec<u8>,
222        op: EndOp,
223    ) -> Result<(), CompressError> {
224        if let Some(e) = &self.error {
225            return Err(e.clone());
226        }
227        output.extend_from_slice(&self.pending[self.pending_pos..]);
228        self.pending.clear();
229        self.pending_pos = 0;
230        if self.frame_done {
231            self.end_frame();
232            // The call only finished the pending frame.
233            if input.is_empty() {
234                return Ok(());
235            }
236        }
237        let mut consumed = 0;
238        loop {
239            if self.frame_done {
240                self.end_frame();
241                return Ok(());
242            }
243            match self.advance(input, &mut consumed, op, output) {
244                Ok(true) => {}
245                Ok(false) => return Ok(()),
246                Err(e) => {
247                    self.error = Some(e.clone());
248                    return Err(e);
249                }
250            }
251        }
252    }
253
254    /// Whether frame bytes are still waiting to be delivered.
255    pub fn has_pending_output(&self) -> bool {
256        self.pending_pos < self.pending.len()
257    }
258
259    /// Discards the current frame and any error, keeping allocations. The
260    /// next call starts a new frame.
261    pub fn reset(&mut self) {
262        self.frame.reset();
263        self.pending.clear();
264        self.pending_pos = 0;
265        self.header_written = false;
266        self.frame_done = false;
267        self.error = None;
268    }
269
270    fn run<O: Output + ?Sized>(
271        &mut self,
272        input: &[u8],
273        output: &mut O,
274        op: EndOp,
275        consumed: &mut usize,
276        produced: &mut usize,
277    ) -> Result<Status, CompressError> {
278        loop {
279            if self.has_pending_output() {
280                let src = &self.pending[self.pending_pos..];
281                let n = output.put(*produced, src);
282                *produced += n;
283                self.pending_pos += n;
284                if self.has_pending_output() {
285                    return Ok(Status::NeedsOutput);
286                }
287            }
288            self.pending.clear();
289            self.pending_pos = 0;
290
291            if self.frame_done {
292                self.end_frame();
293                return Ok(Status::Finished);
294            }
295
296            let mut pending = core::mem::take(&mut self.pending);
297            let advanced = self.advance(input, consumed, op, &mut pending);
298            self.pending = pending;
299            if !advanced? {
300                return Ok(Status::NeedsInput);
301            }
302        }
303    }
304
305    /// Takes input and appends the next block to `out` once one is due,
306    /// with the frame header before the first block and the checksum after
307    /// the last. Returns false if no block is due yet.
308    fn advance(
309        &mut self,
310        input: &[u8],
311        consumed: &mut usize,
312        op: EndOp,
313        out: &mut Vec<u8>,
314    ) -> Result<bool, CompressError> {
315        // The frame ends before any of it went out, so its size is known:
316        // encode it in one pass, as C zstd does. Tables are then sized to
317        // the input, and the header records the content size.
318        if op == EndOp::End && !self.header_written {
319            let rest = &input[*consumed..];
320            if self.frame.buffered() == 0 {
321                self.one_pass(Some(rest), out)?;
322                *consumed = input.len();
323                self.frame_done = true;
324                return Ok(true);
325            }
326            if self.frame.buffered() + rest.len() <= self.frame.block_size() {
327                *consumed += self.frame.take_input(rest);
328                self.one_pass(None, out)?;
329                self.frame_done = true;
330                return Ok(true);
331            }
332        }
333
334        *consumed += self.frame.take_input(&input[*consumed..]);
335        let more_input = *consumed < input.len();
336        let full = self.frame.buffered() == self.frame.block_size();
337
338        // A full block waits for the next byte, so `End` can still mark it
339        // last instead of writing an empty last block after it.
340        let last = !more_input && op == EndOp::End;
341        let due = (full && more_input)
342            || last
343            || (!more_input && op == EndOp::Flush && self.frame.buffered() > 0);
344        if !due {
345            return Ok(false);
346        }
347
348        if !self.header_written {
349            self.frame.write_header(out)?;
350            self.header_written = true;
351        }
352        self.frame.encode_block(last, out)?;
353        if last {
354            self.frame.write_checksum(out);
355            self.frame_done = true;
356        }
357        Ok(true)
358    }
359
360    /// Appends `input`, or the buffered input if `None`, as one whole frame,
361    /// like the one-shot API but in the encoder's tables and buffers.
362    fn one_pass(&mut self, input: Option<&[u8]>, out: &mut Vec<u8>) -> Result<(), CompressError> {
363        let len = input.map_or_else(|| self.frame.buffered(), <[u8]>::len);
364        let params = match (self.frame.dict(), &self.opts) {
365            (Some(dict), _) => {
366                strategy::level_params_with_prefix(self.level, dict.content().len(), len)
367            }
368            (None, Some(opts)) => Some(crate::params_with_options(self.level, len, opts)?),
369            (None, None) => strategy::level_params_for_size(self.level, len),
370        }
371        .ok_or(CompressError::InvalidLevel(self.level))?;
372        self.frame.encode_one_pass(input, &params, out)
373    }
374
375    fn end_frame(&mut self) {
376        self.frame_done = false;
377        self.header_written = false;
378        self.frame.reset();
379    }
380}