Skip to main content

zrip_encode/
push.rs

1//! Push-style (sans-io) encoding: the caller feeds input chunks and drains
2//! frame bytes into its own buffers.
3#![forbid(unsafe_code)]
4
5use alloc::vec::Vec;
6
7use crate::frame_state::EncodeState;
8use crate::strategy::{self, LevelParams};
9use zrip_core::dict::Dictionary;
10use zrip_core::error::CompressError;
11pub use zrip_core::push::{Progress, Status};
12
13/// What [`PushEncoder::encode`] should do once it has consumed the input.
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
15pub enum EndOp {
16    /// Keep buffering; emit blocks only as they fill.
17    Continue,
18    /// End the current block so everything consumed so far can be decoded.
19    Flush,
20    /// End the frame: last block and content checksum.
21    End,
22}
23
24/// Push-style zstd compressor.
25///
26/// Feed input in chunks of any size with [`encode`](Self::encode) and drain
27/// frame bytes into buffers of any size. No I/O, no `std`; it needs only the
28/// `alloc` feature. `FrameEncoder` is the `std::io::Write` counterpart.
29///
30/// Frames carry a content checksum. A frame that ends before any of it was
31/// written, such as all input passed with [`EndOp::End`], is encoded in one
32/// pass like `zrip::compress`: same bytes, content size in the header, and
33/// tables sized to the input. Longer streams have no content size. Hash
34/// tables, window and block buffers are allocated on first use and reused
35/// for every block and every frame.
36///
37/// After an error, every call returns that error until
38/// [`reset`](Self::reset).
39///
40/// ```
41/// use zrip::push::{EndOp, PushEncoder, Status};
42///
43/// let data = b"hello, push world! ".repeat(1000);
44/// let mut enc = PushEncoder::new(1).unwrap();
45/// let mut frame = Vec::new();
46/// let mut buf = [0u8; 256];
47/// for chunk in data.chunks(100) {
48///     let mut input = chunk;
49///     while !input.is_empty() {
50///         let p = enc.encode(input, &mut buf, EndOp::Continue).unwrap();
51///         frame.extend_from_slice(&buf[..p.produced]);
52///         input = &input[p.consumed..];
53///     }
54/// }
55/// loop {
56///     let p = enc.encode(&[], &mut buf, EndOp::End).unwrap();
57///     frame.extend_from_slice(&buf[..p.produced]);
58///     if p.status == Status::Finished {
59///         break;
60///     }
61/// }
62/// assert_eq!(zrip::decompress(&frame).unwrap(), data);
63/// ```
64pub struct PushEncoder {
65    frame: EncodeState,
66    /// Level and options, for frames encoded in one pass.
67    level: i32,
68    opts: Option<strategy::Options>,
69    /// Frame bytes in `pending[pending_pos..]` are not delivered yet.
70    pending: Vec<u8>,
71    pending_pos: usize,
72    header_written: bool,
73    /// The frame's last bytes are in `pending`; report
74    /// [`Status::Finished`] once they are delivered.
75    frame_done: bool,
76    error: Option<CompressError>,
77}
78
79impl core::fmt::Debug for PushEncoder {
80    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
81        f.debug_struct("PushEncoder")
82            .field("pending_output", &(self.pending.len() - self.pending_pos))
83            .field("error", &self.error)
84            .finish_non_exhaustive()
85    }
86}
87
88impl PushEncoder {
89    /// Creates an encoder at the given level (-7..=4; -8 runs as -7).
90    pub fn new(level: i32) -> Result<Self, CompressError> {
91        let params = strategy::level_params(level).ok_or(CompressError::InvalidLevel(level))?;
92        Ok(Self::from_params(level, None, params, None))
93    }
94
95    /// Creates an encoder with [`Options`](strategy::Options) for large
96    /// windows and/or LDM.
97    pub fn with_options(level: i32, opts: &strategy::Options) -> Result<Self, CompressError> {
98        let mut params = strategy::level_params(level).ok_or(CompressError::InvalidLevel(level))?;
99        strategy::apply_options(&mut params, opts);
100        Ok(Self::from_params(level, Some(opts.clone()), params, None))
101    }
102
103    /// Creates an encoder with a dictionary.
104    pub fn with_dict(level: i32, dict: Dictionary) -> Result<Self, CompressError> {
105        let params = strategy::level_params(level).ok_or(CompressError::InvalidLevel(level))?;
106        Ok(Self::from_params(level, None, params, Some(dict)))
107    }
108
109    fn from_params(
110        level: i32,
111        opts: Option<strategy::Options>,
112        params: LevelParams,
113        dict: Option<Dictionary>,
114    ) -> Self {
115        Self {
116            frame: EncodeState::new(params, dict),
117            level,
118            opts,
119            pending: Vec::new(),
120            pending_pos: 0,
121            header_written: false,
122            frame_done: false,
123            error: None,
124        }
125    }
126
127    /// Consumes input and writes frame bytes into `output`.
128    ///
129    /// - [`EndOp::Continue`]: returns [`Status::NeedsInput`] once all input
130    ///   is consumed and every full block is delivered.
131    /// - [`EndOp::Flush`]: also ends the current block; returns
132    ///   [`Status::NeedsInput`] once it is delivered.
133    /// - [`EndOp::End`]: also ends the frame; returns [`Status::Finished`]
134    ///   once its last byte is delivered. The next call starts a new frame.
135    ///
136    /// [`Status::NeedsOutput`] means `output` filled up first: call again
137    /// with the rest of the input (`consumed` tells how much was taken) and
138    /// the same `op`.
139    pub fn encode(
140        &mut self,
141        input: &[u8],
142        output: &mut [u8],
143        op: EndOp,
144    ) -> Result<Progress, CompressError> {
145        if let Some(e) = &self.error {
146            return Err(e.clone());
147        }
148        let mut consumed = 0;
149        let mut produced = 0;
150        let result = self.run(input, output, op, &mut consumed, &mut produced);
151        match result {
152            Ok(status) => Ok(Progress {
153                consumed,
154                produced,
155                status,
156            }),
157            Err(e) => {
158                self.error = Some(e.clone());
159                Err(e)
160            }
161        }
162    }
163
164    /// Encodes all of `input` and appends the frame bytes to `output`.
165    /// With [`EndOp::End`], `output` then ends with a complete frame.
166    ///
167    /// Bytes still pending from [`encode`](Self::encode) go out first. If
168    /// they end a frame, `input` starts the next one.
169    pub fn encode_to_vec(
170        &mut self,
171        input: &[u8],
172        output: &mut Vec<u8>,
173        op: EndOp,
174    ) -> Result<(), CompressError> {
175        if let Some(e) = &self.error {
176            return Err(e.clone());
177        }
178        output.extend_from_slice(&self.pending[self.pending_pos..]);
179        self.pending.clear();
180        self.pending_pos = 0;
181        if self.frame_done {
182            self.end_frame();
183            // The call only finished the pending frame.
184            if input.is_empty() {
185                return Ok(());
186            }
187        }
188        let mut consumed = 0;
189        loop {
190            if self.frame_done {
191                self.end_frame();
192                return Ok(());
193            }
194            match self.advance(input, &mut consumed, op, output) {
195                Ok(true) => {}
196                Ok(false) => return Ok(()),
197                Err(e) => {
198                    self.error = Some(e.clone());
199                    return Err(e);
200                }
201            }
202        }
203    }
204
205    /// Whether frame bytes are still waiting to be delivered.
206    pub fn has_pending_output(&self) -> bool {
207        self.pending_pos < self.pending.len()
208    }
209
210    /// Discards the current frame and any error, keeping allocations. The
211    /// next call starts a new frame.
212    pub fn reset(&mut self) {
213        self.frame.reset();
214        self.pending.clear();
215        self.pending_pos = 0;
216        self.header_written = false;
217        self.frame_done = false;
218        self.error = None;
219    }
220
221    fn run(
222        &mut self,
223        input: &[u8],
224        output: &mut [u8],
225        op: EndOp,
226        consumed: &mut usize,
227        produced: &mut usize,
228    ) -> Result<Status, CompressError> {
229        loop {
230            if self.has_pending_output() {
231                let src = &self.pending[self.pending_pos..];
232                let n = src.len().min(output.len() - *produced);
233                output[*produced..*produced + n].copy_from_slice(&src[..n]);
234                *produced += n;
235                self.pending_pos += n;
236                if self.has_pending_output() {
237                    return Ok(Status::NeedsOutput);
238                }
239            }
240            self.pending.clear();
241            self.pending_pos = 0;
242
243            if self.frame_done {
244                self.end_frame();
245                return Ok(Status::Finished);
246            }
247
248            let mut pending = core::mem::take(&mut self.pending);
249            let advanced = self.advance(input, consumed, op, &mut pending);
250            self.pending = pending;
251            if !advanced? {
252                return Ok(Status::NeedsInput);
253            }
254        }
255    }
256
257    /// Takes input and appends the next block to `out` once one is due,
258    /// with the frame header before the first block and the checksum after
259    /// the last. Returns false if no block is due yet.
260    fn advance(
261        &mut self,
262        input: &[u8],
263        consumed: &mut usize,
264        op: EndOp,
265        out: &mut Vec<u8>,
266    ) -> Result<bool, CompressError> {
267        // The frame ends before any of it went out, so its size is known:
268        // encode it in one pass, as C zstd does. Tables are then sized to
269        // the input, and the header records the content size.
270        if op == EndOp::End && !self.header_written {
271            let rest = &input[*consumed..];
272            if self.frame.buffered() == 0 {
273                self.one_pass(rest, out)?;
274                *consumed = input.len();
275                self.frame_done = true;
276                return Ok(true);
277            }
278            if self.frame.buffered() + rest.len() <= self.frame.block_size() {
279                *consumed += self.frame.take_input(rest);
280                self.one_pass(self.frame.buffered_input(), out)?;
281                self.frame_done = true;
282                return Ok(true);
283            }
284        }
285
286        *consumed += self.frame.take_input(&input[*consumed..]);
287        let more_input = *consumed < input.len();
288        let full = self.frame.buffered() == self.frame.block_size();
289
290        // A full block waits for the next byte, so `End` can still mark it
291        // last instead of writing an empty last block after it.
292        let last = !more_input && op == EndOp::End;
293        let due = (full && more_input)
294            || last
295            || (!more_input && op == EndOp::Flush && self.frame.buffered() > 0);
296        if !due {
297            return Ok(false);
298        }
299
300        if !self.header_written {
301            self.frame.write_header(out)?;
302            self.header_written = true;
303        }
304        self.frame.encode_block(last, out)?;
305        if last {
306            self.frame.write_checksum(out);
307            self.frame_done = true;
308        }
309        Ok(true)
310    }
311
312    /// Appends `src` as one whole frame, like the one-shot API.
313    fn one_pass(&self, src: &[u8], out: &mut Vec<u8>) -> Result<(), CompressError> {
314        if let Some(dict) = self.frame.dict() {
315            return crate::compress_with_dict_to(src, self.level, dict, out);
316        }
317        let params = match &self.opts {
318            Some(opts) => crate::params_with_options(self.level, src.len(), opts)?,
319            None => strategy::level_params_for_size(self.level, src.len())
320                .ok_or(CompressError::InvalidLevel(self.level))?,
321        };
322        crate::compress_frame(src, &params, out)
323    }
324
325    fn end_frame(&mut self) {
326        self.frame_done = false;
327        self.header_written = false;
328        self.frame.reset();
329    }
330}