Skip to main content

zxc/
stdio.rs

1// SPDX-License-Identifier: BSD-3-Clause
2/*
3 * ZXC - High-performance lossless compression
4 *
5 * Copyright (c) Bertrand Lebonnois and contributors.
6 */
7
8//! [`std::io::Read`] / [`std::io::Write`] adapters over the push streaming API.
9//!
10//! Mirror of the wrapper exposed by the Go binding: turns a [`CStream`] /
11//! [`DStream`] pair into the standard streaming traits so ZXC can be plugged
12//! into pipelines that expect them.
13
14use std::io::{self, Read, Write};
15
16use crate::{CStream, CompressOptions, DStream, DecompressOptions, Error};
17
18/// Magic word identifying a ZXC file frame: little-endian `0x9CB02EF5`.
19const ZXC_MAGIC_LE: [u8; 4] = [0xF5, 0x2E, 0xB0, 0x9C];
20
21/// Reports whether `data` starts with the ZXC file magic word.
22///
23/// Useful for content-type sniffing in containers / object stores that need
24/// to decide which decoder to dispatch.
25/// The check is cheap and side-effect free; it does not validate the rest of
26/// the header or the footer.
27///
28/// # Example
29///
30/// ```rust
31/// use zxc::{compress, detect_zxc, Level};
32///
33/// let frame = compress(b"hello", Level::Default, None).unwrap();
34/// assert!(detect_zxc(&frame));
35/// assert!(!detect_zxc(b"not a zxc frame"));
36/// ```
37pub fn detect_zxc(data: &[u8]) -> bool {
38    data.len() >= 4 && data[..4] == ZXC_MAGIC_LE
39}
40
41// ---------------------------------------------------------------------------
42// Encoder
43// ---------------------------------------------------------------------------
44
45/// Streaming compressor implementing [`std::io::Write`].
46///
47/// Writes are forwarded through a [`CStream`] and the resulting ZXC frame is
48/// flushed to the inner writer. The frame is finalised (residual block, EOF
49/// marker, and footer) by [`Encoder::finish`]. [`Drop`] performs a best-effort
50/// finish but cannot surface errors — prefer [`Encoder::finish`] when error
51/// handling matters.
52///
53/// `Encoder` is single-threaded; one stream per writer.
54///
55/// # Example
56///
57/// ```rust,no_run
58/// use std::io::Write;
59/// use zxc::Encoder;
60///
61/// let sink = Vec::new();
62/// let mut enc = Encoder::new(sink).unwrap();
63/// enc.write_all(b"hello world").unwrap();
64/// let compressed: Vec<u8> = enc.finish().unwrap();
65/// ```
66pub struct Encoder<W: Write> {
67    inner: Option<W>,
68    cs: Option<CStream>,
69    out_buf: Vec<u8>,
70    /// `out_buf[out_pos..out_len]` is output the writer has not taken yet.
71    out_pos: usize,
72    out_len: usize,
73}
74
75impl<W: Write> Encoder<W> {
76    /// Creates an encoder with default compression options: no checksum.
77    pub fn new(writer: W) -> Result<Self, Error> {
78        Self::with_options(writer, None)
79    }
80
81    /// Creates an encoder honouring `opts`. `opts.seekable` is ignored (push
82    /// API is single-threaded and never emits a seek table).
83    pub fn with_options(writer: W, opts: Option<&CompressOptions>) -> Result<Self, Error> {
84        let cs = CStream::new(opts)?;
85        let cap = cs.out_size();
86        Ok(Self {
87            inner: Some(writer),
88            cs: Some(cs),
89            out_buf: vec![0u8; cap],
90            out_pos: 0,
91            out_len: 0,
92        })
93    }
94
95    /// Finalises the frame, drains the inner writer, and returns it.
96    ///
97    /// After this call the encoder is consumed; no further writes are
98    /// possible.
99    pub fn finish(mut self) -> io::Result<W> {
100        if let Err(e) = self.do_finish() {
101            // The caller gets no way to retry: Drop must not write to this writer again.
102            self.cs = None;
103            self.out_pos = self.out_len;
104            return Err(e);
105        }
106        Ok(self.inner.take().expect("inner writer present"))
107    }
108
109    /// Returns a reference to the underlying writer.
110    pub fn get_ref(&self) -> &W {
111        self.inner.as_ref().expect("encoder not finished")
112    }
113
114    /// Returns a mutable reference to the underlying writer.
115    pub fn get_mut(&mut self) -> &mut W {
116        self.inner.as_mut().expect("encoder not finished")
117    }
118
119    fn drain(&mut self) -> io::Result<()> {
120        while self.out_pos < self.out_len {
121            let Some(w) = self.inner.as_mut() else {
122                return Ok(());
123            };
124            match w.write(&self.out_buf[self.out_pos..self.out_len]) {
125                Ok(0) => return Err(io::ErrorKind::WriteZero.into()),
126                Ok(n) => self.out_pos += n,
127                Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
128                Err(e) => return Err(e),
129            }
130        }
131        Ok(())
132    }
133
134    fn do_finish(&mut self) -> io::Result<()> {
135        loop {
136            self.drain()?;
137            let Some(cs) = self.cs.as_mut() else {
138                return Ok(());
139            };
140            let p = cs.end(&mut self.out_buf).map_err(map_err)?;
141            (self.out_pos, self.out_len) = (0, p.produced);
142            if p.pending == 0 {
143                self.cs = None;
144            }
145        }
146    }
147}
148
149impl<W: Write> Write for Encoder<W> {
150    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
151        if buf.is_empty() {
152            return Ok(0);
153        }
154        self.drain()?;
155
156        let mut total = 0;
157        while total < buf.len() {
158            let cs = self
159                .cs
160                .as_mut()
161                .ok_or_else(|| io::Error::other("encoder finished"))?;
162            let p = cs
163                .compress(&buf[total..], &mut self.out_buf)
164                .map_err(map_err)?;
165            total += p.consumed;
166            (self.out_pos, self.out_len) = (0, p.produced);
167            if let Err(e) = self.drain() {
168                // Err would mean nothing was consumed. The output is kept, so the
169                // next call retries it and reports a lasting failure.
170                return if total > 0 { Ok(total) } else { Err(e) };
171            }
172            if p.consumed == 0 && p.produced == 0 && p.pending == 0 {
173                // No forward progress is possible — bail out to avoid spin.
174                break;
175            }
176        }
177        Ok(total)
178    }
179
180    /// Flush is a no-op on the inner writer side: the ZXC frame is only valid
181    /// once finalised, so partial flushes would produce a corrupted frame.
182    /// Use [`Encoder::finish`] to complete the frame.
183    fn flush(&mut self) -> io::Result<()> {
184        self.drain()?;
185        if let Some(w) = self.inner.as_mut() {
186            w.flush()
187        } else {
188            Ok(())
189        }
190    }
191}
192
193impl<W: Write> Drop for Encoder<W> {
194    fn drop(&mut self) {
195        // Best-effort: any error is silently swallowed because Drop cannot
196        // surface failure. Callers who care about errors must call finish().
197        let _ = self.do_finish();
198    }
199}
200
201// ---------------------------------------------------------------------------
202// Decoder
203// ---------------------------------------------------------------------------
204
205/// Streaming decompressor implementing [`std::io::Read`].
206///
207/// Pulls compressed bytes from the inner reader and yields decompressed
208/// bytes. Reads until the inner reader ends, so concatenated archives
209/// decode as one stream; on a source that stays open after one frame,
210/// set [`Decoder::multistream`] to `false`. Returns
211/// [`io::ErrorKind::UnexpectedEof`] if the inner reader is drained before the
212/// ZXC footer is reached.
213///
214/// `Decoder` is single-threaded; one stream per reader.
215///
216/// # Example
217///
218/// ```rust,no_run
219/// use std::io::Read;
220/// use zxc::Decoder;
221///
222/// let frame: &[u8] = &[]; // a ZXC frame
223/// let mut dec = Decoder::new(frame).unwrap();
224/// let mut buf = Vec::new();
225/// dec.read_to_end(&mut buf).unwrap();
226/// ```
227pub struct Decoder<R: Read> {
228    inner: R,
229    ds: DStream,
230    in_buf: Vec<u8>,
231    in_pos: usize,
232    in_len: usize,
233    eof: bool,
234    multistream: bool,
235}
236
237impl<R: Read> Decoder<R> {
238    /// Creates a decoder with default decompression options: checksums are
239    /// verified when a frame carries them.
240    pub fn new(reader: R) -> Result<Self, Error> {
241        Self::with_options(reader, None)
242    }
243
244    /// Creates a decoder honouring `opts`.
245    pub fn with_options(reader: R, opts: Option<&DecompressOptions>) -> Result<Self, Error> {
246        let ds = DStream::new(opts)?;
247        let cap = ds.in_size();
248        Ok(Self {
249            inner: reader,
250            ds,
251            in_buf: vec![0u8; cap],
252            in_pos: 0,
253            in_len: 0,
254            eof: false,
255            multistream: true,
256        })
257    }
258
259    /// Decodes concatenated archives as one stream (`true`, the default), or
260    /// stops after the first frame, as on a source that stays open after it.
261    /// Bytes read ahead past that frame are then lost to the caller.
262    pub fn multistream(mut self, enabled: bool) -> Self {
263        self.multistream = enabled;
264        self
265    }
266
267    /// Returns a reference to the underlying reader.
268    pub fn get_ref(&self) -> &R {
269        &self.inner
270    }
271
272    /// Returns a mutable reference to the underlying reader.
273    pub fn get_mut(&mut self) -> &mut R {
274        &mut self.inner
275    }
276
277    /// Consumes the decoder and returns the inner reader.
278    pub fn into_inner(self) -> R {
279        self.inner
280    }
281
282    /// Reports whether the input so far ends on a validated footer.
283    pub fn finished(&self) -> bool {
284        self.ds.finished()
285    }
286}
287
288impl<R: Read> Read for Decoder<R> {
289    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
290        if buf.is_empty() {
291            return Ok(0);
292        }
293        // Stop when src ends, or after the first frame without multistream.
294        loop {
295            if !self.multistream && self.ds.finished() {
296                return Ok(0);
297            }
298            // Try to decompress whatever is currently buffered (or drain mode
299            // when src is at EOF).
300            if self.in_pos < self.in_len || self.eof {
301                let p = self
302                    .ds
303                    .decompress(&self.in_buf[self.in_pos..self.in_len], buf)
304                    .map_err(map_err)?;
305                self.in_pos += p.consumed;
306                if p.produced > 0 {
307                    return Ok(p.produced);
308                }
309                if p.consumed == 0 {
310                    if self.eof {
311                        if self.ds.finished() {
312                            return Ok(0);
313                        }
314                        return Err(io::Error::new(
315                            io::ErrorKind::UnexpectedEof,
316                            "zxc: input drained before footer",
317                        ));
318                    }
319                    // fall through to refill
320                } else {
321                    // consumed > 0 but no output — loop and retry.
322                    continue;
323                }
324            }
325
326            // Refill: shift consumed bytes out, then read more from src.
327            if self.in_pos > 0 {
328                self.in_buf.copy_within(self.in_pos..self.in_len, 0);
329                self.in_len -= self.in_pos;
330                self.in_pos = 0;
331            }
332            let n = self.inner.read(&mut self.in_buf[self.in_len..])?;
333            if n == 0 {
334                self.eof = true;
335            } else {
336                self.in_len += n;
337            }
338        }
339    }
340}
341
342// ---------------------------------------------------------------------------
343// Error mapping
344// ---------------------------------------------------------------------------
345
346fn map_err(e: Error) -> io::Error {
347    io::Error::other(e)
348}
349
350// ---------------------------------------------------------------------------
351// Tests
352// ---------------------------------------------------------------------------
353
354#[cfg(test)]
355mod tests {
356    use super::*;
357    use crate::Level;
358    use crate::{compress, decompress};
359    use std::io::{Cursor, Read, Write};
360    use std::sync::Arc;
361    use std::sync::atomic::{AtomicUsize, Ordering};
362
363    fn roundtrip(data: &[u8]) -> Vec<u8> {
364        let mut enc = Encoder::new(Vec::new()).expect("encoder");
365        enc.write_all(data).expect("write");
366        let frame = enc.finish().expect("finish");
367
368        let mut dec = Decoder::new(Cursor::new(frame)).expect("decoder");
369        let mut out = Vec::new();
370        dec.read_to_end(&mut out).expect("read_to_end");
371        out
372    }
373
374    #[test]
375    fn encoder_decoder_small_roundtrip() {
376        let data = b"Hello std::io::Read / std::io::Write bridge over ZXC.";
377        assert_eq!(roundtrip(data), data);
378    }
379
380    #[test]
381    fn encoder_decoder_large_roundtrip() {
382        let data: Vec<u8> = (0..2 * 1024 * 1024)
383            .map(|i| ((i * 13) % 251) as u8)
384            .collect();
385        assert_eq!(roundtrip(&data), data);
386    }
387
388    #[test]
389    fn encoder_many_small_writes() {
390        let mut enc = Encoder::new(Vec::new()).unwrap();
391        let mut want: Vec<u8> = Vec::with_capacity(64 * 1024);
392        for i in 0u32..4096 {
393            let chunk = [i as u8, (i >> 8) as u8, (i ^ 0x5A) as u8];
394            want.extend_from_slice(&chunk);
395            enc.write_all(&chunk).unwrap();
396        }
397        let frame = enc.finish().unwrap();
398        let mut dec = Decoder::new(Cursor::new(frame)).unwrap();
399        let mut got = Vec::new();
400        dec.read_to_end(&mut got).unwrap();
401        assert_eq!(got, want);
402    }
403
404    #[test]
405    fn encoder_decoder_with_checksum() {
406        use crate::{CompressOptions, DecompressOptions};
407        let data: Vec<u8> = (0..32 * 1024).map(|i| i as u8).collect();
408        let copts = CompressOptions {
409            checksum: true,
410            ..Default::default()
411        };
412        let dopts = DecompressOptions {
413            verify_checksum: Some(true),
414            ..Default::default()
415        };
416        let mut enc = Encoder::with_options(Vec::new(), Some(&copts)).unwrap();
417        enc.write_all(&data).unwrap();
418        let frame = enc.finish().unwrap();
419        let mut dec = Decoder::with_options(Cursor::new(frame), Some(&dopts)).unwrap();
420        let mut got = Vec::new();
421        dec.read_to_end(&mut got).unwrap();
422        assert_eq!(got, data);
423    }
424
425    #[test]
426    fn drop_finishes_frame() {
427        // Encoder is dropped without an explicit finish() — the resulting
428        // frame must still be well-formed (best-effort flush in Drop).
429        let mut sink: Vec<u8> = Vec::new();
430        {
431            let mut enc = Encoder::new(&mut sink).unwrap();
432            enc.write_all(b"drop-flush").unwrap();
433        }
434        let mut dec = Decoder::new(Cursor::new(sink)).unwrap();
435        let mut got = Vec::new();
436        dec.read_to_end(&mut got).unwrap();
437        assert_eq!(got, b"drop-flush");
438    }
439
440    /// Fails once on the `fail_at`-th write, and takes at most `max_write` bytes.
441    struct FlakySink {
442        data: Vec<u8>,
443        writes: Arc<AtomicUsize>,
444        fail_at: usize,
445        max_write: usize,
446    }
447    impl FlakySink {
448        fn new(fail_at: usize, max_write: usize) -> Self {
449            Self {
450                data: Vec::new(),
451                writes: Arc::default(),
452                fail_at,
453                max_write,
454            }
455        }
456    }
457    impl Write for FlakySink {
458        fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
459            if self.writes.fetch_add(1, Ordering::Relaxed) == self.fail_at {
460                return Err(io::ErrorKind::WouldBlock.into());
461            }
462            let n = buf.len().min(self.max_write);
463            self.data.extend_from_slice(&buf[..n]);
464            Ok(n)
465        }
466        fn flush(&mut self) -> io::Result<()> {
467            Ok(())
468        }
469    }
470
471    fn noise(len: usize) -> Vec<u8> {
472        let mut rng: u32 = 0x2E5B_9A17;
473        (0..len)
474            .map(|_| {
475                rng = rng.wrapping_mul(1_103_515_245).wrapping_add(12_345);
476                (rng >> 16) as u8
477            })
478            .collect()
479    }
480
481    #[test]
482    fn encoder_keeps_output_across_a_sink_error() {
483        let payload = noise(600 * 1024);
484        // The first write carries the header and the first block.
485        for (fail_at, max_write) in [(0, usize::MAX), (0, 4096), (1, 4096), (9, 4096)] {
486            let mut enc = Encoder::new(FlakySink::new(fail_at, max_write)).unwrap();
487            let mut done = 0;
488            let mut errors = 0;
489            while done < payload.len() {
490                match enc.write(&payload[done..]) {
491                    Ok(n) => done += n,
492                    Err(e) => {
493                        assert_eq!(e.kind(), io::ErrorKind::WouldBlock);
494                        errors += 1;
495                    }
496                }
497            }
498            assert!(errors <= 1);
499            enc.flush().ok();
500            let sink = enc.finish().unwrap();
501            assert_eq!(decompress(&sink.data).unwrap(), payload);
502        }
503    }
504
505    #[test]
506    fn failed_finish_stops_writing() {
507        // The header goes out on the first write, the rest fails in finish().
508        let sink = FlakySink::new(1, usize::MAX);
509        let writes = sink.writes.clone();
510        let mut enc = Encoder::new(sink).unwrap();
511        enc.write_all(b"never reaches the sink").unwrap();
512        assert_eq!(writes.load(Ordering::Relaxed), 1);
513        assert!(enc.finish().is_err());
514        assert_eq!(writes.load(Ordering::Relaxed), 2, "Drop wrote again");
515    }
516
517    #[test]
518    fn lasting_sink_failure_is_reported() {
519        struct Dead;
520        impl Write for Dead {
521            fn write(&mut self, _: &[u8]) -> io::Result<usize> {
522                Err(io::Error::other("dead"))
523            }
524            fn flush(&mut self) -> io::Result<()> {
525                Ok(())
526            }
527        }
528        let mut enc = Encoder::new(Dead).unwrap();
529        let chunk = noise(1 << 20);
530        // Output of a call that consumed its input fails on the next one.
531        assert!((0..2).try_for_each(|_| enc.write_all(&chunk)).is_err());
532        assert!(enc.flush().is_err());
533
534        let mut enc = Encoder::new(Dead).unwrap();
535        enc.write_all(b"small").unwrap();
536        assert!(enc.finish().is_err());
537    }
538
539    /// Hands out one byte per read, so a read ends on every frame boundary.
540    struct OneByte<'a>(&'a [u8]);
541    impl Read for OneByte<'_> {
542        fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
543            if self.0.is_empty() || buf.is_empty() {
544                return Ok(0);
545            }
546            buf[0] = self.0[0];
547            self.0 = &self.0[1..];
548            Ok(1)
549        }
550    }
551
552    #[test]
553    fn decoder_concatenated_frames() {
554        let a = b"first frame ".repeat(5000);
555        let b = b"second frame ".repeat(7000);
556        let fa = compress(&a, Level::Default, None).unwrap();
557        let joined = [
558            fa.clone(),
559            compress(&b, Level::Default, Some(true)).unwrap(),
560        ]
561        .concat();
562        let mut got = Vec::new();
563        Decoder::new(OneByte(&joined))
564            .unwrap()
565            .read_to_end(&mut got)
566            .unwrap();
567        assert_eq!(got, [a.clone(), b].concat());
568
569        // Without multistream, the decoder stops after the first frame.
570        let mut first = Vec::new();
571        Decoder::new(OneByte(&joined))
572            .unwrap()
573            .multistream(false)
574            .read_to_end(&mut first)
575            .unwrap();
576        assert_eq!(first, a);
577
578        let junk = [fa, b"junk".to_vec()].concat();
579        let mut dec = Decoder::new(Cursor::new(junk)).unwrap();
580        assert!(dec.read_to_end(&mut Vec::new()).is_err());
581    }
582
583    #[test]
584    fn decoder_truncated_frame_errors() {
585        let frame = compress(&vec![b'A'; 32 * 1024], Level::Default, None).unwrap();
586        let truncated = &frame[..frame.len() / 2];
587        let mut dec = Decoder::new(Cursor::new(truncated)).unwrap();
588        let mut got = Vec::new();
589        let err = dec.read_to_end(&mut got).unwrap_err();
590        assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof);
591    }
592
593    #[test]
594    fn detect_zxc_basic() {
595        let frame = compress(b"sniff me", Level::Default, None).unwrap();
596        assert!(detect_zxc(&frame));
597
598        let mut enc = Encoder::new(Vec::new()).unwrap();
599        enc.write_all(b"hi").unwrap();
600        let frame = enc.finish().unwrap();
601        assert!(detect_zxc(&frame));
602
603        assert!(!detect_zxc(&[]));
604        assert!(!detect_zxc(&[0xF5, 0x2E, 0xB0])); // 3 bytes, too short
605        assert!(!detect_zxc(&[0; 4]));
606        assert!(!detect_zxc(b"not a zxc frame at all"));
607    }
608}