exarch_core/formats/
compression.rs

1//! Compression codec support for archive formats.
2//!
3//! This module provides unified handling of compression codecs used with
4//! TAR archives. The same compression codecs can be used for both reading
5//! (decompression) and writing (compression).
6//!
7//! # Supported Codecs
8//!
9//! - **Gzip** (.tar.gz, .tgz): Fast compression with good compatibility
10//! - **Bzip2** (.tar.bz2, .tbz2): Better compression ratio, slower
11//! - **Xz** (.tar.xz, .txz): Best compression ratio, slowest
12//! - **Zstd** (.tar.zst, .tzst): Modern codec with good speed/ratio balance
13
14use std::io;
15use std::io::BufRead;
16use std::io::Read;
17use std::io::Write;
18
19use crate::Result;
20use crate::creation::compression::compression_level_to_bzip2;
21use crate::creation::compression::compression_level_to_flate2;
22use crate::creation::compression::compression_level_to_xz;
23use crate::creation::compression::compression_level_to_zstd;
24use crate::formats::tar_metadata_limit::DecoderMemoryExceeded;
25
26/// Compression codec for archive files.
27///
28/// Represents the compression algorithm used for compressed archives.
29/// Each codec has different trade-offs between compression ratio,
30/// speed, and compatibility.
31///
32/// # Performance Characteristics
33///
34/// | Codec | Compression | Decompression | Ratio | Compatibility |
35/// |-------|-------------|---------------|-------|---------------|
36/// | Gzip  | Fast        | Fast          | Good  | Excellent     |
37/// | Bzip2 | Slow        | Medium        | Better| Good          |
38/// | Xz    | Very Slow   | Medium        | Best  | Good          |
39/// | Zstd  | Fast        | Very Fast     | Good  | Modern        |
40///
41/// # Examples
42///
43/// ```
44/// use exarch_core::formats::compression::CompressionCodec;
45///
46/// // Choose codec based on requirements
47/// let fast_codec = CompressionCodec::Gzip; // Fast compression
48/// let best_codec = CompressionCodec::Xz; // Best compression ratio
49/// let modern_codec = CompressionCodec::Zstd; // Modern balanced approach
50/// ```
51///
52/// `#[non_exhaustive]` so support for a new compression codec is not a
53/// breaking change for downstream matches.
54#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
55#[non_exhaustive]
56pub enum CompressionCodec {
57    /// Gzip compression (deflate algorithm).
58    ///
59    /// Fast compression and decompression with widespread support.
60    /// Good default choice for most use cases.
61    Gzip,
62
63    /// Bzip2 compression (Burrows-Wheeler algorithm).
64    ///
65    /// Better compression ratio than gzip but slower.
66    /// Good for archives that will be distributed.
67    Bzip2,
68
69    /// Xz compression (LZMA2 algorithm).
70    ///
71    /// Best compression ratio but slowest.
72    /// Good for long-term storage or bandwidth-constrained transfers.
73    Xz,
74
75    /// Zstd compression (Zstandard algorithm).
76    ///
77    /// Modern codec with excellent speed and good compression ratio.
78    /// Good for archives that need fast decompression.
79    Zstd,
80}
81
82impl CompressionCodec {
83    /// Returns the typical file extension for this codec when used with TAR.
84    ///
85    /// # Examples
86    ///
87    /// ```
88    /// use exarch_core::formats::compression::CompressionCodec;
89    ///
90    /// assert_eq!(CompressionCodec::Gzip.extension(), "tar.gz");
91    /// assert_eq!(CompressionCodec::Bzip2.extension(), "tar.bz2");
92    /// assert_eq!(CompressionCodec::Xz.extension(), "tar.xz");
93    /// assert_eq!(CompressionCodec::Zstd.extension(), "tar.zst");
94    /// ```
95    #[must_use]
96    pub const fn extension(self) -> &'static str {
97        match self {
98            Self::Gzip => "tar.gz",
99            Self::Bzip2 => "tar.bz2",
100            Self::Xz => "tar.xz",
101            Self::Zstd => "tar.zst",
102        }
103    }
104
105    /// Wraps `input` in this codec's decoder.
106    ///
107    /// This is the only codec-to-decoder mapping used for reading TAR
108    /// streams. The `bufread` decoders take the caller's buffer as-is, so
109    /// the caller decides the buffering once instead of each decoder adding
110    /// its own. `max_decoder_memory` bounds the xz decoder only; the other
111    /// codecs are bounded by their format.
112    ///
113    /// Every gzip member, bzip2 stream and xz stream is decoded, and at least
114    /// one is required. After the last one an all-zero tail is accepted (xz:
115    /// a multiple of four bytes, as the format requires); any other trailing
116    /// bytes are an error.
117    ///
118    /// # Errors
119    ///
120    /// Returns an I/O error if the decoder cannot be initialised.
121    pub(crate) fn decoder<'r, R>(
122        self,
123        input: R,
124        max_decoder_memory: u64,
125    ) -> Result<Box<dyn Read + Send + 'r>>
126    where
127        R: BufRead + Send + 'r,
128    {
129        Ok(match self {
130            Self::Gzip => Box::new(MultiMember::<_, flate2::bufread::GzDecoder<R>>::new(input)),
131            Self::Bzip2 => Box::new(MultiMember::<_, bzip2::bufread::BzDecoder<R>>::new(input)),
132            Self::Xz => {
133                let stream = xz2::stream::Stream::new_stream_decoder(
134                    max_decoder_memory,
135                    xz2::stream::CONCATENATED,
136                )
137                .map_err(io::Error::from)?;
138                Box::new(XzLimited {
139                    inner: xz2::bufread::XzDecoder::new_stream(input, stream),
140                    max: max_decoder_memory,
141                })
142            }
143            Self::Zstd => Box::new(zstd::Decoder::with_buffer(input)?),
144        })
145    }
146
147    /// Wraps `out` in this codec's encoder.
148    ///
149    /// Write-side mirror of [`decoder`](Self::decoder) and the only
150    /// codec-to-encoder mapping used for creating TAR streams. `level` is the
151    /// user-facing 1-9 scale (`None` selects the codec default).
152    ///
153    /// # Errors
154    ///
155    /// Returns an I/O error if the encoder cannot be initialised.
156    pub(crate) fn encoder<W: Write>(self, out: W, level: Option<u8>) -> Result<TarEncoder<W>> {
157        Ok(match self {
158            Self::Gzip => TarEncoder::Gzip(flate2::write::GzEncoder::new(
159                out,
160                compression_level_to_flate2(level),
161            )),
162            Self::Bzip2 => TarEncoder::Bzip2(bzip2::write::BzEncoder::new(
163                out,
164                compression_level_to_bzip2(level),
165            )),
166            Self::Xz => TarEncoder::Xz(xz2::write::XzEncoder::new(
167                out,
168                compression_level_to_xz(level),
169            )),
170            Self::Zstd => {
171                let mut enc = zstd::Encoder::new(out, compression_level_to_zstd(level))?;
172                enc.include_checksum(true)?;
173                TarEncoder::Zstd(enc)
174            }
175        })
176    }
177
178    /// Returns a human-readable name for this codec.
179    ///
180    /// # Examples
181    ///
182    /// ```
183    /// use exarch_core::formats::compression::CompressionCodec;
184    ///
185    /// assert_eq!(CompressionCodec::Gzip.name(), "gzip");
186    /// assert_eq!(CompressionCodec::Bzip2.name(), "bzip2");
187    /// ```
188    #[must_use]
189    pub const fn name(self) -> &'static str {
190        match self {
191            Self::Gzip => "gzip",
192            Self::Bzip2 => "bzip2",
193            Self::Xz => "xz",
194            Self::Zstd => "zstd",
195        }
196    }
197}
198
199/// A decoder for exactly one member of a concatenated stream.
200trait Member<R>: Read {
201    fn start(input: R) -> Self;
202    fn finish(self) -> R;
203}
204
205impl<R: BufRead> Member<R> for flate2::bufread::GzDecoder<R> {
206    fn start(input: R) -> Self {
207        Self::new(input)
208    }
209
210    fn finish(self) -> R {
211        self.into_inner()
212    }
213}
214
215impl<R: BufRead> Member<R> for bzip2::bufread::BzDecoder<R> {
216    fn start(input: R) -> Self {
217        Self::new(input)
218    }
219
220    fn finish(self) -> R {
221        self.into_inner()
222    }
223}
224
225enum MemberState<R, D> {
226    Between(R),
227    Active(D),
228    Done,
229    /// A non-transient error; replayed on every later read so a retry can
230    /// never look like a clean end of stream.
231    Failed(io::ErrorKind, String),
232}
233
234enum Step {
235    Data(usize),
236    Eof,
237    Continue,
238}
239
240/// Decodes every member of a gzip or bzip2 stream, as `gzip -dc` and
241/// `bzip2 -dc` do, so entries in later members are never silently dropped.
242///
243/// At least one member is required: an empty or all-zero input is not a
244/// compressed stream. After the last member an all-zero tail (tape padding) is
245/// accepted; any other trailing bytes are an error. xz is stricter about the
246/// padding: the format only allows multiples of four zero bytes.
247struct MultiMember<R, D> {
248    state: MemberState<R, D>,
249    seen_member: bool,
250}
251
252impl<R, D> MultiMember<R, D> {
253    const fn new(input: R) -> Self {
254        Self {
255            state: MemberState::Between(input),
256            seen_member: false,
257        }
258    }
259}
260
261fn consume_zero_tail<R: BufRead>(input: &mut R) -> io::Result<()> {
262    loop {
263        let buf = input.fill_buf()?;
264        if buf.is_empty() {
265            return Ok(());
266        }
267        if buf.iter().any(|&b| b != 0) {
268            return Err(io::Error::new(
269                io::ErrorKind::InvalidData,
270                "trailing data after the last compressed member",
271            ));
272        }
273        let n = buf.len();
274        input.consume(n);
275    }
276}
277
278impl<R: BufRead, D: Member<R>> MultiMember<R, D> {
279    fn step(&mut self, out: &mut [u8]) -> io::Result<Step> {
280        match &mut self.state {
281            MemberState::Done => Ok(Step::Eof),
282            MemberState::Failed(kind, message) => Err(io::Error::new(*kind, message.clone())),
283            MemberState::Between(input) => {
284                let first = input.fill_buf()?.first().copied();
285                match first {
286                    None | Some(0) if !self.seen_member => Err(io::Error::new(
287                        io::ErrorKind::InvalidData,
288                        "no compressed member found",
289                    )),
290                    None => {
291                        self.state = MemberState::Done;
292                        Ok(Step::Eof)
293                    }
294                    Some(0) => {
295                        consume_zero_tail(input)?;
296                        self.state = MemberState::Done;
297                        Ok(Step::Eof)
298                    }
299                    Some(_) => {
300                        self.seen_member = true;
301                        if let MemberState::Between(input) =
302                            std::mem::replace(&mut self.state, MemberState::Done)
303                        {
304                            self.state = MemberState::Active(D::start(input));
305                        }
306                        Ok(Step::Continue)
307                    }
308                }
309            }
310            MemberState::Active(decoder) => {
311                let n = decoder.read(out)?;
312                if n > 0 {
313                    return Ok(Step::Data(n));
314                }
315                if let MemberState::Active(decoder) =
316                    std::mem::replace(&mut self.state, MemberState::Done)
317                {
318                    self.state = MemberState::Between(decoder.finish());
319                }
320                Ok(Step::Continue)
321            }
322        }
323    }
324}
325
326impl<R: BufRead, D: Member<R>> Read for MultiMember<R, D> {
327    fn read(&mut self, out: &mut [u8]) -> io::Result<usize> {
328        if out.is_empty() {
329            return Ok(0);
330        }
331        loop {
332            match self.step(out) {
333                Ok(Step::Data(n)) => return Ok(n),
334                Ok(Step::Eof) => return Ok(0),
335                Ok(Step::Continue) => {}
336                Err(e) => {
337                    // Interrupted is transient: the state is intact and a retry
338                    // resumes.
339                    if e.kind() != io::ErrorKind::Interrupted
340                        && !matches!(self.state, MemberState::Failed(..))
341                    {
342                        self.state = MemberState::Failed(e.kind(), e.to_string());
343                    }
344                    return Err(e);
345                }
346            }
347        }
348    }
349}
350
351/// xz decoder that reports liblzma's memory-limit error as a typed
352/// [`DecoderMemoryExceeded`] so `budget_violation` can classify it.
353struct XzLimited<R> {
354    inner: xz2::bufread::XzDecoder<R>,
355    max: u64,
356}
357
358impl<R: BufRead> Read for XzLimited<R> {
359    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
360        self.inner.read(buf).map_err(|e| {
361            let mem_limit = e
362                .get_ref()
363                .and_then(|inner| inner.downcast_ref::<xz2::stream::Error>())
364                .is_some_and(|err| matches!(err, xz2::stream::Error::MemLimit));
365            if mem_limit {
366                io::Error::other(DecoderMemoryExceeded { max: self.max })
367            } else {
368                e
369            }
370        })
371    }
372}
373
374/// A compressing writer for one of the [`CompressionCodec`]s.
375///
376/// Must be closed with [`finish`](Self::finish): dropping it can lose the
377/// trailing frame and any flush error.
378pub(crate) enum TarEncoder<W: Write> {
379    Gzip(flate2::write::GzEncoder<W>),
380    Bzip2(bzip2::write::BzEncoder<W>),
381    Xz(xz2::write::XzEncoder<W>),
382    Zstd(zstd::Encoder<'static, W>),
383}
384
385impl<W: Write> TarEncoder<W> {
386    /// Writes the trailer, flushes, and returns the inner writer.
387    pub(crate) fn finish(self) -> io::Result<W> {
388        match self {
389            Self::Gzip(e) => e.finish(),
390            Self::Bzip2(e) => e.finish(),
391            Self::Xz(e) => e.finish(),
392            Self::Zstd(e) => e.finish(),
393        }
394    }
395}
396
397impl<W: Write> Write for TarEncoder<W> {
398    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
399        match self {
400            Self::Gzip(e) => e.write(buf),
401            Self::Bzip2(e) => e.write(buf),
402            Self::Xz(e) => e.write(buf),
403            Self::Zstd(e) => e.write(buf),
404        }
405    }
406
407    fn flush(&mut self) -> io::Result<()> {
408        match self {
409            Self::Gzip(e) => e.flush(),
410            Self::Bzip2(e) => e.flush(),
411            Self::Xz(e) => e.flush(),
412            Self::Zstd(e) => e.flush(),
413        }
414    }
415}
416
417#[cfg(test)]
418#[allow(clippy::unwrap_used)]
419mod tests {
420    use super::*;
421
422    #[test]
423    fn test_codec_extension() {
424        assert_eq!(CompressionCodec::Gzip.extension(), "tar.gz");
425        assert_eq!(CompressionCodec::Bzip2.extension(), "tar.bz2");
426        assert_eq!(CompressionCodec::Xz.extension(), "tar.xz");
427        assert_eq!(CompressionCodec::Zstd.extension(), "tar.zst");
428    }
429
430    #[test]
431    fn test_codec_name() {
432        assert_eq!(CompressionCodec::Gzip.name(), "gzip");
433        assert_eq!(CompressionCodec::Bzip2.name(), "bzip2");
434        assert_eq!(CompressionCodec::Xz.name(), "xz");
435        assert_eq!(CompressionCodec::Zstd.name(), "zstd");
436    }
437
438    fn encode(codec: CompressionCodec, data: &[u8]) -> Vec<u8> {
439        use std::io::Write;
440
441        match codec {
442            CompressionCodec::Gzip => {
443                let mut enc =
444                    flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
445                enc.write_all(data).unwrap();
446                enc.finish().unwrap()
447            }
448            CompressionCodec::Bzip2 => {
449                let mut enc = bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::fast());
450                enc.write_all(data).unwrap();
451                enc.finish().unwrap()
452            }
453            CompressionCodec::Xz => {
454                let mut enc = xz2::write::XzEncoder::new(Vec::new(), 1);
455                enc.write_all(data).unwrap();
456                enc.finish().unwrap()
457            }
458            CompressionCodec::Zstd => zstd::encode_all(data, 1).unwrap(),
459        }
460    }
461
462    const TEST_MEM: u64 = 256 * 1024 * 1024;
463
464    const ALL_CODECS: [CompressionCodec; 4] = [
465        CompressionCodec::Gzip,
466        CompressionCodec::Bzip2,
467        CompressionCodec::Xz,
468        CompressionCodec::Zstd,
469    ];
470
471    #[test]
472    fn test_decoder_round_trips_every_codec() {
473        let data = b"hello decoder ".repeat(1000);
474        for codec in ALL_CODECS {
475            let mut out = Vec::new();
476            codec
477                .decoder(std::io::Cursor::new(encode(codec, &data)), TEST_MEM)
478                .unwrap()
479                .read_to_end(&mut out)
480                .unwrap();
481            assert_eq!(out, data, "{}", codec.name());
482        }
483    }
484
485    #[test]
486    fn test_encoder_round_trips_every_codec_and_level() {
487        let data = b"hello encoder ".repeat(1000);
488        for codec in ALL_CODECS {
489            for level in [None, Some(1), Some(9)] {
490                let mut enc = codec.encoder(Vec::new(), level).unwrap();
491                enc.write_all(&data).unwrap();
492                let encoded = enc.finish().unwrap();
493                let mut out = Vec::new();
494                codec
495                    .decoder(std::io::Cursor::new(encoded), TEST_MEM)
496                    .unwrap()
497                    .read_to_end(&mut out)
498                    .unwrap();
499                assert_eq!(out, data, "{} {level:?}", codec.name());
500            }
501        }
502    }
503
504    fn decode_all(codec: CompressionCodec, bytes: Vec<u8>, mem: u64) -> io::Result<Vec<u8>> {
505        let mut out = Vec::new();
506        codec
507            .decoder(std::io::Cursor::new(bytes), mem)
508            .unwrap()
509            .read_to_end(&mut out)?;
510        Ok(out)
511    }
512
513    #[test]
514    fn test_decoder_reads_every_concatenated_member() {
515        for codec in ALL_CODECS {
516            let mut encoded = encode(codec, b"first-");
517            encoded.extend_from_slice(&encode(codec, b"second-"));
518            encoded.extend_from_slice(&encode(codec, b"third"));
519            assert_eq!(
520                decode_all(codec, encoded, TEST_MEM).unwrap(),
521                b"first-second-third",
522                "{}",
523                codec.name()
524            );
525        }
526    }
527
528    #[test]
529    fn test_decoder_accepts_zero_padding_after_last_member() {
530        for codec in [
531            CompressionCodec::Gzip,
532            CompressionCodec::Bzip2,
533            CompressionCodec::Xz,
534        ] {
535            let mut encoded = encode(codec, b"a");
536            encoded.extend_from_slice(&encode(codec, b"b"));
537            encoded.extend_from_slice(&[0u8; 512]);
538            assert_eq!(
539                decode_all(codec, encoded, TEST_MEM).unwrap(),
540                b"ab",
541                "{}",
542                codec.name()
543            );
544        }
545    }
546
547    struct FailsOnce<R> {
548        inner: R,
549        error: Option<io::ErrorKind>,
550    }
551
552    impl<R: Read> Read for FailsOnce<R> {
553        fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
554            match self.error.take() {
555                Some(kind) => Err(io::Error::from(kind)),
556                None => self.inner.read(buf),
557            }
558        }
559    }
560
561    impl<R: BufRead> BufRead for FailsOnce<R> {
562        fn fill_buf(&mut self) -> io::Result<&[u8]> {
563            match self.error.take() {
564                Some(kind) => Err(io::Error::from(kind)),
565                None => self.inner.fill_buf(),
566            }
567        }
568
569        fn consume(&mut self, amt: usize) {
570            self.inner.consume(amt);
571        }
572    }
573
574    #[test]
575    fn test_read_error_never_turns_into_a_clean_eof() {
576        let mut encoded = encode(CompressionCodec::Gzip, b"first-");
577        encoded.extend_from_slice(&encode(CompressionCodec::Gzip, b"second"));
578        for kind in [io::ErrorKind::Interrupted, io::ErrorKind::BrokenPipe] {
579            let input = FailsOnce {
580                inner: std::io::Cursor::new(encoded.clone()),
581                error: Some(kind),
582            };
583            let mut decoder = CompressionCodec::Gzip.decoder(input, TEST_MEM).unwrap();
584            let mut out = Vec::new();
585            let mut buf = [0u8; 64];
586            let mut errors = 0;
587            loop {
588                match decoder.read(&mut buf) {
589                    Ok(0) => break,
590                    Ok(n) => out.extend_from_slice(&buf[..n]),
591                    Err(e) => {
592                        assert_eq!(e.kind(), kind);
593                        errors += 1;
594                        if kind != io::ErrorKind::Interrupted {
595                            assert_eq!(decoder.read(&mut buf).unwrap_err().kind(), kind);
596                            return;
597                        }
598                    }
599                }
600            }
601            assert_eq!(errors, 1);
602            assert_eq!(out, b"first-second");
603        }
604    }
605
606    #[test]
607    fn test_decoder_rejects_input_without_a_first_member() {
608        for codec in [
609            CompressionCodec::Gzip,
610            CompressionCodec::Bzip2,
611            CompressionCodec::Xz,
612        ] {
613            for input in [Vec::new(), vec![0u8; 64]] {
614                assert!(
615                    decode_all(codec, input, TEST_MEM).is_err(),
616                    "{}",
617                    codec.name()
618                );
619            }
620        }
621    }
622
623    #[test]
624    fn test_decoder_rejects_trailing_garbage() {
625        for codec in [
626            CompressionCodec::Gzip,
627            CompressionCodec::Bzip2,
628            CompressionCodec::Xz,
629        ] {
630            for tail in [&b"TRAILING-BYTES"[..], &b"\0\0\0\0garbage"[..]] {
631                let mut encoded = encode(codec, b"payload");
632                encoded.extend_from_slice(tail);
633                assert!(
634                    decode_all(codec, encoded, TEST_MEM).is_err(),
635                    "{} {tail:?}",
636                    codec.name()
637                );
638            }
639        }
640    }
641
642    #[test]
643    fn test_xz_memlimit_applies_to_later_stream() {
644        let encode_at = |preset: u32, data: &[u8]| {
645            let mut enc = xz2::write::XzEncoder::new(Vec::new(), preset);
646            enc.write_all(data).unwrap();
647            enc.finish().unwrap()
648        };
649        let mut encoded = encode_at(0, b"first");
650        encoded.extend_from_slice(&encode_at(9, b"second"));
651        let err = decode_all(CompressionCodec::Xz, encoded, 16 * 1024 * 1024).unwrap_err();
652        assert!(
653            err.get_ref()
654                .is_some_and(<dyn std::error::Error + Send + Sync>::is::<DecoderMemoryExceeded>)
655        );
656    }
657
658    #[test]
659    fn test_decoder_reads_concatenated_zstd_frames() {
660        let mut encoded = encode(CompressionCodec::Zstd, b"first-");
661        encoded.extend_from_slice(&encode(CompressionCodec::Zstd, b"second"));
662        let mut out = Vec::new();
663        CompressionCodec::Zstd
664            .decoder(std::io::Cursor::new(encoded), TEST_MEM)
665            .unwrap()
666            .read_to_end(&mut out)
667            .unwrap();
668        assert_eq!(out, b"first-second");
669    }
670
671    #[test]
672    fn test_codec_equality() {
673        assert_eq!(CompressionCodec::Gzip, CompressionCodec::Gzip);
674        assert_ne!(CompressionCodec::Gzip, CompressionCodec::Bzip2);
675    }
676
677    #[test]
678    fn test_codec_hash() {
679        use std::collections::HashSet;
680
681        let mut set = HashSet::new();
682        set.insert(CompressionCodec::Gzip);
683        set.insert(CompressionCodec::Bzip2);
684        set.insert(CompressionCodec::Gzip); // Duplicate
685
686        assert_eq!(set.len(), 2);
687        assert!(set.contains(&CompressionCodec::Gzip));
688        assert!(set.contains(&CompressionCodec::Bzip2));
689    }
690}
exarch_core/formats/compression.rs

exarch_core/formats/
compression.rs
