stuffr_cli/compat/gzip/
sniff.rs

1//! Format sniffing for the gzip family: which formats each flavour's
2//! `gunzip` accepts, recognised by magic, and the decoders for the ones that
3//! are not gzip.
4//!
5//! Each table is the reference's. Apple's comes from `gzip.c`'s
6//! `file_gettype` (file_cmds-479, BSD-2-Clause): gzip (`1f 8b`, or the old
7//! `1f 9e`), bzip2 (`BZh` and a digit), `.Z` (`1f 9d`), pack (`1f 1e`), xz
8//! (`fd 37 7a 58`, four bytes only: the source says "we only have 4 bytes")
9//! and lzip (`LZIP`). GNU's is its manual's list (gzip, zip, compress, pack,
10//! LZH) checked by running `gzip -dc` on samples, plus the old gzip magic
11//! `1f 9e`, which GNU 1.15 also decodes; see the ruling on LZH on
12//! [`Detected::Lzh`].
13//!
14//! Every magic is at most four bytes, so [`detect`] takes the [`Lookahead`]
15//! the callers already hold, whose `fill_buf` guarantees [`member::PEEK`]
16//! bytes unless at EOF: a short peek is then a short input, never a short
17//! read.
18
19use std::io::{self, BufRead, Read};
20
21use stuffr::FormatId;
22use stuffr::pack_stream::{PackError, PackReader};
23
24use super::member::{Lookahead, PEEK};
25
26/// Which reference the gzip family follows.
27#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub enum Flavour {
29    /// GNU gzip 1.15, the default outside the BSDs and macOS.
30    Gnu,
31    /// Apple's gzip (FreeBSD/NetBSD lineage), the default on macOS and the
32    /// BSDs.
33    Bsd,
34}
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq)]
37pub(crate) enum Detected {
38    Gzip,
39    /// `compress` (`.Z`, `1f 9d`).
40    Compress,
41    Bzip2,
42    Xz,
43    Lzip,
44    /// The old Unix `pack` (`1f 1e`).
45    Pack,
46    /// A zip local header (`PK\x03\x04`), GNU only.
47    Zip,
48    /// GNU's LZH (`1f a0`, `compress -H`/SCO). Never returned: stuffr has no
49    /// decoder for it, so it is a documented gap and a `1f a0` input is
50    /// `Unknown` (see `foreign_reader`'s callers for the wording).
51    Lzh,
52    Unknown,
53}
54
55/// Classify by magic, peeking without consuming. A format the flavour does
56/// not decode (or this build lacks) is `Unknown`.
57pub(crate) fn detect<R: Read>(r: &mut Lookahead<R>, flavour: Flavour) -> io::Result<Detected> {
58    let buf = r.fill_buf()?;
59    let m = &buf[..buf.len().min(PEEK)];
60    let d = match flavour {
61        Flavour::Gnu => detect_gnu(m),
62        Flavour::Bsd => detect_bsd(m),
63    };
64    Ok(if available(d) { d } else { Detected::Unknown })
65}
66
67/// GNU's table. `1f 9e` (the old gzip magic) is gzip: GNU 1.15 decodes a
68/// member carrying it (observed: `gzip -dc` on one, exit 0; Ruling T5-1).
69fn detect_gnu(m: &[u8]) -> Detected {
70    match m {
71        [0x1f, 0x8b | 0x9e, ..] => Detected::Gzip,
72        [b'P', b'K', 3, 4] => Detected::Zip,
73        [0x1f, 0x9d, ..] => Detected::Compress,
74        [0x1f, 0x1e, ..] => Detected::Pack,
75        _ => Detected::Unknown,
76    }
77}
78
79/// `file_gettype`, in its order.
80fn detect_bsd(m: &[u8]) -> Detected {
81    match m {
82        [0x1f, 0x8b | 0x9e, ..] => Detected::Gzip,
83        [b'B', b'Z', b'h', d, ..] if d.is_ascii_digit() => Detected::Bzip2,
84        [0x1f, 0x9d, ..] => Detected::Compress,
85        [0x1f, 0x1e, ..] => Detected::Pack,
86        [0xfd, b'7', b'z', b'X'] => Detected::Xz,
87        [b'L', b'Z', b'I', b'P'] => Detected::Lzip,
88        _ => Detected::Unknown,
89    }
90}
91
92/// Whether this build can decode `d`. A codec absent from the registry (the
93/// `legacy` tier carries `compress`) makes its format unknown.
94fn available(d: Detected) -> bool {
95    match codec_id(d) {
96        Some(id) => stuffr::registry().codec(FormatId::new(id)).is_some(),
97        None => !matches!(d, Detected::Lzh),
98    }
99}
100
101fn codec_id(d: Detected) -> Option<&'static str> {
102    match d {
103        Detected::Compress => Some("compress"),
104        Detected::Bzip2 => Some("bzip2"),
105        Detected::Xz => Some("xz"),
106        Detected::Lzip => Some("lzip"),
107        _ => None,
108    }
109}
110
111/// A decoder for a non-gzip format, reading `r` to its end. Errors are
112/// `io::Error`s carrying the codec's own error, for the flavour to word.
113///
114/// A pack header's failure keeps its [`PackError`] inside an error of the
115/// kind every other codec gives the same failure: `UnexpectedEof` for an
116/// input that ends in the header or the tables, `InvalidData` for one that
117/// contradicts itself, the reader's own error for a failed read. (The
118/// bitstream's failures come from the reader, `PackError` inside `Other`.)
119pub(crate) fn foreign_reader(d: Detected, r: Box<dyn BufRead + Send>) -> io::Result<Box<dyn Read>> {
120    if d == Detected::Pack {
121        return match PackReader::new(r) {
122            Ok(p) => Ok(Box::new(p)),
123            Err(PackError::Io(e)) => Err(e),
124            Err(e @ (PackError::Truncated | PackError::BadHeader(_))) => {
125                Err(io::Error::new(io::ErrorKind::UnexpectedEof, e))
126            }
127            Err(PackError::Corrupt("Symbol table truncated")) => Err(io::Error::new(
128                io::ErrorKind::UnexpectedEof,
129                PackError::Corrupt("Symbol table truncated"),
130            )),
131            Err(e) => Err(io::Error::new(io::ErrorKind::InvalidData, e)),
132        };
133    }
134    let Some(id) = codec_id(d) else {
135        return Err(io::Error::new(
136            io::ErrorKind::InvalidInput,
137            "not a foreign format",
138        ));
139    };
140    let Some(codec) = stuffr::registry().codec(FormatId::new(id)) else {
141        return Err(io::Error::new(
142            io::ErrorKind::Unsupported,
143            "format not in this build",
144        ));
145    };
146    let src = Box::new(stuffr::StreamOnly::new(r));
147    match codec.decoder(src, &stuffr::DecodeOpts::default()) {
148        Ok(s) => Ok(Box::new(s)),
149        Err(stuffr::Error::Io(e)) => Err(e),
150        Err(e) => Err(io::Error::other(e)),
151    }
152}
153
154/// The first local entry of a zip: what GNU's `gunzip` reads. Method, flags
155/// and sizes are exposed raw; which entries GNU accepts or refuses, and how
156/// it words that, is decided by the personality.
157pub(crate) struct ZipEntry<R> {
158    pub method: u16,
159    pub flags: u16,
160    pub crc: u32,
161    pub compressed_size: u32,
162    pub uncompressed_size: u32,
163    pub name: Vec<u8>,
164    reader: R,
165}
166
167/// Parse the first local header (signature included) and the name/extra
168/// fields, leaving `r` at the start of the entry's data.
169pub(crate) fn zip_first_entry<R: BufRead>(mut r: R) -> io::Result<ZipEntry<R>> {
170    let mut h = [0u8; 30];
171    r.read_exact(&mut h)?;
172    if &h[..4] != b"PK\x03\x04" {
173        return Err(io::Error::new(
174            io::ErrorKind::InvalidData,
175            "not a zip local header",
176        ));
177    }
178    let u16_at = |i: usize| u16::from_le_bytes([h[i], h[i + 1]]);
179    let u32_at = |i: usize| u32::from_le_bytes([h[i], h[i + 1], h[i + 2], h[i + 3]]);
180    let (flags, method) = (u16_at(6), u16_at(8));
181    let (crc, compressed_size, uncompressed_size) = (u32_at(14), u32_at(18), u32_at(22));
182    let (nlen, xlen) = (usize::from(u16_at(26)), u64::from(u16_at(28)));
183    let mut name = vec![0u8; nlen];
184    r.read_exact(&mut name)?;
185    let skipped = io::copy(&mut (&mut r).take(xlen), &mut io::sink())?;
186    if skipped < xlen {
187        return Err(io::ErrorKind::UnexpectedEof.into());
188    }
189    Ok(ZipEntry {
190        method,
191        flags,
192        crc,
193        compressed_size,
194        uncompressed_size,
195        name,
196        reader: r,
197    })
198}
199
200impl<R: BufRead> ZipEntry<R> {
201    /// The entry's data: stored (needs the header's size, so not with a data
202    /// descriptor, flag bit 3) or deflated, which stops where the deflate
203    /// stream does. Any other method is `Unsupported`.
204    pub(crate) fn data(self) -> io::Result<Box<dyn Read>>
205    where
206        R: 'static,
207    {
208        match self.method {
209            0 if self.flags & 8 == 0 => Ok(Box::new(self.reader.take(self.compressed_size.into()))),
210            0 => Err(io::Error::new(
211                io::ErrorKind::Unsupported,
212                "stored entry with a data descriptor",
213            )),
214            8 => Ok(Box::new(flate2::bufread::DeflateDecoder::new(self.reader))),
215            _ => Err(io::Error::new(
216                io::ErrorKind::Unsupported,
217                "zip method not stored or deflated",
218            )),
219        }
220    }
221}
222
223#[cfg(test)]
224mod tests {
225    use super::*;
226    use std::io::Write;
227
228    fn la(b: &[u8]) -> Lookahead<&[u8]> {
229        Lookahead::new(b)
230    }
231
232    fn has(id: &'static str) -> bool {
233        stuffr::registry().codec(FormatId::new(id)).is_some()
234    }
235
236    #[test]
237    fn detect_every_magic_per_flavour() {
238        use Detected::*;
239        let compress = if has("compress") { Compress } else { Unknown };
240        let bzip2 = if has("bzip2") { Bzip2 } else { Unknown };
241        let xz = if has("xz") { Xz } else { Unknown };
242        let lzip = if has("lzip") { Lzip } else { Unknown };
243        let table: &[(&[u8], Flavour, Detected)] = &[
244            (b"\x1f\x8b\x08\x00", Flavour::Gnu, Gzip),
245            (b"\x1f\x9e\x08\x00", Flavour::Gnu, Gzip),
246            (b"PK\x03\x04", Flavour::Gnu, Zip),
247            (b"\x1f\x9d\x90\x00", Flavour::Gnu, compress),
248            (b"\x1f\x1e\x00\x00", Flavour::Gnu, Pack),
249            (b"\x1f\xa0\x00\x00", Flavour::Gnu, Unknown), // LZH gap
250            (b"\x1f\x8b\x08\x00", Flavour::Bsd, Gzip),
251            (b"\x1f\x9e\x08\x00", Flavour::Bsd, Gzip),
252            (b"\x1f\x9d\x90\x00", Flavour::Bsd, compress),
253            (b"BZh9", Flavour::Bsd, bzip2),
254            (b"\xfd7zXZ\x00", Flavour::Bsd, xz),
255            (b"\xfd7zX", Flavour::Bsd, xz),
256            (b"\x1f\x1e\x00\x00", Flavour::Bsd, Pack),
257            (b"LZIP\x01", Flavour::Bsd, lzip),
258        ];
259        for (bytes, fl, want) in table {
260            assert_eq!(
261                detect(&mut la(bytes), *fl).unwrap(),
262                *want,
263                "{bytes:02x?} {fl:?}"
264            );
265        }
266    }
267
268    #[test]
269    fn a_magic_the_flavour_lacks_is_unknown() {
270        let cases: &[(&[u8], Flavour)] = &[
271            (b"BZh9", Flavour::Gnu),
272            (b"\xfd7zXZ\x00", Flavour::Gnu),
273            (b"LZIP\x01", Flavour::Gnu),
274            (b"PK\x03\x04", Flavour::Bsd),
275            (b"\x1f\xa0\x00\x00", Flavour::Bsd),
276            (b"BZhx", Flavour::Bsd), // Apple wants a digit after BZh
277        ];
278        for (bytes, fl) in cases {
279            assert_eq!(
280                detect(&mut la(bytes), *fl).unwrap(),
281                Detected::Unknown,
282                "{bytes:02x?} {fl:?}"
283            );
284        }
285    }
286
287    #[test]
288    fn detect_does_not_consume() {
289        let mut r = la(b"\x1f\x8b\x08\x00rest");
290        detect(&mut r, Flavour::Gnu).unwrap();
291        let mut all = Vec::new();
292        r.read_to_end(&mut all).unwrap();
293        assert_eq!(all, b"\x1f\x8b\x08\x00rest");
294    }
295
296    #[test]
297    fn detect_sees_a_whole_magic_through_tiny_reads() {
298        struct OneByte<'a>(&'a [u8]);
299        impl Read for OneByte<'_> {
300            fn read(&mut self, o: &mut [u8]) -> io::Result<usize> {
301                let n = self.0.len().min(1).min(o.len());
302                o[..n].copy_from_slice(&self.0[..n]);
303                self.0 = &self.0[n..];
304                Ok(n)
305            }
306        }
307        let mut r = Lookahead::new(OneByte(b"PK\x03\x04xxxx"));
308        assert_eq!(detect(&mut r, Flavour::Gnu).unwrap(), Detected::Zip);
309        let mut r = Lookahead::new(OneByte(b"LZIP\x01"));
310        let want = if has("lzip") {
311            Detected::Lzip
312        } else {
313            Detected::Unknown
314        };
315        assert_eq!(detect(&mut r, Flavour::Bsd).unwrap(), want);
316    }
317
318    #[test]
319    fn two_byte_and_empty_inputs() {
320        for fl in [Flavour::Gnu, Flavour::Bsd] {
321            assert_eq!(detect(&mut la(b""), fl).unwrap(), Detected::Unknown);
322            assert_eq!(detect(&mut la(b"\x1f"), fl).unwrap(), Detected::Unknown);
323            assert_eq!(detect(&mut la(b"\x1f\x8b"), fl).unwrap(), Detected::Gzip);
324            assert_eq!(detect(&mut la(b"\x1f\x1e"), fl).unwrap(), Detected::Pack);
325            assert_eq!(detect(&mut la(b"PK"), fl).unwrap(), Detected::Unknown);
326        }
327    }
328
329    fn encode(id: &'static str, data: &[u8]) -> Vec<u8> {
330        use std::sync::{Arc, Mutex};
331        struct Shared(Arc<Mutex<Vec<u8>>>);
332        impl Write for Shared {
333            fn write(&mut self, b: &[u8]) -> io::Result<usize> {
334                self.0.lock().unwrap().extend_from_slice(b);
335                Ok(b.len())
336            }
337            fn flush(&mut self) -> io::Result<()> {
338                Ok(())
339            }
340        }
341        let out = Arc::new(Mutex::new(Vec::new()));
342        let codec = stuffr::registry().codec(FormatId::new(id)).unwrap();
343        let mut enc = codec
344            .encoder(
345                Box::new(Shared(out.clone())),
346                &stuffr::EncodeOpts::default(),
347            )
348            .unwrap();
349        enc.write_all(data).unwrap();
350        enc.finish().unwrap();
351        out.lock().unwrap().clone()
352    }
353
354    fn sample() -> Vec<u8> {
355        (0..5000u32)
356            .flat_map(|i| format!("line {i} of the sample\n").into_bytes())
357            .collect()
358    }
359
360    #[test]
361    fn foreign_reader_round_trips() {
362        let data = sample();
363        let mut cases: Vec<(Detected, Flavour, &str)> = vec![
364            (Detected::Bzip2, Flavour::Bsd, "bzip2"),
365            (Detected::Xz, Flavour::Bsd, "xz"),
366            (Detected::Lzip, Flavour::Bsd, "lzip"),
367        ];
368        // `.Z` is decode-only in some builds; encode it when we can.
369        if has("compress")
370            && stuffr::registry()
371                .codec(FormatId::new("compress"))
372                .unwrap()
373                .caps()
374                .encode
375        {
376            cases.push((Detected::Compress, Flavour::Gnu, "compress"));
377        }
378        for (d, fl, id) in cases {
379            if !has(id) {
380                continue;
381            }
382            let packed = encode(id, &data);
383            let mut r = Lookahead::new(io::Cursor::new(packed));
384            assert_eq!(detect(&mut r, fl).unwrap(), d, "{id}");
385            let mut out = Vec::new();
386            foreign_reader(d, Box::new(r))
387                .unwrap()
388                .read_to_end(&mut out)
389                .unwrap();
390            assert_eq!(out, data, "{id}");
391        }
392    }
393
394    #[test]
395    fn foreign_reader_decodes_pack() {
396        // `/usr/bin/gzip -dc` prints BAAAAAAA for these exact bytes.
397        let p = b"\x1f\x1e\x00\x00\x00\x08\x03\x01\x00\x00AB\x80".to_vec();
398        let mut r = Lookahead::new(io::Cursor::new(p));
399        assert_eq!(detect(&mut r, Flavour::Gnu).unwrap(), Detected::Pack);
400        let mut out = Vec::new();
401        foreign_reader(Detected::Pack, Box::new(r))
402            .unwrap()
403            .read_to_end(&mut out)
404            .unwrap();
405        assert_eq!(out, b"BAAAAAAA");
406    }
407
408    /// Every foreign format's failures through `foreign_reader`: a header
409    /// cut short, a stream cut short, and bytes that contradict
410    /// themselves are each an error (from `foreign_reader` or from the
411    /// reader), never a panic or a quiet short read.
412    #[test]
413    fn foreign_reader_fails_on_bad_input() {
414        let data = sample();
415        let mut formats: Vec<(Detected, Vec<u8>)> = Vec::new();
416        for (d, id) in [
417            (Detected::Bzip2, "bzip2"),
418            (Detected::Xz, "xz"),
419            (Detected::Lzip, "lzip"),
420            (Detected::Compress, "compress"),
421        ] {
422            if has(id)
423                && stuffr::registry()
424                    .codec(FormatId::new(id))
425                    .is_some_and(|c| c.caps().encode)
426            {
427                formats.push((d, encode(id, &data)));
428            }
429        }
430        let decode = |d: Detected, b: Vec<u8>| -> io::Result<Vec<u8>> {
431            let mut out = Vec::new();
432            foreign_reader(d, Box::new(io::Cursor::new(b)))?.read_to_end(&mut out)?;
433            Ok(out)
434        };
435        for (d, packed) in formats {
436            // The header alone, and half the stream. (`.Z` has no end
437            // marker: cut short, it is a shorter stream.)
438            if d != Detected::Compress {
439                for cut in [3, packed.len() / 2] {
440                    assert!(
441                        decode(d, packed[..cut].to_vec()).is_err(),
442                        "{d:?} cut at {cut}"
443                    );
444                }
445            }
446            // A run of bytes overwritten in the middle.
447            let mut bad = packed.clone();
448            let at = packed.len() / 2;
449            for b in &mut bad[at..at + 16.min(packed.len() - at)] {
450                *b ^= 0x5a;
451            }
452            assert!(
453                decode(d, bad).map_or(true, |o| o != data),
454                "{d:?}: corruption went unnoticed"
455            );
456        }
457        // pack: each header failure has its kind and keeps its words.
458        let kind_and_words = |b: &[u8]| {
459            let e = foreign_reader(Detected::Pack, Box::new(io::Cursor::new(b.to_vec())))
460                .err()
461                .unwrap();
462            let words = e.get_ref().map(ToString::to_string).unwrap_or_default();
463            (e.kind(), words)
464        };
465        assert_eq!(
466            kind_and_words(b"\x1f\x1e\x00\x00"),
467            (
468                io::ErrorKind::UnexpectedEof,
469                "header shorter than 7 bytes".into()
470            )
471        );
472        assert_eq!(
473            kind_and_words(b"\x1f\x1e\x00\x00\x00\x08\x03\x01"),
474            (
475                io::ErrorKind::UnexpectedEof,
476                "File appears to be truncated".into()
477            )
478        );
479        assert_eq!(
480            kind_and_words(b"\x1f\x1e\x00\x00\x00\x08\x03\x01\x00\x00A"),
481            (
482                io::ErrorKind::UnexpectedEof,
483                "Symbol table truncated".into()
484            )
485        );
486        assert_eq!(
487            kind_and_words(b"\x1f\x1e\x00\x00\x00\x08\x1a"),
488            (
489                io::ErrorKind::InvalidData,
490                "Huffman tree has insane levels".into()
491            )
492        );
493        // And the bitstream's, from the reader, after what it decoded.
494        let mut r = foreign_reader(
495            Detected::Pack,
496            Box::new(io::Cursor::new(
497                b"\x1f\x1e\x00\x00\x00\x0a\x03\x01\x00\x00AB\x80".to_vec(),
498            )),
499        )
500        .unwrap();
501        let mut out = Vec::new();
502        let e = r.read_to_end(&mut out).unwrap_err();
503        assert_eq!(out, b"BAAAAAAA");
504        assert_eq!(
505            e.get_ref().map(ToString::to_string).unwrap(),
506            "Premature EOF"
507        );
508    }
509
510    #[test]
511    fn foreign_reader_refuses_what_is_not_foreign() {
512        for d in [
513            Detected::Gzip,
514            Detected::Zip,
515            Detected::Lzh,
516            Detected::Unknown,
517        ] {
518            assert!(foreign_reader(d, Box::new(io::Cursor::new(Vec::new()))).is_err());
519        }
520    }
521
522    fn zip_entry(method: u16, flags: u16, data: &[u8], csize: u32, name: &[u8]) -> Vec<u8> {
523        let mut z = b"PK\x03\x04".to_vec();
524        z.extend(20u16.to_le_bytes());
525        z.extend(flags.to_le_bytes());
526        z.extend(method.to_le_bytes());
527        z.extend([0u8; 4]); // time, date
528        z.extend(0xdeadbeefu32.to_le_bytes());
529        z.extend(csize.to_le_bytes());
530        z.extend(7u32.to_le_bytes());
531        z.extend((name.len() as u16).to_le_bytes());
532        z.extend(2u16.to_le_bytes());
533        z.extend(name);
534        z.extend([0xaa, 0xbb]); // extra
535        z.extend(data);
536        z.extend(b"PK\x01\x02trailing central directory");
537        z
538    }
539
540    #[test]
541    fn zip_first_entry_stored_and_deflated() {
542        let z = zip_entry(0, 0, b"payload", 7, b"a.txt");
543        let e = zip_first_entry(io::Cursor::new(z)).unwrap();
544        assert_eq!(
545            (e.method, e.flags, e.crc, e.compressed_size),
546            (0, 0, 0xdeadbeef, 7)
547        );
548        assert_eq!(e.name, b"a.txt");
549        let mut out = Vec::new();
550        e.data().unwrap().read_to_end(&mut out).unwrap();
551        assert_eq!(out, b"payload");
552
553        let mut enc =
554            flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default());
555        enc.write_all(b"deflated payload").unwrap();
556        let d = enc.finish().unwrap();
557        let z = zip_entry(8, 0, &d, d.len() as u32, b"b");
558        let e = zip_first_entry(io::Cursor::new(z)).unwrap();
559        assert_eq!(e.method, 8);
560        let mut out = Vec::new();
561        e.data().unwrap().read_to_end(&mut out).unwrap();
562        assert_eq!(out, b"deflated payload");
563    }
564
565    #[test]
566    fn zip_first_entry_reports_other_methods_and_truncation() {
567        let z = zip_entry(12, 1, b"x", 1, b"c");
568        let e = zip_first_entry(io::Cursor::new(z)).unwrap();
569        assert_eq!((e.method, e.flags), (12, 1));
570        assert_eq!(e.data().err().unwrap().kind(), io::ErrorKind::Unsupported);
571        let z = zip_entry(0, 0, b"", 0, b"name");
572        let err = zip_first_entry(io::Cursor::new(&z[..20])).err().unwrap();
573        assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof);
574    }
575}
stuffr_cli/compat/gzip/sniff.rs

stuffr_cli/compat/gzip/
sniff.rs
