trg/
sniff.rs

1//! Content-based format detection. There is no `-z` flag by design.
2
3/// One tar block: enough to reach the `ustar` magic at offset 257.
4pub const SNIFF_LEN: usize = 512;
5
6#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7pub enum Format {
8    Gzip,
9    Zstd,
10    Xz,
11    Bzip2,
12    Lz4,
13    /// Unix `compress`, `.Z`.
14    CompressZ,
15    Zip,
16    Tar,
17    Plain,
18}
19
20const LZ4_FRAME: [u8; 4] = [0x04, 0x22, 0x4D, 0x18];
21/// `lz4 -l`, the format the Linux kernel uses. lz4_flex reads it too.
22const LZ4_LEGACY: [u8; 4] = [0x02, 0x21, 0x4C, 0x18];
23/// pi and sqrt(pi), in BCD: bzip2's block and end-of-stream magics.
24const BZ_BLOCK: [u8; 6] = [0x31, 0x41, 0x59, 0x26, 0x53, 0x59];
25const BZ_END: [u8; 6] = [0x17, 0x72, 0x45, 0x38, 0x50, 0x90];
26
27fn is_skippable(b: &[u8]) -> bool {
28    b.len() >= 4 && (0x50..=0x5F).contains(&b[0]) && b[1..4] == [0x2A, 0x4D, 0x18]
29}
30
31/// How many bytes of leading skippable frames — each a 4-byte magic, a 4-byte
32/// little-endian length, then that many bytes — `head` starts with. `None` if
33/// one runs past the end of `head`.
34pub fn skippable_prefix(head: &[u8]) -> Option<usize> {
35    let mut at = 0usize;
36    while is_skippable(&head[at..]) {
37        let len = head.get(at + 4..at + 8)?;
38        let len = u32::from_le_bytes(len.try_into().unwrap()) as usize;
39        at = at.checked_add(8 + len).filter(|&next| next <= head.len())?;
40    }
41    Some(at)
42}
43
44/// The format of the first real frame after the skippable ones.
45fn after_skippable(head: &[u8]) -> Format {
46    let Some(at) = skippable_prefix(head) else { return Format::Zstd };
47    match head.get(at..at + 4) {
48        Some(m) if m == LZ4_FRAME || m == LZ4_LEGACY => Format::Lz4,
49        _ => Format::Zstd,
50    }
51}
52
53/// Classify a buffer by its leading bytes. Order matters: the compressed
54/// formats are checked first, because a compressed stream can contain any byte
55/// sequence at all — including a convincing `ustar` at offset 257.
56pub fn sniff(head: &[u8]) -> Format {
57    if head.len() >= 2 && head[0] == 0x1f && head[1] == 0x8b {
58        return Format::Gzip;
59    }
60    // gzip's neighbour: same first byte, and `gzip -d` reads both.
61    if head.len() >= 2 && head[0] == 0x1f && head[1] == 0x9d {
62        return Format::CompressZ;
63    }
64    // A local file header, or the end-of-central-directory record that is all
65    // an empty zip holds. "PK\x07\x08" is the marker a split archive's first
66    // segment starts with, and a single-segment one is an ordinary zip after it.
67    if head.len() >= 4
68        && head[..2] == *b"PK"
69        && matches!(head[2..4], [3, 4] | [5, 6] | [7, 8])
70    {
71        return Format::Zip;
72    }
73    if head.len() >= 4 && head[..4] == [0x28, 0xB5, 0x2F, 0xFD] {
74        return Format::Zstd;
75    }
76    if head.len() >= 4 && (head[..4] == LZ4_FRAME || head[..4] == LZ4_LEGACY) {
77        return Format::Lz4;
78    }
79    // A skippable frame (magic 0x184D2A50..=0x184D2A5F), which pzstd and
80    // others put before the data frames. zstd and lz4 share the range, so the
81    // frame is stepped over and the format read from what follows it. Taken
82    // for plain text, the archive's compressed bytes were searched as-is for
83    // an exit 1 about data never read, so a skippable frame whose end is past
84    // the sniff window still counts as compressed: as zstd, the format that
85    // writes them in practice.
86    if is_skippable(head) {
87        return after_skippable(head);
88    }
89    if head.len() >= 6 && head[..6] == [0xFD, b'7', b'z', b'X', b'Z', 0x00] {
90        return Format::Xz;
91    }
92    // "BZh", a block size '1'..='9', then the magic of the first block or, for
93    // an empty stream, of the end of stream. Four printable bytes alone could
94    // start a log line; the six binary ones after them cannot.
95    if head.len() >= 10
96        && head[..3] == *b"BZh"
97        && (b'1'..=b'9').contains(&head[3])
98        && (head[4..10] == BZ_BLOCK || head[4..10] == BZ_END)
99    {
100        return Format::Bzip2;
101    }
102    if head.len() >= 262 && &head[257..262] == b"ustar" {
103        return Format::Tar;
104    }
105    Format::Plain
106}
107
108#[cfg(test)]
109mod tests {
110    use super::*;
111
112    #[test]
113    fn detects_gzip_from_magic() {
114        assert_eq!(sniff(&[0x1f, 0x8b, 0x08, 0x00]), Format::Gzip);
115    }
116
117    #[test]
118    fn detects_ustar_tar_at_offset_257() {
119        let mut b = vec![0u8; 512];
120        b[257..262].copy_from_slice(b"ustar");
121        assert_eq!(sniff(&b), Format::Tar);
122    }
123
124    #[test]
125    fn detects_gnu_tar_variant() {
126        let mut b = vec![0u8; 512];
127        b[257..265].copy_from_slice(b"ustar  \0");
128        assert_eq!(sniff(&b), Format::Tar);
129    }
130
131    #[test]
132    fn detects_zstd_from_magic() {
133        assert_eq!(sniff(&[0x28, 0xB5, 0x2F, 0xFD, 0x00]), Format::Zstd);
134    }
135
136    /// pzstd and other tools start with a skippable frame (magic
137    /// 0x184D2A50..=0x184D2A5F, little-endian) before any data frame. Taken
138    /// for plain text, the archive's compressed bytes were "searched" and the
139    /// run exited 1: clean logs, from data never read.
140    #[test]
141    fn a_skippable_frame_is_zstd() {
142        for low in 0x50..=0x5F {
143            assert_eq!(sniff(&[low, 0x2A, 0x4D, 0x18, 4, 0, 0, 0]), Format::Zstd, "magic {low:#x}");
144        }
145        assert_eq!(sniff(&[0x60, 0x2A, 0x4D, 0x18, 4, 0, 0, 0]), Format::Plain, "just past the range");
146    }
147
148    #[test]
149    fn zstd_magic_needs_all_four_bytes() {
150        // A short prefix must not be mistaken for a zstd frame.
151        assert_eq!(sniff(&[0x28, 0xB5, 0x2F]), Format::Plain);
152    }
153
154    #[test]
155    fn zstd_wins_over_a_coincidental_ustar() {
156        let mut b = vec![0u8; 512];
157        b[..4].copy_from_slice(&[0x28, 0xB5, 0x2F, 0xFD]);
158        b[257..262].copy_from_slice(b"ustar");
159        assert_eq!(sniff(&b), Format::Zstd);
160    }
161
162    #[test]
163    fn detects_xz_from_magic() {
164        assert_eq!(sniff(&[0xFD, b'7', b'z', b'X', b'Z', 0x00, 0x00]), Format::Xz);
165    }
166
167    #[test]
168    fn xz_magic_needs_all_six_bytes() {
169        assert_eq!(sniff(&[0xFD, b'7', b'z', b'X', b'Z']), Format::Plain);
170    }
171
172    #[test]
173    fn xz_wins_over_a_coincidental_ustar() {
174        let mut b = vec![0u8; 512];
175        b[..6].copy_from_slice(&[0xFD, b'7', b'z', b'X', b'Z', 0x00]);
176        b[257..262].copy_from_slice(b"ustar");
177        assert_eq!(sniff(&b), Format::Xz);
178    }
179
180    fn bz2_head(level: u8) -> Vec<u8> {
181        let mut b = b"BZh".to_vec();
182        b.push(level);
183        b.extend_from_slice(&BZ_BLOCK);
184        b
185    }
186
187    #[test]
188    fn detects_bzip2_from_magic() {
189        for level in b'1'..=b'9' {
190            assert_eq!(sniff(&bz2_head(level)), Format::Bzip2, "BZh{}", level as char);
191        }
192    }
193
194    /// `bzip2 < /dev/null`: no block, straight to the end-of-stream magic.
195    #[test]
196    fn an_empty_bzip2_stream_is_bzip2() {
197        let mut b = b"BZh9".to_vec();
198        b.extend_from_slice(&BZ_END);
199        assert_eq!(sniff(&b), Format::Bzip2);
200    }
201
202    /// "BZh" is three printable letters. A log line starting with "BZh9" must
203    /// stay text.
204    #[test]
205    fn bzip2_needs_more_than_its_printable_prefix() {
206        assert_eq!(sniff(b"BZh9 is not a block size\n"), Format::Plain);
207        assert_eq!(sniff(b"BZh0"), Format::Plain);
208        let mut zero = b"BZh0".to_vec();
209        zero.extend_from_slice(&BZ_BLOCK);
210        assert_eq!(sniff(&zero), Format::Plain, "level 0 does not exist");
211    }
212
213    #[test]
214    fn bzip2_wins_over_a_coincidental_ustar() {
215        let mut b = vec![0u8; 512];
216        b[..10].copy_from_slice(&bz2_head(b'9'));
217        b[257..262].copy_from_slice(b"ustar");
218        assert_eq!(sniff(&b), Format::Bzip2);
219    }
220
221    #[test]
222    fn detects_lz4_frame_and_legacy_magics() {
223        assert_eq!(sniff(&[0x04, 0x22, 0x4D, 0x18, 0x64]), Format::Lz4);
224        assert_eq!(sniff(&[0x02, 0x21, 0x4C, 0x18, 0x00]), Format::Lz4);
225        assert_eq!(sniff(&[0x04, 0x22, 0x4D]), Format::Plain, "needs all four bytes");
226    }
227
228    #[test]
229    fn lz4_wins_over_a_coincidental_ustar() {
230        let mut b = vec![0u8; 512];
231        b[..4].copy_from_slice(&LZ4_FRAME);
232        b[257..262].copy_from_slice(b"ustar");
233        assert_eq!(sniff(&b), Format::Lz4);
234    }
235
236    /// lz4 shares zstd's skippable-frame range, so the frame is stepped over
237    /// and the format read from the next magic.
238    #[test]
239    fn a_skippable_frame_before_lz4_is_lz4() {
240        let mut b = vec![0x50, 0x2A, 0x4D, 0x18, 4, 0, 0, 0, 1, 2, 3, 4];
241        b.extend_from_slice(&LZ4_FRAME);
242        assert_eq!(sniff(&b), Format::Lz4);
243    }
244
245    #[test]
246    fn a_skippable_frame_before_zstd_is_still_zstd() {
247        let mut b = vec![0x5F, 0x2A, 0x4D, 0x18, 0, 0, 0, 0];
248        b.extend_from_slice(&[0x50, 0x2A, 0x4D, 0x18, 2, 0, 0, 0, 9, 9]);
249        b.extend_from_slice(&[0x28, 0xB5, 0x2F, 0xFD]);
250        assert_eq!(sniff(&b), Format::Zstd, "two skippable frames, then zstd");
251    }
252
253    /// A skippable frame longer than the sniff window hides what follows. It
254    /// is compressed either way, and must never fall through to Plain.
255    #[test]
256    fn a_skippable_frame_past_the_window_is_zstd_not_plain() {
257        let mut b = vec![0x50, 0x2A, 0x4D, 0x18, 0xFF, 0xFF, 0xFF, 0xFF];
258        b.resize(SNIFF_LEN, 0);
259        assert_eq!(sniff(&b), Format::Zstd);
260        assert_eq!(sniff(&[0x50, 0x2A, 0x4D, 0x18, 4]), Format::Zstd, "truncated length");
261    }
262
263    #[test]
264    fn detects_compress_z_from_magic() {
265        assert_eq!(sniff(&[0x1f, 0x9d, 0x90]), Format::CompressZ);
266        assert_eq!(sniff(&[0x1f]), Format::Plain);
267    }
268
269    #[test]
270    fn detects_zip_from_its_record_signatures() {
271        assert_eq!(sniff(b"PK\x03\x04\x14\x00"), Format::Zip, "local file header");
272        assert_eq!(sniff(b"PK\x05\x06\x00\x00"), Format::Zip, "empty zip");
273        assert_eq!(sniff(b"PK\x07\x08PK\x03\x04"), Format::Zip, "split marker");
274        assert_eq!(sniff(b"PK is a log line\n"), Format::Plain);
275        assert_eq!(sniff(b"PK\x01\x02"), Format::Plain, "a central record does not start a file");
276    }
277
278    #[test]
279    fn zip_wins_over_a_coincidental_ustar() {
280        let mut b = vec![0u8; 512];
281        b[..4].copy_from_slice(b"PK\x03\x04");
282        b[257..262].copy_from_slice(b"ustar");
283        assert_eq!(sniff(&b), Format::Zip);
284    }
285
286    #[test]
287    fn plain_text_is_plain() {
288        assert_eq!(sniff(b"2026-08-24T00:00:00Z GET /index.php\n"), Format::Plain);
289    }
290
291    #[test]
292    fn short_input_is_plain_not_panic() {
293        assert_eq!(sniff(b""), Format::Plain);
294        assert_eq!(sniff(&[0x1f]), Format::Plain);
295    }
296
297    #[test]
298    fn gzip_wins_over_a_coincidental_ustar() {
299        // a gzip stream whose bytes happen to contain "ustar" at 257
300        let mut b = vec![0u8; 512];
301        b[0] = 0x1f; b[1] = 0x8b;
302        b[257..262].copy_from_slice(b"ustar");
303        assert_eq!(sniff(&b), Format::Gzip);
304    }
305}
trg/sniff.rs

trg/
sniff.rs
