trg/
sniff.rs

1//! Content-based format detection. There is no `-z` flag by design.
2
3/// One tar block: enough to reach the `ustar` magic at offset 257.
4pub const SNIFF_LEN: usize = 512;
5
6#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7pub enum Format {
8    Gzip,
9    Zstd,
10    Xz,
11    Bzip2,
12    Lz4,
13    Tar,
14    Plain,
15}
16
17const LZ4_FRAME: [u8; 4] = [0x04, 0x22, 0x4D, 0x18];
18/// `lz4 -l`, the format the Linux kernel uses. lz4_flex reads it too.
19const LZ4_LEGACY: [u8; 4] = [0x02, 0x21, 0x4C, 0x18];
20/// pi and sqrt(pi), in BCD: bzip2's block and end-of-stream magics.
21const BZ_BLOCK: [u8; 6] = [0x31, 0x41, 0x59, 0x26, 0x53, 0x59];
22const BZ_END: [u8; 6] = [0x17, 0x72, 0x45, 0x38, 0x50, 0x90];
23
24fn is_skippable(b: &[u8]) -> bool {
25    b.len() >= 4 && (0x50..=0x5F).contains(&b[0]) && b[1..4] == [0x2A, 0x4D, 0x18]
26}
27
28/// How many bytes of leading skippable frames — each a 4-byte magic, a 4-byte
29/// little-endian length, then that many bytes — `head` starts with. `None` if
30/// one runs past the end of `head`.
31pub fn skippable_prefix(head: &[u8]) -> Option<usize> {
32    let mut at = 0usize;
33    while is_skippable(&head[at..]) {
34        let len = head.get(at + 4..at + 8)?;
35        let len = u32::from_le_bytes(len.try_into().unwrap()) as usize;
36        at = at.checked_add(8 + len).filter(|&next| next <= head.len())?;
37    }
38    Some(at)
39}
40
41/// The format of the first real frame after the skippable ones.
42fn after_skippable(head: &[u8]) -> Format {
43    let Some(at) = skippable_prefix(head) else { return Format::Zstd };
44    match head.get(at..at + 4) {
45        Some(m) if m == LZ4_FRAME || m == LZ4_LEGACY => Format::Lz4,
46        _ => Format::Zstd,
47    }
48}
49
50/// Classify a buffer by its leading bytes. Order matters: the compressed
51/// formats are checked first, because a compressed stream can contain any byte
52/// sequence at all — including a convincing `ustar` at offset 257.
53pub fn sniff(head: &[u8]) -> Format {
54    if head.len() >= 2 && head[0] == 0x1f && head[1] == 0x8b {
55        return Format::Gzip;
56    }
57    if head.len() >= 4 && head[..4] == [0x28, 0xB5, 0x2F, 0xFD] {
58        return Format::Zstd;
59    }
60    if head.len() >= 4 && (head[..4] == LZ4_FRAME || head[..4] == LZ4_LEGACY) {
61        return Format::Lz4;
62    }
63    // A skippable frame (magic 0x184D2A50..=0x184D2A5F), which pzstd and
64    // others put before the data frames. zstd and lz4 share the range, so the
65    // frame is stepped over and the format read from what follows it. Taken
66    // for plain text, the archive's compressed bytes were searched as-is for
67    // an exit 1 about data never read, so a skippable frame whose end is past
68    // the sniff window still counts as compressed: as zstd, the format that
69    // writes them in practice.
70    if is_skippable(head) {
71        return after_skippable(head);
72    }
73    if head.len() >= 6 && head[..6] == [0xFD, b'7', b'z', b'X', b'Z', 0x00] {
74        return Format::Xz;
75    }
76    // "BZh", a block size '1'..='9', then the magic of the first block or, for
77    // an empty stream, of the end of stream. Four printable bytes alone could
78    // start a log line; the six binary ones after them cannot.
79    if head.len() >= 10
80        && head[..3] == *b"BZh"
81        && (b'1'..=b'9').contains(&head[3])
82        && (head[4..10] == BZ_BLOCK || head[4..10] == BZ_END)
83    {
84        return Format::Bzip2;
85    }
86    if head.len() >= 262 && &head[257..262] == b"ustar" {
87        return Format::Tar;
88    }
89    Format::Plain
90}
91
92#[cfg(test)]
93mod tests {
94    use super::*;
95
96    #[test]
97    fn detects_gzip_from_magic() {
98        assert_eq!(sniff(&[0x1f, 0x8b, 0x08, 0x00]), Format::Gzip);
99    }
100
101    #[test]
102    fn detects_ustar_tar_at_offset_257() {
103        let mut b = vec![0u8; 512];
104        b[257..262].copy_from_slice(b"ustar");
105        assert_eq!(sniff(&b), Format::Tar);
106    }
107
108    #[test]
109    fn detects_gnu_tar_variant() {
110        let mut b = vec![0u8; 512];
111        b[257..265].copy_from_slice(b"ustar  \0");
112        assert_eq!(sniff(&b), Format::Tar);
113    }
114
115    #[test]
116    fn detects_zstd_from_magic() {
117        assert_eq!(sniff(&[0x28, 0xB5, 0x2F, 0xFD, 0x00]), Format::Zstd);
118    }
119
120    /// pzstd and other tools start with a skippable frame (magic
121    /// 0x184D2A50..=0x184D2A5F, little-endian) before any data frame. Taken
122    /// for plain text, the archive's compressed bytes were "searched" and the
123    /// run exited 1: clean logs, from data never read.
124    #[test]
125    fn a_skippable_frame_is_zstd() {
126        for low in 0x50..=0x5F {
127            assert_eq!(sniff(&[low, 0x2A, 0x4D, 0x18, 4, 0, 0, 0]), Format::Zstd, "magic {low:#x}");
128        }
129        assert_eq!(sniff(&[0x60, 0x2A, 0x4D, 0x18, 4, 0, 0, 0]), Format::Plain, "just past the range");
130    }
131
132    #[test]
133    fn zstd_magic_needs_all_four_bytes() {
134        // A short prefix must not be mistaken for a zstd frame.
135        assert_eq!(sniff(&[0x28, 0xB5, 0x2F]), Format::Plain);
136    }
137
138    #[test]
139    fn zstd_wins_over_a_coincidental_ustar() {
140        let mut b = vec![0u8; 512];
141        b[..4].copy_from_slice(&[0x28, 0xB5, 0x2F, 0xFD]);
142        b[257..262].copy_from_slice(b"ustar");
143        assert_eq!(sniff(&b), Format::Zstd);
144    }
145
146    #[test]
147    fn detects_xz_from_magic() {
148        assert_eq!(sniff(&[0xFD, b'7', b'z', b'X', b'Z', 0x00, 0x00]), Format::Xz);
149    }
150
151    #[test]
152    fn xz_magic_needs_all_six_bytes() {
153        assert_eq!(sniff(&[0xFD, b'7', b'z', b'X', b'Z']), Format::Plain);
154    }
155
156    #[test]
157    fn xz_wins_over_a_coincidental_ustar() {
158        let mut b = vec![0u8; 512];
159        b[..6].copy_from_slice(&[0xFD, b'7', b'z', b'X', b'Z', 0x00]);
160        b[257..262].copy_from_slice(b"ustar");
161        assert_eq!(sniff(&b), Format::Xz);
162    }
163
164    fn bz2_head(level: u8) -> Vec<u8> {
165        let mut b = b"BZh".to_vec();
166        b.push(level);
167        b.extend_from_slice(&BZ_BLOCK);
168        b
169    }
170
171    #[test]
172    fn detects_bzip2_from_magic() {
173        for level in b'1'..=b'9' {
174            assert_eq!(sniff(&bz2_head(level)), Format::Bzip2, "BZh{}", level as char);
175        }
176    }
177
178    /// `bzip2 < /dev/null`: no block, straight to the end-of-stream magic.
179    #[test]
180    fn an_empty_bzip2_stream_is_bzip2() {
181        let mut b = b"BZh9".to_vec();
182        b.extend_from_slice(&BZ_END);
183        assert_eq!(sniff(&b), Format::Bzip2);
184    }
185
186    /// "BZh" is three printable letters. A log line starting with "BZh9" must
187    /// stay text.
188    #[test]
189    fn bzip2_needs_more_than_its_printable_prefix() {
190        assert_eq!(sniff(b"BZh9 is not a block size\n"), Format::Plain);
191        assert_eq!(sniff(b"BZh0"), Format::Plain);
192        let mut zero = b"BZh0".to_vec();
193        zero.extend_from_slice(&BZ_BLOCK);
194        assert_eq!(sniff(&zero), Format::Plain, "level 0 does not exist");
195    }
196
197    #[test]
198    fn bzip2_wins_over_a_coincidental_ustar() {
199        let mut b = vec![0u8; 512];
200        b[..10].copy_from_slice(&bz2_head(b'9'));
201        b[257..262].copy_from_slice(b"ustar");
202        assert_eq!(sniff(&b), Format::Bzip2);
203    }
204
205    #[test]
206    fn detects_lz4_frame_and_legacy_magics() {
207        assert_eq!(sniff(&[0x04, 0x22, 0x4D, 0x18, 0x64]), Format::Lz4);
208        assert_eq!(sniff(&[0x02, 0x21, 0x4C, 0x18, 0x00]), Format::Lz4);
209        assert_eq!(sniff(&[0x04, 0x22, 0x4D]), Format::Plain, "needs all four bytes");
210    }
211
212    #[test]
213    fn lz4_wins_over_a_coincidental_ustar() {
214        let mut b = vec![0u8; 512];
215        b[..4].copy_from_slice(&LZ4_FRAME);
216        b[257..262].copy_from_slice(b"ustar");
217        assert_eq!(sniff(&b), Format::Lz4);
218    }
219
220    /// lz4 shares zstd's skippable-frame range, so the frame is stepped over
221    /// and the format read from the next magic.
222    #[test]
223    fn a_skippable_frame_before_lz4_is_lz4() {
224        let mut b = vec![0x50, 0x2A, 0x4D, 0x18, 4, 0, 0, 0, 1, 2, 3, 4];
225        b.extend_from_slice(&LZ4_FRAME);
226        assert_eq!(sniff(&b), Format::Lz4);
227    }
228
229    #[test]
230    fn a_skippable_frame_before_zstd_is_still_zstd() {
231        let mut b = vec![0x5F, 0x2A, 0x4D, 0x18, 0, 0, 0, 0];
232        b.extend_from_slice(&[0x50, 0x2A, 0x4D, 0x18, 2, 0, 0, 0, 9, 9]);
233        b.extend_from_slice(&[0x28, 0xB5, 0x2F, 0xFD]);
234        assert_eq!(sniff(&b), Format::Zstd, "two skippable frames, then zstd");
235    }
236
237    /// A skippable frame longer than the sniff window hides what follows. It
238    /// is compressed either way, and must never fall through to Plain.
239    #[test]
240    fn a_skippable_frame_past_the_window_is_zstd_not_plain() {
241        let mut b = vec![0x50, 0x2A, 0x4D, 0x18, 0xFF, 0xFF, 0xFF, 0xFF];
242        b.resize(SNIFF_LEN, 0);
243        assert_eq!(sniff(&b), Format::Zstd);
244        assert_eq!(sniff(&[0x50, 0x2A, 0x4D, 0x18, 4]), Format::Zstd, "truncated length");
245    }
246
247    #[test]
248    fn plain_text_is_plain() {
249        assert_eq!(sniff(b"2026-08-24T00:00:00Z GET /index.php\n"), Format::Plain);
250    }
251
252    #[test]
253    fn short_input_is_plain_not_panic() {
254        assert_eq!(sniff(b""), Format::Plain);
255        assert_eq!(sniff(&[0x1f]), Format::Plain);
256    }
257
258    #[test]
259    fn gzip_wins_over_a_coincidental_ustar() {
260        // a gzip stream whose bytes happen to contain "ustar" at 257
261        let mut b = vec![0u8; 512];
262        b[0] = 0x1f; b[1] = 0x8b;
263        b[257..262].copy_from_slice(b"ustar");
264        assert_eq!(sniff(&b), Format::Gzip);
265    }
266}
trg/sniff.rs

trg/
sniff.rs
