blob_decoder/
identify.rs

1//! The orchestration engine: identify → dispatch → score → recursively unwrap.
2//!
3//! Every detector here is a thin dispatcher over a mature crate (`plist`,
4//! `flate2`, `snap`, `base64`, `hex`, `uuid`, `serde_json`) — this module owns
5//! only the *identification*, *scoring*, and *recursive unwrap*, never the codec
6//! itself. All input is attacker-controllable, so the invariant is: no panic, no
7//! OOM (every decompression is size-capped and the recursion is depth-capped),
8//! and every failure degrades to a lower-confidence or absent reading.
9
10use std::io::{Cursor, Read};
11
12use base64::Engine as _;
13
14use crate::{BlobKind, Candidate, Confidence, DecodedChain, Limits};
15
16/// Identify a blob with default [`Limits`]. Returns scored candidates, best
17/// (highest [`Confidence`]) first. Always returns at least one candidate (an
18/// [`BlobKind::Unknown`] reading that surfaces the raw head bytes when nothing
19/// else matched).
20#[must_use]
21pub fn identify(bytes: &[u8]) -> Vec<Candidate> {
22    identify_with_limits(bytes, Limits::default(), 0)
23}
24
25/// Identify a blob with explicit resource [`Limits`] and a starting recursion
26/// `depth` — the entry point the recursive unwrap calls into.
27#[must_use]
28pub fn identify_with_limits(bytes: &[u8], limits: Limits, depth: usize) -> Vec<Candidate> {
29    let mut out: Vec<Candidate> = Vec::new();
30
31    // Strong magic / full-parse detectors always run (cheap, high-signal).
32    push(&mut out, detect_binary_plist(bytes));
33    push(&mut out, detect_xml_plist(bytes));
34    push(&mut out, detect_gzip(bytes, limits, depth));
35    push(&mut out, detect_zlib(bytes, limits, depth));
36    push(&mut out, detect_snappy(bytes, limits, depth));
37    push(&mut out, detect_json(bytes));
38    push(&mut out, detect_uuid_string(bytes));
39
40    // Heuristic detectors are bounded by max_input (they scan / decode the whole
41    // blob), so huge inputs get magic-only identification.
42    if bytes.len() <= limits.max_input {
43        push(&mut out, detect_base64(bytes, limits, depth));
44        push(&mut out, detect_hex(bytes, limits, depth));
45        push(&mut out, detect_uuid_bytes(bytes));
46        push(&mut out, detect_utf16le(bytes));
47        push(&mut out, detect_utf8_text(bytes));
48    }
49
50    if out.is_empty() {
51        out.push(unknown(bytes));
52    }
53
54    out.sort_by(|a, b| {
55        b.score
56            .cmp(&a.score)
57            .then_with(|| kind_rank(b.kind).cmp(&kind_rank(a.kind)))
58            .then_with(|| a.kind.label().cmp(b.kind.label()))
59    });
60    out
61}
62
63fn push(out: &mut Vec<Candidate>, c: Option<Candidate>) {
64    if let Some(c) = c {
65        out.push(c);
66    }
67}
68
69/// Specificity tiebreak when two candidates share a [`Confidence`]: a concrete
70/// magic-identified type outranks a generic wrapper, which outranks bare text.
71fn kind_rank(kind: BlobKind) -> u8 {
72    match kind {
73        BlobKind::BinaryPlist
74        | BlobKind::XmlPlist
75        | BlobKind::Gzip
76        | BlobKind::Zlib
77        | BlobKind::Snappy
78        | BlobKind::Json
79        | BlobKind::Uuid => 3,
80        BlobKind::Base64 | BlobKind::Hex => 2,
81        BlobKind::Utf16Le | BlobKind::Utf8Text => 1,
82        BlobKind::Unknown => 0,
83    }
84}
85
86// ---------------------------------------------------------------------------
87// Wrapper payload recursion
88// ---------------------------------------------------------------------------
89
90/// Build the [`DecodedChain`] for a wrapper's decoded payload: recurse to
91/// identify it, unless the depth cap is reached (the DoS backstop for
92/// infinitely-nested wrappers).
93fn build_chain(data: &[u8], capped: bool, limits: Limits, depth: usize) -> DecodedChain {
94    let best = if depth + 1 >= limits.max_depth {
95        Box::new(depth_capped(data))
96    } else {
97        identify_with_limits(data, limits, depth + 1)
98            .into_iter()
99            .next()
100            // cov:unreachable: identify_with_limits never returns empty (pushes Unknown), kept defensive fallback.
101            .map_or_else(|| Box::new(unknown(data)), Box::new)
102    };
103    DecodedChain {
104        decoded_len: data.len(),
105        capped,
106        best,
107    }
108}
109
110fn depth_capped(data: &[u8]) -> Candidate {
111    Candidate {
112        kind: BlobKind::Unknown,
113        score: Confidence::Low,
114        summary: format!(
115            "recursion depth cap reached; {} bytes not further decoded (head: {})",
116            data.len(),
117            head_hex(data)
118        ),
119        citation: BlobKind::Unknown.citation(),
120        inner: None,
121    }
122}
123
124// ---------------------------------------------------------------------------
125// Bounded decompression (the decompression-bomb guard)
126// ---------------------------------------------------------------------------
127
128/// Read at most `cap` bytes from `r`; return the bytes and whether the stream
129/// was *capped* (had more to give). Bounds memory to `cap`, so a decompression
130/// bomb can never exhaust it.
131fn bounded_read<R: Read>(r: R, cap: usize) -> std::io::Result<(Vec<u8>, bool)> {
132    let mut out = Vec::new();
133    // take(cap+1): if we get cap+1 bytes the stream had more → it was capped.
134    r.take(cap as u64 + 1).read_to_end(&mut out)?;
135    let capped = out.len() > cap;
136    if capped {
137        out.truncate(cap);
138    }
139    Ok((out, capped))
140}
141
142// ---------------------------------------------------------------------------
143// Detectors — strong magic / full parse
144// ---------------------------------------------------------------------------
145
146fn detect_binary_plist(bytes: &[u8]) -> Option<Candidate> {
147    if !bytes.starts_with(b"bplist") {
148        return None;
149    }
150    Some(match plist::Value::from_reader(Cursor::new(bytes)) {
151        Ok(v) => leaf(
152            BlobKind::BinaryPlist,
153            Confidence::High,
154            format!("binary plist: {}", describe_plist(&v)),
155        ),
156        Err(e) => leaf(
157            BlobKind::BinaryPlist,
158            Confidence::Medium,
159            format!("bplist magic but parse failed: {e}"),
160        ),
161    })
162}
163
164fn detect_xml_plist(bytes: &[u8]) -> Option<Candidate> {
165    let head = bytes.trim_ascii_start();
166    // Bounded prefix scan for the plist markers, so a huge non-plist XML body is
167    // rejected cheaply. Require the `plist` token to distinguish it from any XML.
168    let probe = &head[..head.len().min(1024)];
169    let starts_xml = probe.starts_with(b"<?xml")
170        || probe.starts_with(b"<plist")
171        || probe.starts_with(b"<!DOCTYPE");
172    if !starts_xml || !contains(probe, b"plist") {
173        return None;
174    }
175    Some(match plist::Value::from_reader(Cursor::new(bytes)) {
176        Ok(v) => leaf(
177            BlobKind::XmlPlist,
178            Confidence::High,
179            format!("XML plist: {}", describe_plist(&v)),
180        ),
181        Err(e) => leaf(
182            BlobKind::XmlPlist,
183            Confidence::Medium,
184            format!("XML plist markup but parse failed: {e}"),
185        ),
186    })
187}
188
189fn detect_gzip(bytes: &[u8], limits: Limits, depth: usize) -> Option<Candidate> {
190    // RFC 1952: magic 1f 8b. A 2-byte magic is specific enough that a decode
191    // failure is still worth reporting (as a truncated/corrupt gzip).
192    if !bytes.starts_with(&[0x1f, 0x8b]) {
193        return None;
194    }
195    match bounded_read(flate2::read::GzDecoder::new(bytes), limits.max_output) {
196        Ok((data, capped)) => Some(wrapper(
197            BlobKind::Gzip,
198            Confidence::High,
199            format!(
200                "gzip stream; {} bytes decompressed{}",
201                data.len(),
202                if capped { " (capped at limit)" } else { "" }
203            ),
204            build_chain(&data, capped, limits, depth),
205        )),
206        Err(e) => Some(leaf(
207            BlobKind::Gzip,
208            Confidence::Medium,
209            format!("gzip magic but decompression failed: {e}"),
210        )),
211    }
212}
213
214fn detect_zlib(bytes: &[u8], limits: Limits, depth: usize) -> Option<Candidate> {
215    // RFC 1950 header has no unique magic — only CM=deflate + the FCHECK mod-31
216    // constraint (a weak ~1/500 filter). So we claim zlib ONLY on a SUCCESSFUL
217    // decompress; a header match that fails to inflate is treated as coincidence
218    // (returns None), never a false Medium on random bytes.
219    if bytes.len() < 2 {
220        return None;
221    }
222    let (cmf, flg) = (bytes[0], bytes[1]);
223    if cmf & 0x0f != 0x08 || cmf >> 4 > 7 {
224        return None;
225    }
226    if !((u16::from(cmf) << 8) | u16::from(flg)).is_multiple_of(31) {
227        return None;
228    }
229    let (data, capped) =
230        bounded_read(flate2::read::ZlibDecoder::new(bytes), limits.max_output).ok()?;
231    Some(wrapper(
232        BlobKind::Zlib,
233        Confidence::High,
234        format!(
235            "zlib stream; {} bytes decompressed{}",
236            data.len(),
237            if capped { " (capped at limit)" } else { "" }
238        ),
239        build_chain(&data, capped, limits, depth),
240    ))
241}
242
243fn detect_snappy(bytes: &[u8], limits: Limits, depth: usize) -> Option<Candidate> {
244    // Snappy framing format: the stream identifier chunk (0xff + "sNaPpY").
245    const MAGIC: &[u8] = &[0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59];
246    if !bytes.starts_with(MAGIC) {
247        return None;
248    }
249    match bounded_read(snap::read::FrameDecoder::new(bytes), limits.max_output) {
250        Ok((data, capped)) => Some(wrapper(
251            BlobKind::Snappy,
252            Confidence::High,
253            format!(
254                "Snappy framed stream; {} bytes decompressed{}",
255                data.len(),
256                if capped { " (capped at limit)" } else { "" }
257            ),
258            build_chain(&data, capped, limits, depth),
259        )),
260        Err(e) => Some(leaf(
261            BlobKind::Snappy,
262            Confidence::Medium,
263            format!("Snappy magic but decompression failed: {e}"),
264        )),
265    }
266}
267
268fn detect_json(bytes: &[u8]) -> Option<Candidate> {
269    let trimmed = bytes.trim_ascii();
270    // Only object/array roots are claimed as JSON: a bare `123` or `true` is
271    // technically JSON but too ambiguous to assert.
272    if !matches!(trimmed.first(), Some(b'{' | b'[')) {
273        return None;
274    }
275    let value: serde_json::Value = serde_json::from_slice(trimmed).ok()?;
276    Some(leaf(
277        BlobKind::Json,
278        Confidence::High,
279        describe_json(&value),
280    ))
281}
282
283fn detect_uuid_string(bytes: &[u8]) -> Option<Candidate> {
284    let s = std::str::from_utf8(bytes).ok()?.trim();
285    // Require the hyphenated (or braced/urn) canonical form: a bare 32-hex run is
286    // better read as hex, so we do not claim it here.
287    if !s.contains('-') {
288        return None;
289    }
290    let u = uuid::Uuid::try_parse(s).ok()?;
291    Some(leaf(
292        BlobKind::Uuid,
293        Confidence::High,
294        format!(
295            "UUID {u} (version {}, variant {:?})",
296            u.get_version_num(),
297            u.get_variant()
298        ),
299    ))
300}
301
302// ---------------------------------------------------------------------------
303// Detectors — heuristic (coincidence-prone, scored Low unless the payload is
304// itself recognised)
305// ---------------------------------------------------------------------------
306
307fn detect_base64(bytes: &[u8], limits: Limits, depth: usize) -> Option<Candidate> {
308    let decoded = try_base64(bytes)?;
309    let chain = build_chain(&decoded, false, limits, depth);
310    let score = wrapper_score(chain.best.kind);
311    Some(wrapper(
312        BlobKind::Base64,
313        score,
314        format!("base64 text; decodes to {} bytes", chain.decoded_len),
315        chain,
316    ))
317}
318
319fn detect_hex(bytes: &[u8], limits: Limits, depth: usize) -> Option<Candidate> {
320    let s = bytes.trim_ascii();
321    if s.len() < 4 || !s.len().is_multiple_of(2) || !s.iter().all(u8::is_ascii_hexdigit) {
322        return None;
323    }
324    let decoded = hex::decode(s).ok()?;
325    let chain = build_chain(&decoded, false, limits, depth);
326    let score = wrapper_score(chain.best.kind);
327    Some(wrapper(
328        BlobKind::Hex,
329        score,
330        format!("hexadecimal text; decodes to {} bytes", chain.decoded_len),
331        chain,
332    ))
333}
334
335fn detect_uuid_bytes(bytes: &[u8]) -> Option<Candidate> {
336    let arr: [u8; 16] = bytes.try_into().ok()?;
337    let u = uuid::Uuid::from_bytes(arr);
338    Some(leaf(
339        BlobKind::Uuid,
340        // Any 16 bytes form a syntactically valid UUID — never over-claim.
341        Confidence::Low,
342        format!("if a UUID: {u} (note: any 16 bytes form a valid UUID)"),
343    ))
344}
345
346fn detect_utf16le(bytes: &[u8]) -> Option<Candidate> {
347    if bytes.len() < 4 || !bytes.len().is_multiple_of(2) {
348        return None;
349    }
350    let has_bom = bytes.starts_with(&[0xff, 0xfe]);
351    let body = if has_bom { &bytes[2..] } else { bytes };
352    let units: Vec<u16> = body
353        .chunks_exact(2)
354        .map(|c| u16::from_le_bytes([c[0], c[1]]))
355        .collect();
356    if units.is_empty() {
357        return None;
358    }
359    let text = String::from_utf16(&units).ok()?;
360    if !mostly_printable(&text) {
361        return None;
362    }
363    // Without a BOM, require the ASCII-plane dominance that real UTF-16LE text
364    // shows (high byte zero), else even-length binary would masquerade as text.
365    let ascii_plane = body.chunks_exact(2).filter(|c| c[1] == 0).count();
366    if !has_bom && (ascii_plane * 2) < units.len() {
367        return None;
368    }
369    Some(leaf(
370        BlobKind::Utf16Le,
371        if has_bom {
372            Confidence::Medium
373        } else {
374            Confidence::Low
375        },
376        format!("UTF-16LE text preview: \"{}\"", preview(&text)),
377    ))
378}
379
380fn detect_utf8_text(bytes: &[u8]) -> Option<Candidate> {
381    let s = std::str::from_utf8(bytes).ok()?;
382    if s.is_empty() || !mostly_printable(s) {
383        return None;
384    }
385    Some(leaf(
386        BlobKind::Utf8Text,
387        Confidence::Low,
388        format!("UTF-8 text preview: \"{}\"", preview(s)),
389    ))
390}
391
392// ---------------------------------------------------------------------------
393// Scoring / construction helpers
394// ---------------------------------------------------------------------------
395
396/// A wrapper (base64/hex) whose decoded payload is itself a concrete recognised
397/// type is Medium; one that decodes only to opaque bytes or plain text is Low
398/// (the decode was probably coincidental).
399fn wrapper_score(inner: BlobKind) -> Confidence {
400    match inner {
401        BlobKind::Unknown | BlobKind::Utf8Text | BlobKind::Utf16Le => Confidence::Low,
402        _ => Confidence::Medium,
403    }
404}
405
406fn leaf(kind: BlobKind, score: Confidence, summary: String) -> Candidate {
407    Candidate {
408        kind,
409        score,
410        summary,
411        citation: kind.citation(),
412        inner: None,
413    }
414}
415
416fn wrapper(kind: BlobKind, score: Confidence, summary: String, chain: DecodedChain) -> Candidate {
417    Candidate {
418        kind,
419        score,
420        summary,
421        citation: kind.citation(),
422        inner: Some(Box::new(chain)),
423    }
424}
425
426fn unknown(bytes: &[u8]) -> Candidate {
427    Candidate {
428        kind: BlobKind::Unknown,
429        score: Confidence::Low,
430        summary: if bytes.is_empty() {
431            "unrecognized: empty input".to_owned()
432        } else {
433            format!(
434                "unrecognized; {} bytes (head: {})",
435                bytes.len(),
436                head_hex(bytes)
437            )
438        },
439        citation: BlobKind::Unknown.citation(),
440        inner: None,
441    }
442}
443
444/// Validate + decode base64 (standard or URL-safe, whitespace-tolerant). Returns
445/// the decoded bytes, or `None` if the input is not clean, correctly-padded
446/// base64 — delegating the decode itself to the `base64` crate.
447fn try_base64(bytes: &[u8]) -> Option<Vec<u8>> {
448    let cleaned: Vec<u8> = bytes
449        .iter()
450        .copied()
451        .filter(|b| !b.is_ascii_whitespace())
452        .collect();
453    if cleaned.len() < 8 || !cleaned.len().is_multiple_of(4) {
454        return None;
455    }
456    let eq = cleaned
457        .iter()
458        .position(|&b| b == b'=')
459        .unwrap_or(cleaned.len());
460    let (body, padding) = cleaned.split_at(eq);
461    if padding.len() > 2 || padding.iter().any(|&b| b != b'=') || body.is_empty() {
462        return None;
463    }
464    let is_std = body
465        .iter()
466        .all(|&b| b.is_ascii_alphanumeric() || b == b'+' || b == b'/');
467    let is_url = body
468        .iter()
469        .all(|&b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_');
470    if is_std {
471        base64::engine::general_purpose::STANDARD
472            .decode(&cleaned)
473            .ok()
474    } else if is_url {
475        base64::engine::general_purpose::URL_SAFE
476            .decode(&cleaned)
477            .ok()
478    } else {
479        None
480    }
481}
482
483fn describe_plist(v: &plist::Value) -> String {
484    match v {
485        plist::Value::Array(a) => format!("array with {} items", a.len()),
486        plist::Value::Dictionary(d) => format!("dict with {} entries", d.len()),
487        plist::Value::Boolean(_) => "boolean".to_owned(),
488        plist::Value::Data(d) => format!("data ({} bytes)", d.len()),
489        plist::Value::Date(_) => "date".to_owned(),
490        plist::Value::Real(_) => "real".to_owned(),
491        plist::Value::Integer(_) => "integer".to_owned(),
492        plist::Value::String(_) => "string".to_owned(),
493        plist::Value::Uid(_) => "uid".to_owned(),
494        _ => "value".to_owned(),
495    }
496}
497
498fn describe_json(v: &serde_json::Value) -> String {
499    match v {
500        serde_json::Value::Object(m) => format!("JSON object with {} keys", m.len()),
501        serde_json::Value::Array(a) => format!("JSON array with {} elements", a.len()),
502        // Unreachable: detect_json only enters on `{`/`[` roots.
503        _ => "JSON value".to_owned(),
504    }
505}
506
507fn contains(haystack: &[u8], needle: &[u8]) -> bool {
508    haystack.windows(needle.len()).any(|w| w == needle)
509}
510
511fn mostly_printable(s: &str) -> bool {
512    let total = s.chars().count();
513    if total == 0 {
514        return false;
515    }
516    let printable = s
517        .chars()
518        .filter(|c| !c.is_control() || matches!(c, '\t' | '\n' | '\r'))
519        .count();
520    (printable * 100) >= (total * 90)
521}
522
523fn head_hex(bytes: &[u8]) -> String {
524    let n = bytes.len().min(16);
525    let mut s = hex::encode(&bytes[..n]);
526    if bytes.len() > n {
527        s = format!("{s} (+{} more)", bytes.len() - n);
528    }
529    s
530}
531
532fn preview(s: &str) -> String {
533    const MAX: usize = 48;
534    let flat: String = s
535        .chars()
536        .map(|c| if c.is_control() { ' ' } else { c })
537        .collect();
538    if flat.chars().count() <= MAX {
539        flat
540    } else {
541        let cut: String = flat.chars().take(MAX).collect();
542        format!("{cut}… ({} chars total)", s.chars().count())
543    }
544}
blob_decoder/identify.rs

blob_decoder/
identify.rs
