face_core/input/
sniff.rs

1//! §4.1 input format sniffing.
2//!
3//! [`sniff_format`] peeks the first ~4 KiB of a [`BufRead`] without
4//! consuming the content bytes (BOM bytes are consumed). The same
5//! buffered reader is then handed to a parser, which sees the
6//! still-pristine stream.
7
8use std::io::BufRead;
9
10use crate::{Envelope, FaceError, InputFormat};
11
12/// Maximum prefix size used for sniffing, per §4.1 ("first ~4 KiB").
13pub const SNIFF_PROBE_BYTES: usize = 4096;
14
15/// UTF-8 byte-order mark.
16const UTF8_BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
17
18/// Outcome of [`sniff_format`].
19#[derive(Debug, Clone, PartialEq, Eq)]
20#[non_exhaustive]
21pub struct SniffOutcome {
22    /// Format chosen for the input.
23    pub format: InputFormat,
24    /// Number of leading bytes already consumed from the reader.
25    /// Currently this is either `0` or `3` — set when a UTF-8 BOM was
26    /// stripped. Content bytes are never consumed.
27    pub bom_bytes: usize,
28}
29
30/// Peek the input and decide its format per §4.1.
31///
32/// The decision tree:
33///
34/// 1. Strip a leading UTF-8 BOM (`EF BB BF`) if present, advancing the
35///    reader by exactly three bytes.
36/// 2. If the prefix has the §9 face-envelope signature
37///    (`{"result":...,"clusters":...}` at the structural level),
38///    return [`InputFormat::FaceEnvelope`].
39/// 3. If the first non-whitespace byte is `{` or `[` and the prefix
40///    parses as a single JSON value or an incomplete prefix of one,
41///    return [`InputFormat::Json`].
42/// 4. If the prefix has multiple top-level JSON values separated by
43///    newlines, return [`InputFormat::Jsonl`].
44/// 5. Otherwise: try JSON, then JSONL on parse failure (§4.1's
45///    ambiguity fallback), then CSV/TSV delimiter detection.
46///
47/// # Errors
48///
49/// - I/O failures from the underlying reader.
50/// - [`FaceError::InputParse`] when the prefix cannot be classified.
51///
52/// # Examples
53///
54/// ```
55/// use std::io::BufReader;
56/// use face_core::input::sniff::sniff_format;
57/// use face_core::InputFormat;
58///
59/// let mut reader = BufReader::new(b"[1, 2, 3]" as &[u8]);
60/// let outcome = sniff_format(&mut reader).unwrap();
61/// assert_eq!(outcome.format, InputFormat::Json);
62/// assert_eq!(outcome.bom_bytes, 0);
63/// ```
64pub fn sniff_format(reader: &mut impl BufRead) -> Result<SniffOutcome, FaceError> {
65    let bom_bytes = strip_bom(reader)?;
66    let probe = take_probe(reader)?;
67
68    if Envelope::looks_like_envelope(&probe) {
69        return Ok(SniffOutcome {
70            format: InputFormat::FaceEnvelope,
71            bom_bytes,
72        });
73    }
74
75    let first_non_ws = probe.iter().find(|b| !b.is_ascii_whitespace());
76    let format = match first_non_ws {
77        // Empty / whitespace-only input is unclassifiable. Surface it as
78        // a parse error rather than guessing — a downstream parser
79        // would emit a less informative "unexpected end" message.
80        None => {
81            return Err(FaceError::InputParse {
82                format: InputFormat::Json,
83                message: "input is empty (no bytes to classify)".to_string(),
84            });
85        }
86        Some(b'{') | Some(b'[') => classify_brace_or_bracket(&probe)?,
87        Some(_) => {
88            // Per §4.1 ambiguity fallback: try JSON, then JSONL.
89            if parses_as_json_value(&probe) {
90                InputFormat::Json
91            } else if parses_as_jsonl(&probe) {
92                InputFormat::Jsonl
93            } else if let Some(format) = sniff_delimited(&probe) {
94                format
95            } else {
96                return Err(FaceError::InputParse {
97                    format: InputFormat::Json,
98                    message: "could not classify input as JSON, JSONL, CSV, or TSV".to_string(),
99                });
100            }
101        }
102    };
103
104    Ok(SniffOutcome { format, bom_bytes })
105}
106
107/// Detect JSON vs JSONL when the first non-whitespace byte is `{` or `[`.
108///
109/// Strategy:
110/// - If the probe parses as a single JSON value, return JSON.
111/// - Otherwise, if the probe contains multiple newline-separated JSON
112///   values, return JSONL.
113/// - If the probe is an incomplete prefix of a larger JSON value, return
114///   JSON and let the full parser consume stdin beyond the sniff limit.
115fn classify_brace_or_bracket(probe: &[u8]) -> Result<InputFormat, FaceError> {
116    match json_value_probe_state(probe) {
117        JsonProbeState::Complete => Ok(InputFormat::Json),
118        JsonProbeState::Incomplete => Ok(InputFormat::Json),
119        JsonProbeState::Invalid => {
120            if parses_as_jsonl(probe) {
121                Ok(InputFormat::Jsonl)
122            } else {
123                // This path began with `{` or `[`, so it is not a delimited
124                // table. Report a clean parse error instead of guessing.
125                Err(FaceError::InputParse {
126                    format: InputFormat::Json,
127                    message: "input begins with `{` or `[` but parses as neither \
128                              JSON nor JSONL"
129                        .to_string(),
130                })
131            }
132        }
133    }
134}
135
136/// Strip a UTF-8 BOM if present, advancing the reader past it.
137fn strip_bom(reader: &mut impl BufRead) -> Result<usize, FaceError> {
138    let buf = reader.fill_buf()?;
139    if buf.starts_with(&UTF8_BOM) {
140        reader.consume(UTF8_BOM.len());
141        Ok(UTF8_BOM.len())
142    } else {
143        Ok(0)
144    }
145}
146
147/// Capture up to `SNIFF_PROBE_BYTES` of the reader's buffered prefix
148/// without consuming any of it.
149///
150/// Note: a buffered reader may hand back fewer bytes than the probe
151/// limit on the first `fill_buf` call (e.g. a small `BufReader` capacity
152/// or a slow source). For the sniff probe we accept whatever the first
153/// non-empty fill returns — small inputs often live entirely in the
154/// first chunk, and partial probes still classify the obvious cases.
155fn take_probe(reader: &mut impl BufRead) -> Result<Vec<u8>, FaceError> {
156    let buf = reader.fill_buf()?;
157    let len = buf.len().min(SNIFF_PROBE_BYTES);
158    Ok(buf[..len].to_vec())
159}
160
161/// Does the byte slice parse as a single JSON value?
162///
163/// `from_slice` ignores trailing whitespace but rejects trailing
164/// non-whitespace bytes — so a JSONL stream (`{...}\n{...}`) yields
165/// an error, which is what we want.
166fn parses_as_json_value(probe: &[u8]) -> bool {
167    matches!(json_value_probe_state(probe), JsonProbeState::Complete)
168}
169
170enum JsonProbeState {
171    Complete,
172    Incomplete,
173    Invalid,
174}
175
176fn json_value_probe_state(probe: &[u8]) -> JsonProbeState {
177    match serde_json::from_slice::<serde_json::Value>(probe) {
178        Ok(_) => JsonProbeState::Complete,
179        Err(error) if error.is_eof() => JsonProbeState::Incomplete,
180        Err(_) => JsonProbeState::Invalid,
181    }
182}
183
184/// Does the byte slice look like JSONL — multiple newline-separated
185/// JSON values?
186///
187/// We require **at least two** non-empty lines to parse, so a single
188/// JSON line is still classified as JSON above. Parse failures on
189/// individual lines are tolerated — the §11.1 skip-and-warn contract
190/// applies once parsing begins, and sniff should not refuse a stream
191/// just because one or two lines are malformed.
192fn parses_as_jsonl(probe: &[u8]) -> bool {
193    let text = match std::str::from_utf8(probe) {
194        Ok(t) => t,
195        Err(_) => return false,
196    };
197
198    let lines: Vec<&str> = text
199        .split('\n')
200        .map(|l| l.trim_end_matches('\r'))
201        .filter(|l| !l.trim().is_empty())
202        .collect();
203
204    if lines.len() < 2 {
205        return false;
206    }
207
208    let ok_count = lines
209        .iter()
210        .filter(|line| serde_json::from_str::<serde_json::Value>(line).is_ok())
211        .count();
212    ok_count >= 2
213}
214
215fn sniff_delimited(probe: &[u8]) -> Option<InputFormat> {
216    const CANDIDATES: [u8; 4] = [b',', b';', b'\t', b'|'];
217    let lines = sample_nonempty_lines(probe);
218    let mut best = None;
219    for delimiter in CANDIDATES {
220        let counts = lines
221            .iter()
222            .map(|line| count_delimiter_outside_quotes(line, delimiter))
223            .collect::<Vec<_>>();
224        let Some((&first, rest)) = counts.split_first() else {
225            continue;
226        };
227        if first == 0 || rest.iter().any(|count| *count != first) {
228            continue;
229        }
230        if best.is_none_or(|(_, best_count)| first > best_count) {
231            best = Some((delimiter, first));
232        }
233    }
234    match best.map(|(delimiter, _)| delimiter) {
235        Some(b'\t') => Some(InputFormat::Tsv),
236        Some(_) => Some(InputFormat::Csv),
237        None => None,
238    }
239}
240
241fn sample_nonempty_lines(bytes: &[u8]) -> Vec<&[u8]> {
242    bytes
243        .split(|b| *b == b'\n')
244        .map(|line| line.strip_suffix(b"\r").unwrap_or(line))
245        .filter(|line| line.iter().any(|b| !b.is_ascii_whitespace()))
246        .take(8)
247        .collect()
248}
249
250fn count_delimiter_outside_quotes(line: &[u8], delimiter: u8) -> usize {
251    let mut count = 0usize;
252    let mut in_quotes = false;
253    let mut i = 0usize;
254    while let Some(&byte) = line.get(i) {
255        if byte == b'"' {
256            if in_quotes && line.get(i + 1) == Some(&b'"') {
257                i += 2;
258                continue;
259            }
260            in_quotes = !in_quotes;
261        } else if byte == delimiter && !in_quotes {
262            count += 1;
263        }
264        i += 1;
265    }
266    count
267}
268
269#[cfg(test)]
270mod tests {
271    use super::*;
272    use std::io::BufReader;
273
274    #[test]
275    fn sniffs_json_array() {
276        let mut r = BufReader::new(b"[1, 2, 3]" as &[u8]);
277        let out = sniff_format(&mut r).unwrap();
278        assert_eq!(out.format, InputFormat::Json);
279        assert_eq!(out.bom_bytes, 0);
280        // The reader still sees the original bytes.
281        let buf = r.fill_buf().unwrap();
282        assert_eq!(buf, b"[1, 2, 3]");
283    }
284
285    #[test]
286    fn sniffs_json_object() {
287        let mut r = BufReader::new(b"{\"a\": 1}" as &[u8]);
288        let out = sniff_format(&mut r).unwrap();
289        assert_eq!(out.format, InputFormat::Json);
290    }
291
292    #[test]
293    fn sniffs_jsonl() {
294        let mut r = BufReader::new(b"{\"a\":1}\n{\"a\":2}\n{\"a\":3}\n" as &[u8]);
295        let out = sniff_format(&mut r).unwrap();
296        assert_eq!(out.format, InputFormat::Jsonl);
297    }
298
299    #[test]
300    fn sniffs_face_envelope() {
301        let env = br#"{"result":{"input_total":0},"clusters":[]}"#;
302        let mut r = BufReader::new(&env[..]);
303        let out = sniff_format(&mut r).unwrap();
304        assert_eq!(out.format, InputFormat::FaceEnvelope);
305    }
306
307    #[test]
308    fn strips_utf8_bom() {
309        let mut bytes = vec![0xEF, 0xBB, 0xBF];
310        bytes.extend_from_slice(b"{\"a\": 1}");
311        let mut r = BufReader::new(&bytes[..]);
312        let out = sniff_format(&mut r).unwrap();
313        assert_eq!(out.format, InputFormat::Json);
314        assert_eq!(out.bom_bytes, 3);
315        // Reader is now positioned past the BOM.
316        let buf = r.fill_buf().unwrap();
317        assert_eq!(&buf[..8], b"{\"a\": 1}");
318    }
319
320    #[test]
321    fn sniffs_csv() {
322        let mut r = BufReader::new(b"a,b,c\n1,2,3\n4,5,6\n" as &[u8]);
323        let out = sniff_format(&mut r).unwrap();
324        assert_eq!(out.format, InputFormat::Csv);
325    }
326
327    #[test]
328    fn empty_input_errors() {
329        let mut r = BufReader::new(b"" as &[u8]);
330        let err = sniff_format(&mut r).unwrap_err();
331        match err {
332            FaceError::InputParse { .. } => {}
333            other => panic!("unexpected: {other:?}"),
334        }
335    }
336}
face_core/input/sniff.rs

face_core/input/
sniff.rs
