Skip to main content

dataprof_json/
lib.rs

1//! JSON and JSONL scanning and profiling for `dataprof`.
2//!
3//! This crate is an implementation detail of the `dataprof` facade, which
4//! re-exports [`JsonParserConfig`], [`analyze_json_from_reader`], and
5//! [`analyze_json_file`]. Depend on `dataprof` unless you need JSON support
6//! without the rest of the workspace.
7//!
8//! Two entry shapes are offered. [`scan_json_from_reader`] hands each record to
9//! a callback and reports what it read; [`analyze_json_from_reader`] and
10//! [`analyze_json_file`] build column profiles and a full report on top of it.
11//!
12//! ```
13//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
14//! use std::io::Cursor;
15//!
16//! use dataprof_json::{JsonParserConfig, analyze_json_from_reader};
17//!
18//! let jsonl = "{\"id\": 1, \"city\": \"Rome\"}\n{\"id\": 2, \"city\": \"Milan\"}\n";
19//! let (profiles, _stats, rows_read, malformed_lines, _format) =
20//!     analyze_json_from_reader(Cursor::new(jsonl), &JsonParserConfig::jsonl())?;
21//!
22//! assert_eq!(rows_read, 2);
23//! assert_eq!(malformed_lines, 0);
24//! assert_eq!(profiles[0].name, "id");
25//! # Ok(())
26//! # }
27//! ```
28
29use std::collections::{HashMap, HashSet};
30use std::io::BufRead;
31use std::path::Path;
32
33pub use dataprof_core::JsonErrorPolicy;
34use dataprof_core::{
35    AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat,
36    QualityDimension, SemanticHints, TruncationReason, Utf8BomReader,
37};
38use dataprof_runtime::{
39    ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
40};
41use serde::Deserialize;
42use serde_json::Value;
43
44/// Which grammar a JSON source is read with.
45///
46/// The two are deliberately distinct: whitespace is insignificant inside a
47/// standard JSON document and load-bearing in JSONL, so one input cannot be
48/// valid under both readings and mean the same thing. Callers pick with
49/// `format="json"` / `format="jsonl"`, or by file extension.
50#[derive(Debug, Clone, Copy, PartialEq, Eq)]
51pub enum JsonFormat {
52    /// A standard JSON document, in either of the two shapes that carry
53    /// records: an array of objects (`[{...}, {...}]`), or a single object
54    /// (`{...}`) as one record. Values may span lines and be pretty-printed,
55    /// and exactly one document must fill the whole input.
56    Json,
57    /// JSON Lines — one record per physical line. A record may not span lines,
58    /// and a line may not hold more than one value.
59    Jsonl,
60}
61
62/// Configuration for JSON/JSONL parsing and scanning.
63///
64/// The default detects the grammar from the input and reads every record,
65/// skipping malformed ones. The two format constructors say which grammar to
66/// read with, which is what a caller who knows should do: detection cannot tell
67/// a JSON object document from a JSONL file, since both open with `{`.
68///
69/// # Examples
70///
71/// The same two records under each grammar:
72///
73/// ```
74/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
75/// use std::io::Cursor;
76///
77/// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
78///
79/// let document = r#"[{"id": 1}, {"id": 2}]"#;
80/// let lines = "{\"id\": 1}\n{\"id\": 2}\n";
81///
82/// let from_document = scan_json_from_reader(
83///     Cursor::new(document),
84///     &JsonParserConfig::json_document(),
85///     |_| {},
86/// )?;
87/// let from_lines =
88///     scan_json_from_reader(Cursor::new(lines), &JsonParserConfig::jsonl(), |_| {})?;
89///
90/// assert_eq!(from_document.rows_read, 2);
91/// assert_eq!(from_lines.rows_read, 2);
92/// # Ok(())
93/// # }
94/// ```
95#[derive(Debug, Clone, Default)]
96pub struct JsonParserConfig {
97    /// The grammar to read with. `None` detects it from the first byte, which
98    /// resolves `[` to [`JsonFormat::Json`] and everything else to
99    /// [`JsonFormat::Jsonl`] — a leading `{` opens both a JSON object document
100    /// and a JSONL file, so callers that know which they have (from the file
101    /// extension or an explicit `format=`) should say so rather than rely on
102    /// detection.
103    pub format: Option<JsonFormat>,
104    /// Maximum rows to process (None = all rows).
105    pub max_rows: Option<usize>,
106    /// How to react to a malformed record (default: skip).
107    pub error_policy: JsonErrorPolicy,
108}
109
110impl JsonParserConfig {
111    /// Set the maximum number of rows to process.
112    ///
113    /// The cap counts profileable records, not lines: blank lines and records
114    /// skipped as malformed do not consume it.
115    ///
116    /// A cap is only *truncation* when a record still remained once it was
117    /// reached, so a source holding exactly `max_rows` records reads as
118    /// complete. [`JsonScanSummary::truncated`] carries that distinction, and
119    /// [`analyze_json_file`] turns it into a [`TruncationReason::MaxRows`] on
120    /// the report.
121    ///
122    /// # Examples
123    ///
124    /// ```
125    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
126    /// use std::io::Cursor;
127    ///
128    /// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
129    ///
130    /// let lines = "{\"id\": 1}\n{\"id\": 2}\n{\"id\": 3}\n";
131    ///
132    /// // Two of three records: a third remained, so the scan was cut short.
133    /// let capped = JsonParserConfig::jsonl().with_max_rows(2);
134    /// let summary = scan_json_from_reader(Cursor::new(lines), &capped, |_| {})?;
135    /// assert_eq!(summary.rows_read, 2);
136    /// assert!(summary.truncated);
137    ///
138    /// // A cap the source never reaches past is a complete read, not a
139    /// // truncated one.
140    /// let exact = JsonParserConfig::jsonl().with_max_rows(3);
141    /// let summary = scan_json_from_reader(Cursor::new(lines), &exact, |_| {})?;
142    /// assert_eq!(summary.rows_read, 3);
143    /// assert!(!summary.truncated);
144    /// # Ok(())
145    /// # }
146    /// ```
147    pub fn with_max_rows(mut self, max_rows: usize) -> Self {
148        self.max_rows = Some(max_rows);
149        self
150    }
151
152    /// Set how malformed records are handled.
153    ///
154    /// Under the default [`JsonErrorPolicy::Skip`] a malformed record is
155    /// counted in [`JsonScanSummary::malformed_lines`] and scanning continues,
156    /// so a partial profile never looks like a clean one. Under
157    /// [`JsonErrorPolicy::Strict`] the first one fails the scan.
158    ///
159    /// # Examples
160    ///
161    /// ```
162    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
163    /// use std::io::Cursor;
164    ///
165    /// use dataprof_json::{JsonErrorPolicy, JsonParserConfig, scan_json_from_reader};
166    ///
167    /// let lines = "{\"id\": 1}\nnot json\n{\"id\": 2}\n";
168    ///
169    /// let skipping = JsonParserConfig::jsonl();
170    /// let summary = scan_json_from_reader(Cursor::new(lines), &skipping, |_| {})?;
171    /// assert_eq!(summary.rows_read, 2);
172    /// assert_eq!(summary.malformed_lines, 1);
173    ///
174    /// let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
175    /// assert!(scan_json_from_reader(Cursor::new(lines), &strict, |_| {}).is_err());
176    /// # Ok(())
177    /// # }
178    /// ```
179    pub fn with_error_policy(mut self, policy: JsonErrorPolicy) -> Self {
180        self.error_policy = policy;
181        self
182    }
183
184    /// Read as JSON Lines: one record per physical line.
185    pub fn jsonl() -> Self {
186        Self {
187            format: Some(JsonFormat::Jsonl),
188            ..Default::default()
189        }
190    }
191
192    /// Read as a standard JSON document: an array of records, or a single
193    /// object as one record.
194    pub fn json_document() -> Self {
195        Self {
196            format: Some(JsonFormat::Json),
197            ..Default::default()
198        }
199    }
200
201    /// Set the grammar explicitly.
202    pub fn with_format(mut self, format: JsonFormat) -> Self {
203        self.format = Some(format);
204        self
205    }
206}
207
208/// Borrowed JSON object callback payload.
209///
210/// The workspace enables `serde_json/preserve_order`, so iterating this map
211/// yields fields in the order they appear in the source document rather than
212/// alphabetically. That is what makes the column-ordering contract on
213/// [`analyze_json_from_reader`] possible.
214pub type JsonObject = serde_json::Map<String, Value>;
215
216/// Summary of a JSON/JSONL scan.
217#[derive(Debug, Clone, PartialEq, Eq)]
218#[non_exhaustive]
219pub struct JsonScanSummary {
220    pub rows_read: usize,
221    /// Records that were read but not profiled: malformed JSON, and valid JSON
222    /// values that are not objects and therefore carry no fields. Both are
223    /// counted so a tolerant scan never looks like a clean one.
224    pub malformed_lines: usize,
225    pub format: FileFormat,
226    /// The scan stopped at `max_rows` while records still remained. A source
227    /// holding exactly `max_rows` records was read in full, so this stays false.
228    pub truncated: bool,
229}
230
231/// Scan JSON or JSONL input and invoke `on_object` for each object record.
232///
233/// - **JSONL**: one record per physical line. A record may not span lines and a
234///   line may not hold more than one value, so a concatenation whose delimiter
235///   was lost is reported rather than read as several clean records. Blank lines
236///   are separators: neither records nor errors.
237/// - **JSON**: one standard document. An array of objects streams element by
238///   element without buffering the whole input; a single object is read whole
239///   and is one record, which is why it may be pretty-printed across lines.
240///
241/// # Record policy
242///
243/// Only JSON objects are profileable records — they are the only JSON value
244/// with named fields to turn into columns. A record that is valid JSON but not
245/// an object (a scalar, an array, `null`) is never silently discarded: under
246/// [`JsonErrorPolicy::Skip`] it is counted in
247/// [`JsonScanSummary::malformed_lines`] and scanning continues with the next
248/// record, and under [`JsonErrorPolicy::Strict`] the first one fails the scan.
249///
250/// # Examples
251///
252/// Reading JSONL, keeping one field per record:
253///
254/// ```
255/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
256/// use std::io::Cursor;
257///
258/// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
259///
260/// let lines = "{\"city\": \"Rome\", \"pop\": 2800000}\n\
261///              {\"city\": \"Milan\", \"pop\": 1400000}\n";
262///
263/// let mut cities = Vec::new();
264/// let summary = scan_json_from_reader(
265///     Cursor::new(lines),
266///     &JsonParserConfig::jsonl(),
267///     |object| {
268///         if let Some(city) = object.get("city").and_then(|value| value.as_str()) {
269///             cities.push(city.to_string());
270///         }
271///     },
272/// )?;
273///
274/// assert_eq!(cities, ["Rome", "Milan"]);
275/// assert_eq!(summary.rows_read, 2);
276/// assert!(!summary.truncated);
277/// # Ok(())
278/// # }
279/// ```
280///
281/// The same records as a JSON array. The array streams element by element, and
282/// a record may be pretty-printed across lines because whitespace carries no
283/// meaning under this grammar:
284///
285/// ```
286/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
287/// use std::io::Cursor;
288///
289/// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
290///
291/// let document = r#"[
292///     {"city": "Rome", "pop": 2800000},
293///     {"city": "Milan", "pop": 1400000}
294/// ]"#;
295///
296/// let mut populations = Vec::new();
297/// let summary = scan_json_from_reader(
298///     Cursor::new(document),
299///     &JsonParserConfig::json_document(),
300///     |object| {
301///         // A field that is absent, or present but not a number, stays
302///         // `None`. Reading it as 0 instead would turn a missing value into
303///         // a real measurement, which is the one thing a profile must never
304///         // do.
305///         populations.push(object.get("pop").and_then(|value| value.as_u64()));
306///     },
307/// )?;
308///
309/// assert_eq!(populations, [Some(2_800_000), Some(1_400_000)]);
310/// assert_eq!(summary.rows_read, 2);
311/// assert_eq!(summary.malformed_lines, 0);
312/// # Ok(())
313/// # }
314/// ```
315pub fn scan_json_from_reader<R, F>(
316    reader: R,
317    config: &JsonParserConfig,
318    mut on_object: F,
319) -> Result<JsonScanSummary, DataProfilerError>
320where
321    R: BufRead,
322    F: FnMut(&JsonObject),
323{
324    let mut reader = Utf8BomReader::new(reader).map_err(DataProfilerError::from)?;
325    let format = match config.format {
326        Some(JsonFormat::Json) => FileFormat::Json,
327        Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
328        // A leading `{` is the first byte of both a JSON object document and a
329        // JSONL file, so detection can only resolve the array case and has to
330        // fall back to JSONL. Callers that know better pass `format`.
331        None => match consume_leading_whitespace(&mut reader)? {
332            Some(b'[') => FileFormat::Json,
333            _ => FileFormat::Jsonl,
334        },
335    };
336
337    let mut rows_read = 0;
338    let mut malformed_lines = 0;
339    let mut truncated = false;
340
341    match format {
342        // JSONL is line-delimited: exactly one record per non-blank physical
343        // line. Reading it as a whitespace-delimited stream of values instead
344        // made `{"a":1}{"b":2}` — a concatenation with the delimiter lost —
345        // profile as two clean rows, which is precisely the corruption a
346        // profiler exists to surface.
347        FileFormat::Jsonl => {
348            let mut line = String::new();
349            let mut line_number = 0;
350            loop {
351                if let Some(max) = config.max_rows
352                    && rows_read >= max
353                {
354                    // Reaching the cap is not truncation unless a record
355                    // actually remains: a file with exactly `max_rows` records
356                    // was read in full. Blank lines are not records.
357                    truncated = next_record_line(&mut reader, &mut line, &mut line_number)?;
358                    break;
359                }
360
361                if !next_record_line(&mut reader, &mut line, &mut line_number)? {
362                    break;
363                }
364
365                match read_jsonl_line(&line) {
366                    JsonRecord::Object(obj) => {
367                        on_object(&obj);
368                        rows_read += 1;
369                    }
370                    JsonRecord::NonObject(kind) => {
371                        if config.error_policy == JsonErrorPolicy::Strict {
372                            return Err(non_object_record_error(
373                                kind,
374                                rows_read + malformed_lines + 1,
375                            ));
376                        }
377                        malformed_lines += 1;
378                    }
379                    JsonRecord::Malformed(err) => {
380                        if config.error_policy == JsonErrorPolicy::Strict {
381                            return Err(malformed_jsonl_record_error(line_number, err));
382                        }
383                        malformed_lines += 1;
384                    }
385                }
386            }
387        }
388        FileFormat::Json => {
389            // Which of the two record-carrying shapes this document is. The
390            // first non-whitespace byte decides, and is left unconsumed so the
391            // object branch still sees the value it opens.
392            let found_array = consume_leading_whitespace(&mut reader)? == Some(b'[');
393
394            if !found_array {
395                // The other shape a standard JSON document takes: a single
396                // object, which is one record. It may be pretty-printed across
397                // lines, which is why this is not the JSONL grammar.
398                let (object_rows, object_malformed, object_truncated) =
399                    scan_single_json_object(&mut reader, config, &mut on_object)?;
400                rows_read += object_rows;
401                malformed_lines += object_malformed;
402                truncated |= object_truncated;
403            } else {
404                // The array loop starts after the opening bracket.
405                consume_peeked(&mut reader)?;
406
407                let mut expect_value = true;
408                let mut allow_end = true;
409                let mut array_closed = false;
410                let mut drain_remainder = false;
411
412                loop {
413                    let Some(next) = consume_leading_whitespace(&mut reader)? else {
414                        if config.error_policy == JsonErrorPolicy::Strict {
415                            return Err(malformed_array_error(
416                                "unexpected end of input before closing ']'",
417                            ));
418                        }
419                        malformed_lines += 1;
420                        drain_remainder = true;
421                        break;
422                    };
423
424                    if expect_value {
425                        if next == b']' {
426                            if !allow_end {
427                                if config.error_policy == JsonErrorPolicy::Strict {
428                                    return Err(malformed_array_error(
429                                        "trailing comma before closing ']'",
430                                    ));
431                                }
432                                malformed_lines += 1;
433                                drain_remainder = true;
434                                break;
435                            }
436                            consume_peeked(&mut reader)?;
437                            array_closed = true;
438                            break;
439                        }
440
441                        if let Some(max) = config.max_rows
442                            && rows_read >= max
443                        {
444                            // A value remains unread. Do not validate the unread
445                            // suffix of an intentionally bounded scan.
446                            truncated = true;
447                            break;
448                        }
449
450                        if next == b',' {
451                            if config.error_policy == JsonErrorPolicy::Strict {
452                                return Err(malformed_array_error(
453                                    "unexpected comma where an array value was required",
454                                ));
455                            }
456                            malformed_lines += 1;
457                            drain_remainder = true;
458                            break;
459                        }
460
461                        match read_json_record(&mut reader, next)? {
462                            JsonRecord::Object(obj) => {
463                                on_object(&obj);
464                                rows_read += 1;
465                            }
466                            JsonRecord::NonObject(kind) => {
467                                if config.error_policy == JsonErrorPolicy::Strict {
468                                    return Err(non_object_record_error(
469                                        kind,
470                                        rows_read + malformed_lines + 1,
471                                    ));
472                                }
473                                // The element was consumed in full, so the array
474                                // grammar is still intact and the objects after
475                                // it are still profileable.
476                                malformed_lines += 1;
477                            }
478                            JsonRecord::Malformed(err) => {
479                                if config.error_policy == JsonErrorPolicy::Strict {
480                                    return Err(malformed_record_error(err));
481                                }
482                                malformed_lines += 1;
483                                drain_remainder = true;
484                                break;
485                            }
486                        }
487                        expect_value = false;
488                    } else {
489                        match next {
490                            b',' => {
491                                consume_peeked(&mut reader)?;
492                                expect_value = true;
493                                allow_end = false;
494                            }
495                            b']' => {
496                                consume_peeked(&mut reader)?;
497                                array_closed = true;
498                                break;
499                            }
500                            _ => {
501                                if config.error_policy == JsonErrorPolicy::Strict {
502                                    return Err(malformed_array_error(
503                                        "expected ',' or ']' after an array value",
504                                    ));
505                                }
506                                malformed_lines += 1;
507                                drain_remainder = true;
508                                break;
509                            }
510                        }
511                    }
512                }
513
514                if drain_remainder {
515                    drain_to_end(&mut reader)?;
516                } else if array_closed && consume_leading_whitespace(&mut reader)?.is_some() {
517                    if config.error_policy == JsonErrorPolicy::Strict {
518                        return Err(malformed_array_error(
519                            "non-whitespace content follows the closing ']'",
520                        ));
521                    }
522                    malformed_lines += 1;
523                    drain_to_end(&mut reader)?;
524                }
525            }
526        }
527        _ => unreachable!("json scanner only returns json or jsonl formats"),
528    }
529
530    Ok(JsonScanSummary {
531        rows_read,
532        malformed_lines,
533        format,
534        truncated,
535    })
536}
537
538/// Read the next line that holds a record into `line`, skipping blank ones.
539///
540/// Returns whether a record line was found. Blank lines are separators, not
541/// records: they are skipped without counting as rows or as errors. `line_number`
542/// counts every physical line read, blank ones included, so a diagnostic points
543/// at the line the user would open the file to.
544fn next_record_line<R: BufRead>(
545    reader: &mut R,
546    line: &mut String,
547    line_number: &mut usize,
548) -> Result<bool, DataProfilerError> {
549    loop {
550        line.clear();
551        let read = reader.read_line(line).map_err(DataProfilerError::from)?;
552        if read == 0 {
553            return Ok(false);
554        }
555        *line_number += 1;
556        if !line.trim().is_empty() {
557            return Ok(true);
558        }
559    }
560}
561
562/// Parse one JSONL line as exactly one JSON value.
563///
564/// `serde_json::from_str` requires the value to fill the whole input, so a line
565/// carrying two adjacent or space-separated values is a malformed record rather
566/// than two clean ones — the boundary rule that separates JSONL from a stream of
567/// JSON values.
568fn read_jsonl_line(line: &str) -> JsonRecord {
569    match serde_json::from_str::<Value>(line.trim()) {
570        Ok(Value::Object(obj)) => JsonRecord::Object(obj),
571        Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
572        Err(err) => JsonRecord::Malformed(err),
573    }
574}
575
576/// Read a whole-input single JSON object as one record.
577///
578/// Returns `(rows_read, malformed, truncated)`. Anything that is not exactly one object
579/// filling the input is an error under [`JsonErrorPolicy::Strict`] and a counted
580/// malformed record otherwise, so a truncated or concatenated document never
581/// profiles as clean.
582fn scan_single_json_object<R, F>(
583    reader: &mut R,
584    config: &JsonParserConfig,
585    on_object: &mut F,
586) -> Result<(usize, usize, bool), DataProfilerError>
587where
588    R: BufRead,
589    F: FnMut(&JsonObject),
590{
591    let mut text = String::new();
592    reader
593        .read_to_string(&mut text)
594        .map_err(DataProfilerError::from)?;
595
596    if text.trim().is_empty() {
597        return Ok((0, 0, false));
598    }
599
600    // A row cap of zero asks for no records, so the document is not read. The
601    // record it holds is left unread, which is a truncation and has to be
602    // reported as one — an unread record and an empty source must not produce
603    // the same report.
604    if config.max_rows == Some(0) {
605        return Ok((0, 0, true));
606    }
607
608    match serde_json::from_str::<Value>(text.trim()) {
609        Ok(Value::Object(obj)) => {
610            on_object(&obj);
611            Ok((1, 0, false))
612        }
613        Ok(value) => {
614            if config.error_policy == JsonErrorPolicy::Strict {
615                return Err(non_object_record_error(json_value_kind(&value), 1));
616            }
617            Ok((0, 1, false))
618        }
619        Err(err) => {
620            if config.error_policy == JsonErrorPolicy::Strict {
621                return Err(json_document_error(err));
622            }
623            Ok((0, 1, false))
624        }
625    }
626}
627
628/// One record read from a JSON or JSONL source.
629enum JsonRecord {
630    /// A JSON object — the only value with named fields to profile as a row.
631    Object(JsonObject),
632    /// Valid JSON that is not an object, carrying the value's kind for the
633    /// error message.
634    NonObject(&'static str),
635    /// The bytes are not valid JSON.
636    Malformed(serde_json::Error),
637}
638
639/// Read one JSON value and leave the reader positioned immediately after it.
640///
641/// `first` is the value's first byte, already located by
642/// [`consume_leading_whitespace`] but not yet consumed.
643fn read_json_record<R: BufRead>(
644    reader: &mut R,
645    first: u8,
646) -> Result<JsonRecord, DataProfilerError> {
647    // A number is the only JSON value that is not self-delimiting: serde finds
648    // its end by peeking one byte past it and then drops that byte along with
649    // the deserializer. Inside an array that byte is the `,` or `]` the scanner
650    // needs next, so numbers are read byte-wise instead.
651    if first == b'-' || first.is_ascii_digit() {
652        let token = read_number_token(reader)?;
653        return Ok(match serde_json::from_str::<Value>(&token) {
654            Ok(_) => JsonRecord::NonObject("number"),
655            Err(err) => JsonRecord::Malformed(err),
656        });
657    }
658
659    let mut deserializer = serde_json::Deserializer::from_reader(reader);
660    Ok(match Value::deserialize(&mut deserializer) {
661        Ok(Value::Object(obj)) => JsonRecord::Object(obj),
662        Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
663        Err(err) => JsonRecord::Malformed(err),
664    })
665}
666
667/// Read the bytes that can spell a JSON number, stopping at the first byte that
668/// cannot. The token is returned unvalidated — the caller decides whether it
669/// actually parses, so `1.2.3` stays a malformed record rather than a number.
670///
671/// Leading whitespace is consumed first: [`consume_leading_whitespace`] reports
672/// the next value's first byte without consuming what precedes it, because
673/// serde needs that whitespace to keep its line and column context.
674fn read_number_token<R: BufRead>(reader: &mut R) -> Result<String, DataProfilerError> {
675    skip_ascii_whitespace(reader)?;
676    let mut token = String::new();
677    loop {
678        let (consume, token_ended) = {
679            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
680            if buf.is_empty() {
681                break;
682            }
683            let consume = buf
684                .iter()
685                .take_while(|byte| is_json_number_byte(**byte))
686                .count();
687            // decode-audit: impossible — every byte accepted above is ASCII.
688            token.push_str(
689                std::str::from_utf8(&buf[..consume]).expect("JSON number bytes are ASCII"),
690            );
691            (consume, consume < buf.len())
692        };
693        reader.consume(consume);
694        if token_ended {
695            break;
696        }
697    }
698    Ok(token)
699}
700
701fn is_json_number_byte(byte: u8) -> bool {
702    byte.is_ascii_digit() || matches!(byte, b'-' | b'+' | b'.' | b'e' | b'E')
703}
704
705/// Consume the byte [`consume_leading_whitespace`] just reported, along with the
706/// whitespace it looked past.
707///
708/// That function only peeks — it leaves the whitespace queued so serde keeps its
709/// line and column context — so consuming one byte would eat a space rather than
710/// the delimiter. Getting this wrong made a pretty-printed JSON array report a
711/// phantom malformed record, because the `]` survived the consume and then
712/// looked like content after the closing bracket.
713fn consume_peeked<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
714    skip_ascii_whitespace(reader)?;
715    reader.consume(1);
716    Ok(())
717}
718
719/// Advance the reader past any ASCII whitespace at its current position.
720fn skip_ascii_whitespace<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
721    loop {
722        let (consume, done) = {
723            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
724            if buf.is_empty() {
725                return Ok(());
726            }
727            let consume = buf
728                .iter()
729                .take_while(|byte| byte.is_ascii_whitespace())
730                .count();
731            (consume, consume < buf.len())
732        };
733        reader.consume(consume);
734        if done {
735            return Ok(());
736        }
737    }
738}
739
740fn json_value_kind(value: &Value) -> &'static str {
741    match value {
742        Value::Null => "null",
743        Value::Bool(_) => "boolean",
744        Value::Number(_) => "number",
745        Value::String(_) => "string",
746        Value::Array(_) => "array",
747        Value::Object(_) => "object",
748    }
749}
750
751/// Build a strict-mode error for valid JSON that is not a row object. This is a
752/// distinct category from a syntax failure: the document parsed, it just does
753/// not hold records. `position` is 1-based over the records scanned so far.
754fn non_object_record_error(kind: &str, position: usize) -> DataProfilerError {
755    // No source: the document parsed cleanly, it simply does not hold records,
756    // so there is no decoder error underneath to retain.
757    DataProfilerError::json_parsing_error(&format!(
758        "non-object JSON record at position {position}: \
759         expected an object with fields to profile, found {kind}"
760    ))
761}
762
763/// Build a strict-mode parse error from a serde failure. The serde message
764/// carries line/column context but never the record contents, so it is safe to
765/// surface directly.
766fn malformed_record_error(err: serde_json::Error) -> DataProfilerError {
767    let message = format!("malformed JSON record: {err}");
768    DataProfilerError::json_parsing_with_source(message, err)
769}
770
771/// Build a strict-mode parse error for a JSONL record, located by its physical
772/// line.
773///
774/// Each line is parsed on its own, so the decoder's own line number is always 1
775/// and would be misleading; the column it reports is within the line and so is
776/// also the column in the file. The record's text is never included. Matches the
777/// phrasing the Python bytes reader uses for the same failure.
778fn malformed_jsonl_record_error(line: usize, err: serde_json::Error) -> DataProfilerError {
779    let message = format!(
780        "malformed JSON record on line {line}, column {}: a JSONL record must be one complete JSON value on one line",
781        err.column()
782    );
783    DataProfilerError::json_parsing_with_source(message, err)
784}
785
786fn malformed_array_error(message: &str) -> DataProfilerError {
787    // Callers here hold only a message, not the decoder error, so there is
788    // nothing to retain.
789    DataProfilerError::json_parsing_error(&format!("malformed JSON array: {message}"))
790}
791
792/// Build a strict-mode error for a standard JSON document that is not exactly
793/// one value.
794///
795/// The most common cause is JSONL read with the JSON grammar — several objects
796/// back to back parse as one value plus trailing characters — so the message
797/// names the option that would read it correctly.
798fn json_document_error(err: serde_json::Error) -> DataProfilerError {
799    let message = format!(
800        // One source line on purpose: rustfmt may join a `\`-continued
801        // literal and leave its indentation inside the string, which silently
802        // corrupted this very message once.
803        "malformed JSON document: {err}. A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
804    );
805    DataProfilerError::json_parsing_with_source(message, err)
806}
807
808fn drain_to_end<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
809    loop {
810        let available = reader.fill_buf().map_err(DataProfilerError::from)?.len();
811        if available == 0 {
812            return Ok(());
813        }
814        reader.consume(available);
815    }
816}
817
818fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
819    loop {
820        let mut bytes_to_consume = 0;
821        let first_non_whitespace = {
822            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
823            if buf.is_empty() {
824                return Ok(None);
825            }
826
827            let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
828            if first_non_whitespace.is_none() {
829                bytes_to_consume = buf.len();
830            }
831            first_non_whitespace
832        };
833
834        if first_non_whitespace.is_some() {
835            return Ok(first_non_whitespace);
836        }
837
838        reader.consume(bytes_to_consume);
839    }
840}
841
842/// Convert a JSON [`Value`] to a flat string for column storage.
843fn json_value_to_string(value: &Value) -> String {
844    match value {
845        Value::Null => String::new(),
846        Value::Bool(boolean) => boolean.to_string(),
847        Value::Number(number) => number.to_string(),
848        Value::String(string) => string.to_string(),
849        // decode-audit: impossible — serializing an in-memory Value back to a
850        // string cannot fail; a panic here beats folding "" (= null) into stats.
851        Value::Array(_) | Value::Object(_) => {
852            serde_json::to_string(value).expect("re-serializing a parsed JSON value cannot fail")
853        }
854    }
855}
856
857/// Feed a JSON object's fields into a [`StreamingColumnCollection`].
858///
859/// Columns are registered in the order the fields appear in each record, so the
860/// resulting column order is the first record's field order with later-only
861/// fields appended where they were first seen.
862fn feed_json_object(
863    obj: &JsonObject,
864    prior_rows: usize,
865    known_columns: &mut Vec<String>,
866    known_columns_set: &mut HashSet<String>,
867    column_stats: &mut StreamingColumnCollection,
868) {
869    for key in obj.keys() {
870        if known_columns_set.insert(key.clone()) {
871            known_columns.push(key.clone());
872            column_stats.init_column_with_missing(key, prior_rows);
873        }
874    }
875
876    let values: Vec<String> = known_columns
877        .iter()
878        .map(|column| {
879            // decode-audit: no-data — a key absent from this object is a
880            // missing field, and "" is the profiler's textual null.
881            obj.get(column)
882                .map(json_value_to_string)
883                .unwrap_or_default()
884        })
885        .collect();
886
887    column_stats.process_record(known_columns, values);
888}
889
890/// Analyze JSON/JSONL data from a buffered reader using streaming statistics.
891///
892/// Returns `(column_profiles, streaming_stats, rows_read, malformed_lines, detected_format)`.
893///
894/// # Column order
895///
896/// Columns are returned in source order — the first record's field order, with
897/// fields that only appear in later records appended in first-seen order. This
898/// matches CSV (header order) and Parquet (schema order), so the same logical
899/// dataset profiles to the same column order in every format.
900///
901/// # Examples
902///
903/// The second record lists its fields in a different order and adds one. The
904/// first record still decides the order, and the new field is appended:
905///
906/// ```
907/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
908/// use std::io::Cursor;
909///
910/// use dataprof_json::{JsonParserConfig, analyze_json_from_reader};
911///
912/// let lines = "{\"id\": 1, \"city\": \"Rome\"}\n\
913///              {\"city\": \"Milan\", \"id\": 2, \"region\": \"Lombardy\"}\n";
914///
915/// let (profiles, _stats, rows_read, malformed_lines, format) =
916///     analyze_json_from_reader(Cursor::new(lines), &JsonParserConfig::jsonl())?;
917///
918/// let names: Vec<&str> = profiles.iter().map(|profile| profile.name.as_str()).collect();
919/// assert_eq!(names, ["id", "city", "region"]);
920/// assert_eq!(rows_read, 2);
921/// assert_eq!(malformed_lines, 0);
922/// assert_eq!(format, dataprof_core::FileFormat::Jsonl);
923///
924/// // The first record has no `region`, which counts as a missing value.
925/// let region = &profiles[2];
926/// assert_eq!(region.total_count, 2);
927/// assert_eq!(region.null_count, 1);
928/// # Ok(())
929/// # }
930/// ```
931pub fn analyze_json_from_reader<R: BufRead>(
932    reader: R,
933    config: &JsonParserConfig,
934) -> Result<
935    (
936        Vec<ColumnProfile>,
937        StreamingColumnCollection,
938        usize,
939        usize,
940        FileFormat,
941    ),
942    DataProfilerError,
943> {
944    analyze_json_from_reader_with_hints(reader, config, &SemanticHints::default())
945}
946
947pub fn analyze_json_from_reader_with_hints<R: BufRead>(
948    reader: R,
949    config: &JsonParserConfig,
950    semantic_hints: &SemanticHints,
951) -> Result<
952    (
953        Vec<ColumnProfile>,
954        StreamingColumnCollection,
955        usize,
956        usize,
957        FileFormat,
958    ),
959    DataProfilerError,
960> {
961    let options = AnalysisOptions::default().with_semantic_hints(semantic_hints.clone());
962    let (profiles, stats, rows_read, malformed_lines, format, _truncated) =
963        analyze_json_from_reader_full(reader, config, &options)?;
964    Ok((profiles, stats, rows_read, malformed_lines, format))
965}
966
967/// As [`analyze_json_from_reader_with_hints`], but also reports whether the scan
968/// stopped early. Kept private so the public tuple stays stable.
969#[allow(clippy::type_complexity)]
970fn analyze_json_from_reader_full<R: BufRead>(
971    reader: R,
972    config: &JsonParserConfig,
973    options: &AnalysisOptions,
974) -> Result<
975    (
976        Vec<ColumnProfile>,
977        StreamingColumnCollection,
978        usize,
979        usize,
980        FileFormat,
981        bool,
982    ),
983    DataProfilerError,
984> {
985    let semantic_hints = options.semantic_hints();
986    let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
987    let mut known_columns = Vec::new();
988    let mut known_columns_set = HashSet::new();
989    let mut rows_seen = 0;
990
991    let summary = scan_json_from_reader(reader, config, |obj| {
992        feed_json_object(
993            obj,
994            rows_seen,
995            &mut known_columns,
996            &mut known_columns_set,
997            &mut column_stats,
998        );
999        rows_seen += 1;
1000    })?;
1001
1002    let profiles = profile_builder::profiles_from_streaming_with_hints(
1003        &column_stats,
1004        !options.include_statistics(),
1005        !options.include_patterns(),
1006        options.locale(),
1007        semantic_hints,
1008    );
1009
1010    Ok((
1011        profiles,
1012        column_stats,
1013        summary.rows_read,
1014        summary.malformed_lines,
1015        summary.format,
1016        summary.truncated,
1017    ))
1018}
1019
1020/// Analyze a JSON or JSONL file, returning a full [`ProfileReport`].
1021///
1022/// With no explicit [`JsonParserConfig::format`] the file extension decides:
1023/// `.json` reads as a standard document, `.jsonl` and `.ndjson` as JSON Lines,
1024/// and any other name falls back to sniffing the first byte.
1025///
1026/// # Examples
1027///
1028/// ```
1029/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
1030/// use std::io::Write;
1031///
1032/// use dataprof_json::{JsonParserConfig, analyze_json_file};
1033///
1034/// let mut file = tempfile::NamedTempFile::with_suffix(".jsonl")?;
1035/// writeln!(file, "{{\"id\": 1, \"score\": 9.5}}")?;
1036/// writeln!(file, "{{\"id\": 2, \"score\": 7.0}}")?;
1037/// file.flush()?;
1038///
1039/// let report = analyze_json_file(file.path(), &JsonParserConfig::default())?;
1040///
1041/// assert_eq!(report.execution.rows_processed, 2);
1042/// assert_eq!(report.execution.columns_detected, 2);
1043/// assert!(report.execution.truncation_reason.is_none());
1044/// # Ok(())
1045/// # }
1046/// ```
1047pub fn analyze_json_file(
1048    file_path: &Path,
1049    config: &JsonParserConfig,
1050) -> Result<ProfileReport, DataProfilerError> {
1051    analyze_json_file_with_dimensions(file_path, config, None)
1052}
1053
1054/// Like [`analyze_json_file`] but only computes the requested quality dimensions.
1055pub fn analyze_json_file_with_dimensions(
1056    file_path: &Path,
1057    config: &JsonParserConfig,
1058    quality_dimensions: Option<&[QualityDimension]>,
1059) -> Result<ProfileReport, DataProfilerError> {
1060    analyze_json_file_with_dimensions_and_hints(
1061        file_path,
1062        config,
1063        quality_dimensions,
1064        &SemanticHints::default(),
1065    )
1066}
1067
1068pub fn analyze_json_file_with_dimensions_and_hints(
1069    file_path: &Path,
1070    config: &JsonParserConfig,
1071    quality_dimensions: Option<&[QualityDimension]>,
1072    semantic_hints: &SemanticHints,
1073) -> Result<ProfileReport, DataProfilerError> {
1074    let options = AnalysisOptions::default()
1075        .with_quality_dimensions(quality_dimensions.map(<[_]>::to_vec))
1076        .with_semantic_hints(semantic_hints.clone());
1077    analyze_json_file_with_options(file_path, config, &options)
1078}
1079
1080/// Analyze a JSON or JSONL file, honouring the caller's full analysis selection.
1081///
1082/// This is the entry point that carries metric packs and locale as well as
1083/// dimensions and hints, so a JSON profile reports exactly the analysis the
1084/// caller asked for — the same selection the CSV engines apply.
1085pub fn analyze_json_file_with_options(
1086    file_path: &Path,
1087    config: &JsonParserConfig,
1088    options: &AnalysisOptions,
1089) -> Result<ProfileReport, DataProfilerError> {
1090    // An explicit grammar wins; otherwise the file name gets a say before
1091    // content sniffing, which cannot tell a JSON object from a JSONL stream.
1092    let mut config = config.clone();
1093    if config.format.is_none() {
1094        config.format = grammar_for_extension(file_path);
1095    }
1096    let config = &config;
1097    let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
1098    let start = std::time::Instant::now();
1099
1100    let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
1101    let buf_reader = std::io::BufReader::new(file);
1102
1103    let (column_profiles, column_stats, rows_read, malformed_lines, format, truncated) =
1104        analyze_json_from_reader_full(buf_reader, config, options)?;
1105    let read_as_json_document = format == FileFormat::Json;
1106
1107    let file_source = DataSource::File {
1108        path: file_path.display().to_string(),
1109        format,
1110        size_bytes: metadata.len(),
1111        modified_at: None,
1112        parquet_metadata: None,
1113    };
1114
1115    if rows_read == 0 {
1116        if malformed_lines > 0 {
1117            // Nothing parsed under the JSON grammar most often means the source
1118            // is JSONL, so say so here too: under the default skip policy this
1119            // is the only error the caller sees.
1120            let hint = if read_as_json_document {
1121                ". A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
1122            } else {
1123                ""
1124            };
1125            // Aggregate over many discarded records: no single decoder error
1126            // is the cause, so none is retained.
1127            return Err(DataProfilerError::json_parsing_error(&format!(
1128                "No valid JSON records found in file \
1129                 (every record was malformed or not a JSON object){hint}"
1130            )));
1131        }
1132        // A row cap that stopped the scan before the first record still has to
1133        // be disclosed. Without this the report reads exactly like a complete
1134        // profile of an empty source, which is the one thing a truncated scan
1135        // must never be mistaken for.
1136        let mut execution =
1137            ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("json");
1138        if truncated && let Some(max) = config.max_rows {
1139            execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1140        }
1141        return Ok(ReportAssembler::new(file_source, execution)
1142            .columns(column_profiles)
1143            .with_quality_data(HashMap::new())
1144            .with_analysis_options(options)
1145            .build());
1146    }
1147
1148    let sample_columns = profile_builder::quality_check_samples(&column_stats);
1149    let scan_time_ms = start.elapsed().as_millis();
1150    let num_columns = column_profiles.len();
1151
1152    let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms)
1153        .with_engine("json")
1154        // Tolerant scans surface skipped malformed records here so callers can
1155        // tell a partial profile from a clean one.
1156        .with_error_count(malformed_lines);
1157    // `truncated` is set only when a record still remained at the cap, so a source
1158    // holding exactly `max_rows` records is reported as fully read, not cut short.
1159    if truncated && let Some(max) = config.max_rows {
1160        execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1161    }
1162
1163    Ok(ReportAssembler::new(file_source, execution)
1164        .columns(column_profiles)
1165        .with_quality_data(sample_columns)
1166        .with_row_duplicates(column_stats.row_duplicate_summary())
1167        .with_row_completeness(column_stats.row_completeness_summary())
1168        .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
1169        .with_analysis_options(options)
1170        .build())
1171}
1172
1173/// The grammar a file name advertises, if it advertises one.
1174///
1175/// Content sniffing resolves a leading `[` on its own, but not a leading `{`:
1176/// that is the first byte of both a JSON object document and a JSONL file. The
1177/// extension is the evidence that separates them, and a `.json` file holding one
1178/// pretty-printed object is an ordinary thing to profile. `None` means the name
1179/// says nothing and the content decides.
1180fn grammar_for_extension(file_path: &Path) -> Option<JsonFormat> {
1181    let extension = file_path.extension().and_then(|ext| ext.to_str())?;
1182    if extension.eq_ignore_ascii_case("json") {
1183        Some(JsonFormat::Json)
1184    } else if extension.eq_ignore_ascii_case("jsonl") || extension.eq_ignore_ascii_case("ndjson") {
1185        Some(JsonFormat::Jsonl)
1186    } else {
1187        None
1188    }
1189}
1190
1191fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
1192    if error.kind() == std::io::ErrorKind::NotFound {
1193        DataProfilerError::FileNotFound {
1194            path: file_path.display().to_string(),
1195        }
1196    } else {
1197        DataProfilerError::from(error)
1198    }
1199}
1200
1201#[cfg(test)]
1202mod tests {
1203    use super::*;
1204    use std::io::{Cursor, Write};
1205    use tempfile::NamedTempFile;
1206
1207    /// A temp file with no extension, so `analyze_json_file` falls back to the
1208    /// JSONL grammar the way an unnamed stream does.
1209    fn write_file(content: &str) -> NamedTempFile {
1210        let mut file = NamedTempFile::new().unwrap();
1211        write!(file, "{}", content).unwrap();
1212        file.flush().unwrap();
1213        file
1214    }
1215
1216    /// A temp file named `.json`, so `analyze_json_file` reads it as a standard
1217    /// JSON document — the grammar the extension advertises.
1218    fn write_json_file(content: &str) -> NamedTempFile {
1219        let mut file = NamedTempFile::with_suffix(".json").unwrap();
1220        write!(file, "{}", content).unwrap();
1221        file.flush().unwrap();
1222        file
1223    }
1224
1225    fn write_bytes(content: &[u8]) -> NamedTempFile {
1226        let mut file = NamedTempFile::new().unwrap();
1227        file.write_all(content).unwrap();
1228        file.flush().unwrap();
1229        file
1230    }
1231
1232    #[test]
1233    fn test_scan_jsonl_detects_format_and_rows() {
1234        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1235        let mut rows = 0;
1236
1237        let summary = scan_json_from_reader(
1238            Cursor::new(data.as_ref()),
1239            &JsonParserConfig::default(),
1240            |_| {
1241                rows += 1;
1242            },
1243        )
1244        .unwrap();
1245
1246        assert_eq!(summary.format, FileFormat::Jsonl);
1247        assert_eq!(summary.rows_read, 3);
1248        assert_eq!(summary.malformed_lines, 0);
1249        assert_eq!(rows, 3);
1250    }
1251
1252    #[test]
1253    fn test_scan_json_array_detects_format_and_rows() {
1254        let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
1255        let mut keys = Vec::new();
1256
1257        let summary = scan_json_from_reader(
1258            Cursor::new(data.as_ref()),
1259            &JsonParserConfig::default(),
1260            |obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
1261        )
1262        .unwrap();
1263
1264        assert_eq!(summary.format, FileFormat::Json);
1265        assert_eq!(summary.rows_read, 2);
1266        assert_eq!(keys.len(), 2);
1267        assert_eq!(keys[0], vec!["name".to_string()]);
1268    }
1269
1270    #[test]
1271    fn test_scan_accepts_one_leading_utf8_bom_for_json_and_jsonl() {
1272        for (data, config, expected_format) in [
1273            (
1274                b"\xEF\xBB\xBF[{\"x\":1},{\"x\":2}]".as_slice(),
1275                JsonParserConfig::default(),
1276                FileFormat::Json,
1277            ),
1278            (
1279                b"\xEF\xBB\xBF{\"x\":1}\n{\"x\":2}\n".as_slice(),
1280                JsonParserConfig::default(),
1281                FileFormat::Jsonl,
1282            ),
1283        ] {
1284            let mut rows = 0;
1285            let summary = scan_json_from_reader(Cursor::new(data), &config, |_| rows += 1)
1286                .expect("leading UTF-8 BOM should be accepted");
1287
1288            assert_eq!(rows, 2);
1289            assert_eq!(summary.rows_read, 2);
1290            assert_eq!(summary.malformed_lines, 0);
1291            assert_eq!(summary.format, expected_format);
1292        }
1293    }
1294
1295    #[test]
1296    fn test_scan_does_not_strip_nonleading_or_second_utf8_bom() {
1297        let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1298
1299        for data in [
1300            b" \xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1301            b"\xEF\xBB\xBF\xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1302            b"{\"x\":1}\n\xEF\xBB\xBF{\"x\":2}\n".as_slice(),
1303        ] {
1304            scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1305                .expect_err("only the BOM at absolute byte offset zero may be stripped");
1306        }
1307    }
1308
1309    #[test]
1310    fn test_bom_only_input_matches_empty_input_policy() {
1311        for config in [JsonParserConfig::default(), JsonParserConfig::jsonl()] {
1312            let empty = scan_json_from_reader(Cursor::new(b""), &config, |_| {}).unwrap();
1313            let bom =
1314                scan_json_from_reader(Cursor::new(b"\xEF\xBB\xBF".as_slice()), &config, |_| {})
1315                    .unwrap();
1316            assert_eq!(bom, empty);
1317        }
1318    }
1319
1320    #[test]
1321    fn test_json_array_container_grammar_obeys_error_policy() {
1322        let cases = [
1323            (br#"[{"x":1}"#.as_ref(), 1, "missing closing bracket"),
1324            (br#"[{"x":1},"#.as_ref(), 1, "comma followed by EOF"),
1325            (br#"[{"x":1} {"x":2}]"#.as_ref(), 1, "missing comma"),
1326            (br#"[{"x":1},,{"x":2}]"#.as_ref(), 1, "doubled comma"),
1327            (br#"[,{"x":1}]"#.as_ref(), 0, "leading comma"),
1328            (br#"[{"x":1},]"#.as_ref(), 1, "trailing comma"),
1329            (br#"[{"x":1}] trailing"#.as_ref(), 1, "trailing content"),
1330            (
1331                br#"[{"x":1}][{"x":2}]"#.as_ref(),
1332                1,
1333                "second top-level value",
1334            ),
1335        ];
1336
1337        for (data, expected_rows, case) in cases {
1338            let skip = JsonParserConfig::json_document();
1339            let summary =
1340                scan_json_from_reader(Cursor::new(data), &skip, |_| {}).expect("tolerant scan");
1341            assert_eq!(summary.rows_read, expected_rows, "{case}");
1342            assert_eq!(summary.malformed_lines, 1, "{case}");
1343
1344            let strict =
1345                JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict);
1346            let err = scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1347                .expect_err("strict mode must reject invalid array grammar");
1348            assert!(
1349                err.to_string()
1350                    .to_lowercase()
1351                    .contains("malformed json array"),
1352                "{case}: {err}"
1353            );
1354        }
1355    }
1356
1357    #[test]
1358    fn test_json_array_max_rows_does_not_validate_unread_values() {
1359        let data = br#"[{"x":1},not-valid-json"#;
1360        let config = JsonParserConfig::json_document().with_max_rows(1);
1361        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1362
1363        assert_eq!(summary.rows_read, 1);
1364        assert_eq!(summary.malformed_lines, 0);
1365        assert!(summary.truncated);
1366    }
1367
1368    #[test]
1369    fn test_scan_respects_max_rows() {
1370        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
1371        let config = JsonParserConfig::default().with_max_rows(2);
1372        let mut rows = 0;
1373
1374        let summary =
1375            scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
1376
1377        assert_eq!(summary.rows_read, 2);
1378        assert_eq!(rows, 2);
1379    }
1380
1381    #[test]
1382    fn test_scan_skips_malformed_jsonl_lines() {
1383        let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
1384        let mut rows = 0;
1385
1386        let summary = scan_json_from_reader(
1387            Cursor::new(data.as_ref()),
1388            &JsonParserConfig::jsonl(),
1389            |_| {
1390                rows += 1;
1391            },
1392        )
1393        .unwrap();
1394
1395        assert_eq!(summary.format, FileFormat::Jsonl);
1396        assert_eq!(summary.rows_read, 2);
1397        assert_eq!(summary.malformed_lines, 1);
1398        assert_eq!(rows, 2);
1399    }
1400
1401    /// A pretty-printed root object is one record under the JSON grammar and a
1402    /// malformed record under JSONL (#486). It cannot be both: whitespace is
1403    /// insignificant in a JSON document and is the record delimiter in JSONL, so
1404    /// reading a multi-line object as JSONL would mean each of its lines is a
1405    /// record — none of which is valid JSON on its own.
1406    #[test]
1407    fn test_pretty_printed_root_object_is_one_row_under_the_json_grammar() {
1408        let data = br#"{
1409    "type": "FeatureCollection",
1410    "features": [
1411        {"id": 1},
1412        {"id": 2}
1413  ]
1414}"#;
1415        let mut object_field_counts = Vec::new();
1416
1417        let summary = scan_json_from_reader(
1418            Cursor::new(data.as_ref()),
1419            &JsonParserConfig::json_document(),
1420            |obj| object_field_counts.push(obj.len()),
1421        )
1422        .unwrap();
1423
1424        assert_eq!(summary.format, FileFormat::Json);
1425        assert_eq!(summary.rows_read, 1);
1426        assert_eq!(summary.malformed_lines, 0);
1427        assert_eq!(object_field_counts, vec![2]);
1428    }
1429
1430    #[test]
1431    fn test_pretty_printed_root_object_is_not_valid_jsonl() {
1432        let data = br#"{
1433    "type": "FeatureCollection"
1434}"#;
1435        let err = scan_json_from_reader(
1436            Cursor::new(data.as_ref()),
1437            &JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict),
1438            |_| {},
1439        )
1440        .expect_err("a record spanning lines is not JSONL");
1441
1442        assert!(
1443            err.to_string().contains("malformed JSON record"),
1444            "{err}, expected the malformed category"
1445        );
1446    }
1447
1448    /// Every JSON value that is not an object, so none of them can be a row.
1449    /// Numbers are listed twice on purpose: they are the one value serde reads
1450    /// from a reader by peeking one byte past the end.
1451    const NON_OBJECT_VALUES: &[(&str, &str)] = &[
1452        ("null", "null"),
1453        ("boolean", "true"),
1454        ("number", "42"),
1455        ("number", "-1.5e3"),
1456        ("string", r#""text""#),
1457        ("array", "[1, 2]"),
1458    ];
1459
1460    #[test]
1461    fn test_jsonl_counts_non_object_records_and_keeps_scanning() {
1462        for (kind, value) in NON_OBJECT_VALUES {
1463            let data = format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n");
1464            let mut rows = 0;
1465
1466            let summary = scan_json_from_reader(
1467                Cursor::new(data.as_bytes()),
1468                &JsonParserConfig::jsonl(),
1469                |_| rows += 1,
1470            )
1471            .unwrap();
1472
1473            assert_eq!(summary.rows_read, 2, "{kind}: record after it was lost");
1474            assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1475            assert_eq!(rows, 2, "{kind}");
1476        }
1477    }
1478
1479    #[test]
1480    fn test_json_array_counts_non_object_elements_and_keeps_scanning() {
1481        for (kind, value) in NON_OBJECT_VALUES {
1482            let data = format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]");
1483            let mut rows = 0;
1484
1485            let summary = scan_json_from_reader(
1486                Cursor::new(data.as_bytes()),
1487                &JsonParserConfig::json_document(),
1488                |_| rows += 1,
1489            )
1490            .unwrap();
1491
1492            assert_eq!(summary.rows_read, 2, "{kind}: element after it was lost");
1493            assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1494            assert_eq!(rows, 2, "{kind}");
1495        }
1496    }
1497
1498    #[test]
1499    fn test_strict_policy_rejects_the_first_non_object_record() {
1500        for (kind, value) in NON_OBJECT_VALUES {
1501            for (config, data) in [
1502                (
1503                    JsonParserConfig::jsonl(),
1504                    format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n"),
1505                ),
1506                (
1507                    JsonParserConfig::json_document(),
1508                    format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]"),
1509                ),
1510            ] {
1511                let config = config.with_error_policy(JsonErrorPolicy::Strict);
1512                let err = scan_json_from_reader(Cursor::new(data.as_bytes()), &config, |_| {})
1513                    .expect_err("{kind}: strict mode must reject a non-object record");
1514
1515                let message = err.to_string();
1516                assert!(message.contains("non-object JSON record"), "{message}");
1517                assert!(message.contains("at position 2"), "{message}");
1518                assert!(message.contains(&format!("found {kind}")), "{message}");
1519            }
1520        }
1521    }
1522
1523    #[test]
1524    fn test_number_element_does_not_swallow_the_array_delimiter() {
1525        // serde ends a number by peeking one byte past it and drops that byte
1526        // with the deserializer; here it is the `,` the array scanner needs.
1527        let data = br#"[{"id":1}, 1, 2, 3, {"id":2}, 4, {"id":3}]"#;
1528        let mut rows = 0;
1529
1530        let summary = scan_json_from_reader(
1531            Cursor::new(data.as_ref()),
1532            &JsonParserConfig::json_document(),
1533            |_| rows += 1,
1534        )
1535        .unwrap();
1536
1537        assert_eq!(summary.rows_read, 3);
1538        assert_eq!(summary.malformed_lines, 4);
1539        assert_eq!(rows, 3);
1540    }
1541
1542    #[test]
1543    fn test_number_shaped_garbage_stays_a_malformed_record() {
1544        // `1.2.3` is read by the byte-wise number scanner but is not a number,
1545        // so it must not be reported as a valid non-object record.
1546        let data = b"{\"id\":1}\n1.2.3\n";
1547        let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1548        let err = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {})
1549            .expect_err("number-shaped garbage should be malformed");
1550
1551        assert!(
1552            err.to_string().contains("malformed JSON record"),
1553            "{err}, expected the malformed category"
1554        );
1555    }
1556
1557    #[test]
1558    fn test_input_of_only_non_object_records_fails() {
1559        let file = write_file("\"just a string\"\n1\n[2]\n");
1560        let err = analyze_json_file(file.path(), &JsonParserConfig::jsonl())
1561            .expect_err("nothing profileable should not return a clean empty profile");
1562
1563        assert!(
1564            err.to_string().contains("No valid JSON records found"),
1565            "{err}"
1566        );
1567    }
1568
1569    #[test]
1570    /// The JSON grammar accepts both shapes that carry records, so a bare object
1571    /// is one row rather than an error (#486).
1572    fn test_json_grammar_reads_a_bare_object_as_one_row() {
1573        let data = br#"{"x":1}"#;
1574        let mut rows = 0;
1575        let summary = scan_json_from_reader(
1576            Cursor::new(data.as_ref()),
1577            &JsonParserConfig::json_document(),
1578            |_| rows += 1,
1579        )
1580        .unwrap();
1581
1582        assert_eq!(summary.rows_read, 1);
1583        assert_eq!(summary.malformed_lines, 0);
1584        assert_eq!(rows, 1);
1585    }
1586
1587    #[test]
1588    /// Several objects back to back are not a JSON document. The error names
1589    /// the option that reads them, because this is what a JSONL file looks like
1590    /// when it is read with the wrong grammar.
1591    fn test_json_grammar_rejects_concatenated_objects() {
1592        let data = br#"{"x":1}{"x":2}"#;
1593        let err = scan_json_from_reader(
1594            Cursor::new(data.as_ref()),
1595            &JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict),
1596            |_| {},
1597        )
1598        .expect_err("two objects are not one JSON document");
1599
1600        let message = err.to_string();
1601        assert!(message.contains("malformed JSON document"), "{message}");
1602        assert!(message.contains("format=\"jsonl\""), "{message}");
1603    }
1604
1605    #[test]
1606    fn test_analyze_json_from_reader_jsonl_streaming() {
1607        let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
1608        let cursor = Cursor::new(data.as_ref());
1609        let config = JsonParserConfig::default();
1610
1611        let (profiles, _stats, rows, _malformed, format) =
1612            analyze_json_from_reader(cursor, &config).unwrap();
1613        assert_eq!(format, FileFormat::Jsonl);
1614        assert_eq!(rows, 3);
1615        assert_eq!(profiles.len(), 2);
1616    }
1617
1618    #[test]
1619    fn test_analyze_json_from_reader_json_array() {
1620        let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
1621        let cursor = Cursor::new(data.as_ref());
1622        let config = JsonParserConfig::default();
1623
1624        let (profiles, _stats, rows, _malformed, format) =
1625            analyze_json_from_reader(cursor, &config).unwrap();
1626        assert_eq!(format, FileFormat::Json);
1627        assert_eq!(rows, 2);
1628        assert_eq!(profiles.len(), 2);
1629    }
1630
1631    #[test]
1632    fn test_analyze_json_from_reader_max_rows() {
1633        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
1634        let cursor = Cursor::new(data.as_ref());
1635        let config = JsonParserConfig::default().with_max_rows(3);
1636
1637        let (_profiles, _stats, rows, _malformed, _format) =
1638            analyze_json_from_reader(cursor, &config).unwrap();
1639        assert_eq!(rows, 3);
1640    }
1641
1642    #[test]
1643    fn test_analyze_json_from_reader_missing_fields() {
1644        let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
1645        let cursor = Cursor::new(data.as_ref());
1646        let config = JsonParserConfig::jsonl();
1647
1648        let (profiles, _stats, rows, _malformed, _format) =
1649            analyze_json_from_reader(cursor, &config).unwrap();
1650        assert_eq!(rows, 2);
1651
1652        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1653        assert_eq!(col_b.total_count, 2);
1654    }
1655
1656    #[test]
1657    fn test_analyze_json_file_quality_report() {
1658        let file = write_file(r#"[{"x":1},{"x":2}]"#);
1659        let config = JsonParserConfig::default();
1660        let report = analyze_json_file(file.path(), &config).unwrap();
1661
1662        assert_eq!(report.execution.rows_processed, 2);
1663        assert_eq!(report.column_profiles.len(), 1);
1664        assert!(report.quality_score().unwrap() >= 0.0);
1665    }
1666
1667    #[test]
1668    fn test_jsonl_skips_malformed_lines() {
1669        let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
1670        let cursor = Cursor::new(data.as_ref());
1671        let config = JsonParserConfig::jsonl();
1672
1673        let (profiles, _stats, rows, _malformed, format) =
1674            analyze_json_from_reader(cursor, &config).unwrap();
1675        assert_eq!(format, FileFormat::Jsonl);
1676        assert_eq!(rows, 2);
1677        assert_eq!(profiles[0].total_count, 2);
1678    }
1679
1680    #[test]
1681    fn test_strict_policy_errors_on_malformed_jsonl() {
1682        // Malformed in first / middle / last position all abort under Strict.
1683        for data in [
1684            b"not-json\n{\"x\":1}\n{\"x\":2}\n".as_ref(),
1685            b"{\"x\":1}\nnot-json\n{\"x\":2}\n".as_ref(),
1686            b"{\"x\":1}\n{\"x\":2}\nnot-json\n".as_ref(),
1687        ] {
1688            let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1689            let err = scan_json_from_reader(Cursor::new(data), &config, |_| {})
1690                .expect_err("strict mode must reject malformed records");
1691            let message = err.to_string().to_lowercase();
1692            assert!(message.contains("malformed json record"), "got: {message}");
1693            // The offending record text must never be echoed back.
1694            assert!(
1695                !err.to_string().contains("not-json"),
1696                "leaked record: {err}"
1697            );
1698        }
1699    }
1700
1701    #[test]
1702    fn test_tolerant_policy_counts_skipped_records() {
1703        let data = b"{\"x\":1}\nnot-json\n{\"x\":2}\n";
1704        let config = JsonParserConfig::jsonl(); // default Skip
1705        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1706        assert_eq!(summary.rows_read, 2);
1707        assert_eq!(summary.malformed_lines, 1);
1708    }
1709
1710    #[test]
1711    fn test_blank_lines_are_not_counted_as_malformed() {
1712        let data = b"{\"x\":1}\n\n   \n{\"x\":2}\n";
1713        for policy in [JsonErrorPolicy::Skip, JsonErrorPolicy::Strict] {
1714            let config = JsonParserConfig::jsonl().with_error_policy(policy);
1715            let summary =
1716                scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1717            assert_eq!(summary.rows_read, 2, "policy {policy:?}");
1718            assert_eq!(summary.malformed_lines, 0, "policy {policy:?}");
1719        }
1720    }
1721
1722    #[test]
1723    fn test_truncated_final_record_obeys_error_policy() {
1724        let data = b"{\"x\":1}\n{\"x\":2";
1725
1726        let skip = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Skip);
1727        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &skip, |_| {}).unwrap();
1728        assert_eq!(summary.rows_read, 1);
1729        assert_eq!(summary.malformed_lines, 1);
1730
1731        let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1732        let err = scan_json_from_reader(Cursor::new(data.as_ref()), &strict, |_| {})
1733            .expect_err("strict mode must reject an incomplete trailing record");
1734        assert!(
1735            err.to_string()
1736                .to_lowercase()
1737                .contains("malformed json record")
1738        );
1739    }
1740
1741    #[test]
1742    fn test_all_malformed_file_fails() {
1743        let file = write_file("not-json\nalso-bad\n");
1744        let config = JsonParserConfig::jsonl();
1745        let err = analyze_json_file(file.path(), &config)
1746            .expect_err("a file with no valid records must fail");
1747        let message = err.to_string().to_lowercase();
1748        assert!(message.contains("no valid json records"), "got: {message}");
1749    }
1750
1751    #[test]
1752    fn test_tolerant_file_reports_error_count() {
1753        let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1754        let config = JsonParserConfig::jsonl();
1755        let report = analyze_json_file(file.path(), &config).unwrap();
1756        assert_eq!(report.execution.rows_processed, 2);
1757        assert_eq!(report.execution.error_count, 1);
1758    }
1759
1760    #[test]
1761    fn test_strict_file_errors_on_malformed() {
1762        let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1763        let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1764        let err = analyze_json_file(file.path(), &config)
1765            .expect_err("strict mode must reject the malformed record");
1766        assert!(
1767            err.to_string()
1768                .to_lowercase()
1769                .contains("malformed json record")
1770        );
1771    }
1772
1773    #[test]
1774    fn test_analyze_json_with_large_leading_whitespace() {
1775        let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
1776        let cursor = Cursor::new(data.into_bytes());
1777        let config = JsonParserConfig::default();
1778
1779        let (_profiles, _stats, rows, malformed, format) =
1780            analyze_json_from_reader(cursor, &config).unwrap();
1781        assert_eq!(format, FileFormat::Json);
1782        assert_eq!(rows, 1);
1783        assert_eq!(malformed, 0);
1784    }
1785
1786    #[test]
1787    fn test_analyze_json_array() {
1788        let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
1789        let config = JsonParserConfig::default();
1790        let report = analyze_json_file(json.path(), &config).unwrap();
1791        let profiles = &report.column_profiles;
1792
1793        assert_eq!(profiles.len(), 2);
1794        let names: Vec<&str> = profiles
1795            .iter()
1796            .map(|profile| profile.name.as_str())
1797            .collect();
1798        assert!(names.contains(&"name"));
1799        assert!(names.contains(&"age"));
1800
1801        let age = profiles
1802            .iter()
1803            .find(|profile| profile.name == "age")
1804            .unwrap();
1805        assert_eq!(age.total_count, 2);
1806        assert_eq!(age.null_count, 0);
1807    }
1808
1809    #[test]
1810    fn test_analyze_jsonl() {
1811        let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
1812        let config = JsonParserConfig::default();
1813        let report = analyze_json_file(jsonl.path(), &config).unwrap();
1814        let profiles = &report.column_profiles;
1815
1816        assert_eq!(profiles.len(), 1);
1817        assert_eq!(profiles[0].name, "x");
1818        assert_eq!(profiles[0].total_count, 3);
1819    }
1820
1821    #[test]
1822    fn test_analyze_json_with_nulls() {
1823        let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
1824        let config = JsonParserConfig::default();
1825        let report = analyze_json_file(json.path(), &config).unwrap();
1826        let profiles = &report.column_profiles;
1827
1828        let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
1829        assert_eq!(col_a.null_count, 1);
1830
1831        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1832        assert_eq!(col_b.null_count, 1);
1833    }
1834
1835    #[test]
1836    fn test_analyze_json_with_missing_fields() {
1837        let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
1838        let config = JsonParserConfig::default();
1839        let report = analyze_json_file(json.path(), &config).unwrap();
1840        let profiles = &report.column_profiles;
1841
1842        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1843        assert_eq!(col_b.total_count, 2);
1844    }
1845
1846    /// Deliberately non-alphabetical field order: sorting would give
1847    /// `active, amount, date, id`, so any re-sort is visible.
1848    const UNSORTED_FIELDS: &str = r#"{"id":1,"amount":12.5,"active":true,"date":"2026-07-23"}"#;
1849    const UNSORTED_ORDER: [&str; 4] = ["id", "amount", "active", "date"];
1850
1851    fn profile_names(report: &ProfileReport) -> Vec<&str> {
1852        report
1853            .column_profiles
1854            .iter()
1855            .map(|profile| profile.name.as_str())
1856            .collect()
1857    }
1858
1859    #[test]
1860    fn test_json_column_order_follows_source_not_alphabet() {
1861        let array = write_file(&format!("[{UNSORTED_FIELDS},{UNSORTED_FIELDS}]"));
1862        let jsonl = write_file(&format!("{UNSORTED_FIELDS}\n{UNSORTED_FIELDS}\n"));
1863
1864        for file in [&array, &jsonl] {
1865            let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1866            assert_eq!(profile_names(&report), UNSORTED_ORDER);
1867        }
1868    }
1869
1870    #[test]
1871    fn test_json_column_order_appends_later_keys_in_first_seen_order() {
1872        // Each record introduces two keys in reverse-alphabetical order, so
1873        // sorting would reshuffle both the leading pair and the appended pair.
1874        let data = b"{\"zulu\":1,\"mike\":2}\n{\"zulu\":3,\"mike\":4,\"delta\":5,\"alpha\":6}\n";
1875        let file = write_bytes(data);
1876        let report = analyze_json_file(file.path(), &JsonParserConfig::jsonl()).unwrap();
1877
1878        assert_eq!(profile_names(&report), ["zulu", "mike", "delta", "alpha"]);
1879    }
1880
1881    #[test]
1882    fn test_json_column_order_survives_report_serialization() {
1883        let file = write_file(&format!("[{UNSORTED_FIELDS}]"));
1884        let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1885
1886        let json = serde_json::to_string(&report).unwrap();
1887        let restored: Value = serde_json::from_str(&json).unwrap();
1888        let names: Vec<&str> = restored["column_profiles"]
1889            .as_array()
1890            .unwrap()
1891            .iter()
1892            .map(|profile| profile["name"].as_str().unwrap())
1893            .collect();
1894
1895        assert_eq!(names, UNSORTED_ORDER);
1896    }
1897
1898    #[test]
1899    fn test_analyze_json_backfills_columns_discovered_after_first_row() {
1900        let json = write_file(r#"[{"a":1},{"a":2,"b":3}]"#);
1901        let config = JsonParserConfig::default();
1902        let report = analyze_json_file(json.path(), &config).unwrap();
1903        let col_b = report
1904            .column_profiles
1905            .iter()
1906            .find(|profile| profile.name == "b")
1907            .unwrap();
1908
1909        assert_eq!(col_b.total_count, 2);
1910        assert_eq!(col_b.null_count, 1);
1911        assert_eq!(col_b.unique_count, Some(1));
1912    }
1913
1914    #[test]
1915    fn test_analyze_json_empty_array() {
1916        let json = write_file("[]");
1917        let config = JsonParserConfig::default();
1918        let report = analyze_json_file(json.path(), &config).unwrap();
1919        assert!(report.column_profiles.is_empty());
1920        // An empty dataset has nothing to assess: no score is fabricated.
1921        assert_eq!(report.quality_score(), None);
1922    }
1923
1924    #[test]
1925    fn test_analyze_json_malformed_returns_error() {
1926        let json = write_file("this is entirely invalid json");
1927        let config = JsonParserConfig::default();
1928        let err = analyze_json_file(json.path(), &config)
1929            .expect_err("malformed JSON should return an error");
1930
1931        let message = err.to_string().to_lowercase();
1932        assert!(message.contains("malformed") && message.contains("json"));
1933    }
1934
1935    #[test]
1936    fn test_analyze_json_file_detects_format() {
1937        let json_array = write_file(r#"[{"x":1}]"#);
1938        let config = JsonParserConfig::default();
1939        let report = analyze_json_file(json_array.path(), &config).unwrap();
1940        assert!(matches!(
1941            report.data_source,
1942            DataSource::File {
1943                format: FileFormat::Json,
1944                ..
1945            }
1946        ));
1947
1948        let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
1949        let report = analyze_json_file(jsonl.path(), &config).unwrap();
1950        assert!(matches!(
1951            report.data_source,
1952            DataSource::File {
1953                format: FileFormat::Jsonl,
1954                ..
1955            }
1956        ));
1957    }
1958
1959    #[test]
1960    fn test_bom_prefixed_file_preserves_source_size() {
1961        let data = b"\xEF\xBB\xBF[{\"x\":1}]";
1962        let json = write_bytes(data);
1963        let report = analyze_json_file(json.path(), &JsonParserConfig::json_document()).unwrap();
1964
1965        assert_eq!(report.execution.rows_processed, 1);
1966        assert!(matches!(
1967            report.data_source,
1968            DataSource::File {
1969                format: FileFormat::Json,
1970                size_bytes,
1971                ..
1972            } if size_bytes == data.len() as u64
1973        ));
1974    }
1975
1976    #[test]
1977    fn test_analyze_json_file_empty() {
1978        let json = write_file("");
1979        let config = JsonParserConfig::default();
1980        let report = analyze_json_file(json.path(), &config).unwrap();
1981        assert_eq!(report.execution.rows_processed, 0);
1982        assert!(report.column_profiles.is_empty());
1983    }
1984
1985    #[test]
1986    fn test_analyze_json_boolean_and_nested() {
1987        let json =
1988            write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
1989        let config = JsonParserConfig::default();
1990        let report = analyze_json_file(json.path(), &config).unwrap();
1991        let profiles = &report.column_profiles;
1992
1993        let flag = profiles
1994            .iter()
1995            .find(|profile| profile.name == "flag")
1996            .unwrap();
1997        assert_eq!(flag.total_count, 2);
1998
1999        let nested = profiles
2000            .iter()
2001            .find(|profile| profile.name == "nested")
2002            .unwrap();
2003        assert_eq!(nested.total_count, 2);
2004    }
2005
2006    #[test]
2007    fn test_single_root_object_compact_yields_one_row() {
2008        let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
2009        let cursor = Cursor::new(data.as_ref());
2010        let config = JsonParserConfig::default();
2011
2012        let (profiles, _stats, rows, malformed, _format) =
2013            analyze_json_from_reader(cursor, &config).unwrap();
2014        assert_eq!(rows, 1);
2015        assert_eq!(malformed, 0);
2016        assert_eq!(profiles.len(), 2);
2017
2018        let names: Vec<&str> = profiles
2019            .iter()
2020            .map(|profile| profile.name.as_str())
2021            .collect();
2022        assert!(names.contains(&"type"));
2023        assert!(names.contains(&"features"));
2024    }
2025
2026    #[test]
2027    fn test_jsonl_multi_object_still_works() {
2028        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
2029        let cursor = Cursor::new(data.as_ref());
2030        let config = JsonParserConfig::default();
2031
2032        let (_profiles, _stats, rows, _malformed, format) =
2033            analyze_json_from_reader(cursor, &config).unwrap();
2034        assert_eq!(format, FileFormat::Jsonl);
2035        assert_eq!(rows, 3);
2036    }
2037
2038    #[test]
2039    fn test_single_root_object_via_analyze_json_file() {
2040        let file =
2041            write_json_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
2042        let config = JsonParserConfig::default();
2043        let report = analyze_json_file(file.path(), &config).unwrap();
2044
2045        assert_eq!(report.execution.rows_processed, 1);
2046        assert_eq!(report.column_profiles.len(), 2);
2047    }
2048
2049    #[test]
2050    fn test_single_root_object_pretty_printed_yields_one_row() {
2051        let data = br#"{
2052  "type": "FeatureCollection",
2053  "features": [
2054    {"id": 1},
2055    {"id": 2}
2056  ]
2057}"#;
2058        let cursor = Cursor::new(data.as_ref());
2059        // A reader has no file name, so the grammar cannot be inferred and is
2060        // named explicitly; `analyze_json_file` takes it from the extension.
2061        let config = JsonParserConfig::json_document();
2062
2063        let (profiles, _stats, rows, malformed, format) =
2064            analyze_json_from_reader(cursor, &config).unwrap();
2065        assert_eq!(format, FileFormat::Json);
2066        assert_eq!(rows, 1);
2067        assert_eq!(malformed, 0);
2068        assert_eq!(profiles.len(), 2);
2069    }
2070
2071    #[test]
2072    fn test_single_root_object_pretty_printed_via_analyze_json_file() {
2073        let file = write_json_file(
2074            "{\n  \"type\": \"FeatureCollection\",\n  \"features\": [\n    {\"id\": 1},\n    {\"id\": 2}\n  ]\n}\n",
2075        );
2076        let config = JsonParserConfig::default();
2077        let report = analyze_json_file(file.path(), &config).unwrap();
2078
2079        assert_eq!(report.execution.rows_processed, 1);
2080        assert_eq!(report.column_profiles.len(), 2);
2081    }
2082}