Skip to main content

dataprof_json/
lib.rs

1//! JSON and JSONL scanning and profiling for `dataprof`.
2//!
3//! This crate is an implementation detail of the `dataprof` facade, which
4//! re-exports [`JsonParserConfig`], [`analyze_json_from_reader`], and
5//! [`analyze_json_file`]. Depend on `dataprof` unless you need JSON support
6//! without the rest of the workspace.
7//!
8//! Two entry shapes are offered. [`scan_json_from_reader`] hands each record to
9//! a callback and reports what it read; [`analyze_json_from_reader`] and
10//! [`analyze_json_file`] build column profiles and a full report on top of it.
11//!
12//! ```
13//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
14//! use std::io::Cursor;
15//!
16//! use dataprof_json::{JsonParserConfig, analyze_json_from_reader};
17//!
18//! let jsonl = "{\"id\": 1, \"city\": \"Rome\"}\n{\"id\": 2, \"city\": \"Milan\"}\n";
19//! let (profiles, _stats, rows_read, malformed_lines, _format) =
20//!     analyze_json_from_reader(Cursor::new(jsonl), &JsonParserConfig::jsonl())?;
21//!
22//! assert_eq!(rows_read, 2);
23//! assert_eq!(malformed_lines, 0);
24//! assert_eq!(profiles[0].name, "id");
25//! # Ok(())
26//! # }
27//! ```
28
29use std::collections::{HashMap, HashSet};
30use std::io::BufRead;
31use std::path::Path;
32
33pub use dataprof_core::JsonErrorPolicy;
34use dataprof_core::{
35    AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat,
36    QualityDimension, SemanticHints, TruncationReason, Utf8BomReader,
37};
38use dataprof_runtime::{
39    ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
40};
41use serde::Deserialize;
42use serde_json::Value;
43
44/// Which grammar a JSON source is read with.
45///
46/// The two are deliberately distinct: whitespace is insignificant inside a
47/// standard JSON document and load-bearing in JSONL, so one input cannot be
48/// valid under both readings and mean the same thing. Callers pick with
49/// `format="json"` / `format="jsonl"`, or by file extension.
50#[derive(Debug, Clone, Copy, PartialEq, Eq)]
51pub enum JsonFormat {
52    /// A standard JSON document, in either of the two shapes that carry
53    /// records: an array of objects (`[{...}, {...}]`), or a single object
54    /// (`{...}`) as one record. Values may span lines and be pretty-printed,
55    /// and exactly one document must fill the whole input.
56    Json,
57    /// JSON Lines — one record per physical line. A record may not span lines,
58    /// and a line may not hold more than one value.
59    Jsonl,
60}
61
62/// Configuration for JSON/JSONL parsing and scanning.
63///
64/// The default detects the grammar from the input and reads every record,
65/// skipping malformed ones. The two format constructors say which grammar to
66/// read with, which is what a caller who knows should do: detection cannot tell
67/// a JSON object document from a JSONL file, since both open with `{`.
68///
69/// # Examples
70///
71/// The same two records under each grammar:
72///
73/// ```
74/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
75/// use std::io::Cursor;
76///
77/// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
78///
79/// let document = r#"[{"id": 1}, {"id": 2}]"#;
80/// let lines = "{\"id\": 1}\n{\"id\": 2}\n";
81///
82/// let from_document = scan_json_from_reader(
83///     Cursor::new(document),
84///     &JsonParserConfig::json_document(),
85///     |_| {},
86/// )?;
87/// let from_lines =
88///     scan_json_from_reader(Cursor::new(lines), &JsonParserConfig::jsonl(), |_| {})?;
89///
90/// assert_eq!(from_document.rows_read, 2);
91/// assert_eq!(from_lines.rows_read, 2);
92/// # Ok(())
93/// # }
94/// ```
95#[derive(Debug, Clone, Default)]
96pub struct JsonParserConfig {
97    /// The grammar to read with. `None` detects it from the first byte, which
98    /// resolves `[` to [`JsonFormat::Json`] and everything else to
99    /// [`JsonFormat::Jsonl`] — a leading `{` opens both a JSON object document
100    /// and a JSONL file, so callers that know which they have (from the file
101    /// extension or an explicit `format=`) should say so rather than rely on
102    /// detection.
103    pub format: Option<JsonFormat>,
104    /// Maximum rows to process (None = all rows).
105    pub max_rows: Option<usize>,
106    /// How to react to a malformed record (default: skip).
107    pub error_policy: JsonErrorPolicy,
108}
109
110impl JsonParserConfig {
111    /// Set the maximum number of rows to process.
112    ///
113    /// The cap counts profileable records, not lines: blank lines and records
114    /// skipped as malformed do not consume it.
115    ///
116    /// A cap is only *truncation* when a record still remained once it was
117    /// reached, so a source holding exactly `max_rows` records reads as
118    /// complete. [`JsonScanSummary::truncated`] carries that distinction, and
119    /// [`analyze_json_file`] turns it into a [`TruncationReason::MaxRows`] on
120    /// the report.
121    ///
122    /// # Examples
123    ///
124    /// ```
125    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
126    /// use std::io::Cursor;
127    ///
128    /// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
129    ///
130    /// let lines = "{\"id\": 1}\n{\"id\": 2}\n{\"id\": 3}\n";
131    ///
132    /// // Two of three records: a third remained, so the scan was cut short.
133    /// let capped = JsonParserConfig::jsonl().with_max_rows(2);
134    /// let summary = scan_json_from_reader(Cursor::new(lines), &capped, |_| {})?;
135    /// assert_eq!(summary.rows_read, 2);
136    /// assert!(summary.truncated);
137    ///
138    /// // A cap the source never reaches past is a complete read, not a
139    /// // truncated one.
140    /// let exact = JsonParserConfig::jsonl().with_max_rows(3);
141    /// let summary = scan_json_from_reader(Cursor::new(lines), &exact, |_| {})?;
142    /// assert_eq!(summary.rows_read, 3);
143    /// assert!(!summary.truncated);
144    /// # Ok(())
145    /// # }
146    /// ```
147    pub fn with_max_rows(mut self, max_rows: usize) -> Self {
148        self.max_rows = Some(max_rows);
149        self
150    }
151
152    /// Set how malformed records are handled.
153    ///
154    /// Under the default [`JsonErrorPolicy::Skip`] a malformed record is
155    /// counted in [`JsonScanSummary::malformed_lines`] and scanning continues,
156    /// so a partial profile never looks like a clean one. Under
157    /// [`JsonErrorPolicy::Strict`] the first one fails the scan.
158    ///
159    /// # Examples
160    ///
161    /// ```
162    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
163    /// use std::io::Cursor;
164    ///
165    /// use dataprof_json::{JsonErrorPolicy, JsonParserConfig, scan_json_from_reader};
166    ///
167    /// let lines = "{\"id\": 1}\nnot json\n{\"id\": 2}\n";
168    ///
169    /// let skipping = JsonParserConfig::jsonl();
170    /// let summary = scan_json_from_reader(Cursor::new(lines), &skipping, |_| {})?;
171    /// assert_eq!(summary.rows_read, 2);
172    /// assert_eq!(summary.malformed_lines, 1);
173    ///
174    /// let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
175    /// assert!(scan_json_from_reader(Cursor::new(lines), &strict, |_| {}).is_err());
176    /// # Ok(())
177    /// # }
178    /// ```
179    pub fn with_error_policy(mut self, policy: JsonErrorPolicy) -> Self {
180        self.error_policy = policy;
181        self
182    }
183
184    /// Read as JSON Lines: one record per physical line.
185    pub fn jsonl() -> Self {
186        Self {
187            format: Some(JsonFormat::Jsonl),
188            ..Default::default()
189        }
190    }
191
192    /// Read as a standard JSON document: an array of records, or a single
193    /// object as one record.
194    pub fn json_document() -> Self {
195        Self {
196            format: Some(JsonFormat::Json),
197            ..Default::default()
198        }
199    }
200
201    /// Set the grammar explicitly.
202    pub fn with_format(mut self, format: JsonFormat) -> Self {
203        self.format = Some(format);
204        self
205    }
206}
207
208/// Borrowed JSON object callback payload.
209///
210/// The workspace enables `serde_json/preserve_order`, so iterating this map
211/// yields fields in the order they appear in the source document rather than
212/// alphabetically. That is what makes the column-ordering contract on
213/// [`analyze_json_from_reader`] possible.
214pub type JsonObject = serde_json::Map<String, Value>;
215
216/// Summary of a JSON/JSONL scan.
217#[derive(Debug, Clone, PartialEq, Eq)]
218#[non_exhaustive]
219pub struct JsonScanSummary {
220    pub rows_read: usize,
221    /// Records that were read but not profiled: malformed JSON, and valid JSON
222    /// values that are not objects and therefore carry no fields. Both are
223    /// counted so a tolerant scan never looks like a clean one.
224    pub malformed_lines: usize,
225    pub format: FileFormat,
226    /// The scan stopped at `max_rows` while records still remained. A source
227    /// holding exactly `max_rows` records was read in full, so this stays false.
228    pub truncated: bool,
229}
230
231/// Scan JSON or JSONL input and invoke `on_object` for each object record.
232///
233/// - **JSONL**: one record per physical line. A record may not span lines and a
234///   line may not hold more than one value, so a concatenation whose delimiter
235///   was lost is reported rather than read as several clean records. Blank lines
236///   are separators: neither records nor errors.
237/// - **JSON**: one standard document. An array of objects streams element by
238///   element without buffering the whole input; a single object is read whole
239///   and is one record, which is why it may be pretty-printed across lines.
240///
241/// # Record policy
242///
243/// Only JSON objects are profileable records — they are the only JSON value
244/// with named fields to turn into columns. A record that is valid JSON but not
245/// an object (a scalar, an array, `null`) is never silently discarded: under
246/// [`JsonErrorPolicy::Skip`] it is counted in
247/// [`JsonScanSummary::malformed_lines`] and scanning continues with the next
248/// record, and under [`JsonErrorPolicy::Strict`] the first one fails the scan.
249///
250/// # Examples
251///
252/// Reading JSONL, keeping one field per record:
253///
254/// ```
255/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
256/// use std::io::Cursor;
257///
258/// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
259///
260/// let lines = "{\"city\": \"Rome\", \"pop\": 2800000}\n\
261///              {\"city\": \"Milan\", \"pop\": 1400000}\n";
262///
263/// let mut cities = Vec::new();
264/// let summary = scan_json_from_reader(
265///     Cursor::new(lines),
266///     &JsonParserConfig::jsonl(),
267///     |object| {
268///         if let Some(city) = object.get("city").and_then(|value| value.as_str()) {
269///             cities.push(city.to_string());
270///         }
271///     },
272/// )?;
273///
274/// assert_eq!(cities, ["Rome", "Milan"]);
275/// assert_eq!(summary.rows_read, 2);
276/// assert!(!summary.truncated);
277/// # Ok(())
278/// # }
279/// ```
280///
281/// The same records as a JSON array. The array streams element by element, and
282/// a record may be pretty-printed across lines because whitespace carries no
283/// meaning under this grammar:
284///
285/// ```
286/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
287/// use std::io::Cursor;
288///
289/// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
290///
291/// let document = r#"[
292///     {"city": "Rome", "pop": 2800000},
293///     {"city": "Milan", "pop": 1400000}
294/// ]"#;
295///
296/// let mut populations = Vec::new();
297/// let summary = scan_json_from_reader(
298///     Cursor::new(document),
299///     &JsonParserConfig::json_document(),
300///     |object| {
301///         // A field that is absent, or present but not a number, stays
302///         // `None`. Reading it as 0 instead would turn a missing value into
303///         // a real measurement, which is the one thing a profile must never
304///         // do.
305///         populations.push(object.get("pop").and_then(|value| value.as_u64()));
306///     },
307/// )?;
308///
309/// assert_eq!(populations, [Some(2_800_000), Some(1_400_000)]);
310/// assert_eq!(summary.rows_read, 2);
311/// assert_eq!(summary.malformed_lines, 0);
312/// # Ok(())
313/// # }
314/// ```
315pub fn scan_json_from_reader<R, F>(
316    reader: R,
317    config: &JsonParserConfig,
318    mut on_object: F,
319) -> Result<JsonScanSummary, DataProfilerError>
320where
321    R: BufRead,
322    F: FnMut(&JsonObject),
323{
324    let mut reader = Utf8BomReader::new(reader).map_err(DataProfilerError::from)?;
325    let format = match config.format {
326        Some(JsonFormat::Json) => FileFormat::Json,
327        Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
328        // A leading `{` is the first byte of both a JSON object document and a
329        // JSONL file, so detection can only resolve the array case and has to
330        // fall back to JSONL. Callers that know better pass `format`.
331        None => match consume_leading_whitespace(&mut reader)? {
332            Some(b'[') => FileFormat::Json,
333            _ => FileFormat::Jsonl,
334        },
335    };
336
337    let mut rows_read = 0;
338    let mut malformed_lines = 0;
339    let mut truncated = false;
340
341    match format {
342        // JSONL is line-delimited: exactly one record per non-blank physical
343        // line. Reading it as a whitespace-delimited stream of values instead
344        // made `{"a":1}{"b":2}` — a concatenation with the delimiter lost —
345        // profile as two clean rows, which is precisely the corruption a
346        // profiler exists to surface.
347        FileFormat::Jsonl => {
348            let mut line = String::new();
349            let mut line_number = 0;
350            loop {
351                if let Some(max) = config.max_rows
352                    && rows_read >= max
353                {
354                    // Reaching the cap is not truncation unless a record
355                    // actually remains: a file with exactly `max_rows` records
356                    // was read in full. Blank lines are not records.
357                    truncated = next_record_line(&mut reader, &mut line, &mut line_number)?;
358                    break;
359                }
360
361                if !next_record_line(&mut reader, &mut line, &mut line_number)? {
362                    break;
363                }
364
365                match read_jsonl_line(&line) {
366                    JsonRecord::Object(obj) => {
367                        on_object(&obj);
368                        rows_read += 1;
369                    }
370                    JsonRecord::NonObject(kind) => {
371                        if config.error_policy == JsonErrorPolicy::Strict {
372                            return Err(non_object_record_error(
373                                kind,
374                                rows_read + malformed_lines + 1,
375                            ));
376                        }
377                        malformed_lines += 1;
378                    }
379                    JsonRecord::Malformed(err) => {
380                        if config.error_policy == JsonErrorPolicy::Strict {
381                            return Err(malformed_jsonl_record_error(line_number, err));
382                        }
383                        malformed_lines += 1;
384                    }
385                }
386            }
387        }
388        FileFormat::Json => {
389            // Which of the two record-carrying shapes this document is. The
390            // first non-whitespace byte decides, and is left unconsumed so the
391            // object branch still sees the value it opens.
392            let found_array = consume_leading_whitespace(&mut reader)? == Some(b'[');
393
394            if !found_array {
395                // The other shape a standard JSON document takes: a single
396                // object, which is one record. It may be pretty-printed across
397                // lines, which is why this is not the JSONL grammar.
398                let (object_rows, object_malformed, object_truncated) =
399                    scan_single_json_object(&mut reader, config, &mut on_object)?;
400                rows_read += object_rows;
401                malformed_lines += object_malformed;
402                truncated |= object_truncated;
403            } else {
404                // The array loop starts after the opening bracket.
405                consume_peeked(&mut reader)?;
406
407                let mut expect_value = true;
408                let mut allow_end = true;
409                let mut array_closed = false;
410                let mut drain_remainder = false;
411
412                loop {
413                    let Some(next) = consume_leading_whitespace(&mut reader)? else {
414                        if config.error_policy == JsonErrorPolicy::Strict {
415                            return Err(malformed_array_error(
416                                "unexpected end of input before closing ']'",
417                            ));
418                        }
419                        malformed_lines += 1;
420                        drain_remainder = true;
421                        break;
422                    };
423
424                    if expect_value {
425                        if next == b']' {
426                            if !allow_end {
427                                if config.error_policy == JsonErrorPolicy::Strict {
428                                    return Err(malformed_array_error(
429                                        "trailing comma before closing ']'",
430                                    ));
431                                }
432                                malformed_lines += 1;
433                                drain_remainder = true;
434                                break;
435                            }
436                            consume_peeked(&mut reader)?;
437                            array_closed = true;
438                            break;
439                        }
440
441                        if let Some(max) = config.max_rows
442                            && rows_read >= max
443                        {
444                            // A value remains unread. Do not validate the unread
445                            // suffix of an intentionally bounded scan.
446                            truncated = true;
447                            break;
448                        }
449
450                        if next == b',' {
451                            if config.error_policy == JsonErrorPolicy::Strict {
452                                return Err(malformed_array_error(
453                                    "unexpected comma where an array value was required",
454                                ));
455                            }
456                            malformed_lines += 1;
457                            drain_remainder = true;
458                            break;
459                        }
460
461                        match read_json_record(&mut reader, next)? {
462                            JsonRecord::Object(obj) => {
463                                on_object(&obj);
464                                rows_read += 1;
465                            }
466                            JsonRecord::NonObject(kind) => {
467                                if config.error_policy == JsonErrorPolicy::Strict {
468                                    return Err(non_object_record_error(
469                                        kind,
470                                        rows_read + malformed_lines + 1,
471                                    ));
472                                }
473                                // The element was consumed in full, so the array
474                                // grammar is still intact and the objects after
475                                // it are still profileable.
476                                malformed_lines += 1;
477                            }
478                            JsonRecord::Malformed(err) => {
479                                if config.error_policy == JsonErrorPolicy::Strict {
480                                    return Err(malformed_record_error(err));
481                                }
482                                malformed_lines += 1;
483                                drain_remainder = true;
484                                break;
485                            }
486                        }
487                        expect_value = false;
488                    } else {
489                        match next {
490                            b',' => {
491                                consume_peeked(&mut reader)?;
492                                expect_value = true;
493                                allow_end = false;
494                            }
495                            b']' => {
496                                consume_peeked(&mut reader)?;
497                                array_closed = true;
498                                break;
499                            }
500                            _ => {
501                                if config.error_policy == JsonErrorPolicy::Strict {
502                                    return Err(malformed_array_error(
503                                        "expected ',' or ']' after an array value",
504                                    ));
505                                }
506                                malformed_lines += 1;
507                                drain_remainder = true;
508                                break;
509                            }
510                        }
511                    }
512                }
513
514                if drain_remainder {
515                    drain_to_end(&mut reader)?;
516                } else if array_closed && consume_leading_whitespace(&mut reader)?.is_some() {
517                    if config.error_policy == JsonErrorPolicy::Strict {
518                        return Err(malformed_array_error(
519                            "non-whitespace content follows the closing ']'",
520                        ));
521                    }
522                    malformed_lines += 1;
523                    drain_to_end(&mut reader)?;
524                }
525            }
526        }
527        _ => unreachable!("json scanner only returns json or jsonl formats"),
528    }
529
530    Ok(JsonScanSummary {
531        rows_read,
532        malformed_lines,
533        format,
534        truncated,
535    })
536}
537
538/// Read the next line that holds a record into `line`, skipping blank ones.
539///
540/// Returns whether a record line was found. Blank lines are separators, not
541/// records: they are skipped without counting as rows or as errors. `line_number`
542/// counts every physical line read, blank ones included, so a diagnostic points
543/// at the line the user would open the file to.
544fn next_record_line<R: BufRead>(
545    reader: &mut R,
546    line: &mut String,
547    line_number: &mut usize,
548) -> Result<bool, DataProfilerError> {
549    loop {
550        line.clear();
551        let read = reader.read_line(line).map_err(DataProfilerError::from)?;
552        if read == 0 {
553            return Ok(false);
554        }
555        *line_number += 1;
556        if !line.trim().is_empty() {
557            return Ok(true);
558        }
559    }
560}
561
562/// Parse one JSONL line as exactly one JSON value.
563///
564/// `serde_json::from_str` requires the value to fill the whole input, so a line
565/// carrying two adjacent or space-separated values is a malformed record rather
566/// than two clean ones — the boundary rule that separates JSONL from a stream of
567/// JSON values.
568fn read_jsonl_line(line: &str) -> JsonRecord {
569    match serde_json::from_str::<Value>(line.trim()) {
570        Ok(Value::Object(obj)) => JsonRecord::Object(obj),
571        Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
572        Err(err) => JsonRecord::Malformed(err),
573    }
574}
575
576/// Read a whole-input single JSON object as one record.
577///
578/// Returns `(rows_read, malformed, truncated)`. Anything that is not exactly one object
579/// filling the input is an error under [`JsonErrorPolicy::Strict`] and a counted
580/// malformed record otherwise, so a truncated or concatenated document never
581/// profiles as clean.
582fn scan_single_json_object<R, F>(
583    reader: &mut R,
584    config: &JsonParserConfig,
585    on_object: &mut F,
586) -> Result<(usize, usize, bool), DataProfilerError>
587where
588    R: BufRead,
589    F: FnMut(&JsonObject),
590{
591    let mut text = String::new();
592    reader
593        .read_to_string(&mut text)
594        .map_err(DataProfilerError::from)?;
595
596    if text.trim().is_empty() {
597        return Ok((0, 0, false));
598    }
599
600    // A row cap of zero asks for no records, so the document is not read. The
601    // record it holds is left unread, which is a truncation and has to be
602    // reported as one — an unread record and an empty source must not produce
603    // the same report.
604    if config.max_rows == Some(0) {
605        return Ok((0, 0, true));
606    }
607
608    match serde_json::from_str::<Value>(text.trim()) {
609        Ok(Value::Object(obj)) => {
610            on_object(&obj);
611            Ok((1, 0, false))
612        }
613        Ok(value) => {
614            if config.error_policy == JsonErrorPolicy::Strict {
615                return Err(non_object_record_error(json_value_kind(&value), 1));
616            }
617            Ok((0, 1, false))
618        }
619        Err(err) => {
620            if config.error_policy == JsonErrorPolicy::Strict {
621                return Err(json_document_error(err));
622            }
623            Ok((0, 1, false))
624        }
625    }
626}
627
628/// One record read from a JSON or JSONL source.
629enum JsonRecord {
630    /// A JSON object — the only value with named fields to profile as a row.
631    Object(JsonObject),
632    /// Valid JSON that is not an object, carrying the value's kind for the
633    /// error message.
634    NonObject(&'static str),
635    /// The bytes are not valid JSON.
636    Malformed(serde_json::Error),
637}
638
639/// Read one JSON value and leave the reader positioned immediately after it.
640///
641/// `first` is the value's first byte, already located by
642/// [`consume_leading_whitespace`] but not yet consumed.
643fn read_json_record<R: BufRead>(
644    reader: &mut R,
645    first: u8,
646) -> Result<JsonRecord, DataProfilerError> {
647    // A number is the only JSON value that is not self-delimiting: serde finds
648    // its end by peeking one byte past it and then drops that byte along with
649    // the deserializer. Inside an array that byte is the `,` or `]` the scanner
650    // needs next, so numbers are read byte-wise instead.
651    if first == b'-' || first.is_ascii_digit() {
652        let token = read_number_token(reader)?;
653        return Ok(match serde_json::from_str::<Value>(&token) {
654            Ok(_) => JsonRecord::NonObject("number"),
655            Err(err) => JsonRecord::Malformed(err),
656        });
657    }
658
659    let mut deserializer = serde_json::Deserializer::from_reader(reader);
660    Ok(match Value::deserialize(&mut deserializer) {
661        Ok(Value::Object(obj)) => JsonRecord::Object(obj),
662        Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
663        Err(err) => JsonRecord::Malformed(err),
664    })
665}
666
667/// Read the bytes that can spell a JSON number, stopping at the first byte that
668/// cannot. The token is returned unvalidated — the caller decides whether it
669/// actually parses, so `1.2.3` stays a malformed record rather than a number.
670///
671/// Leading whitespace is consumed first: [`consume_leading_whitespace`] reports
672/// the next value's first byte without consuming what precedes it, because
673/// serde needs that whitespace to keep its line and column context.
674fn read_number_token<R: BufRead>(reader: &mut R) -> Result<String, DataProfilerError> {
675    skip_ascii_whitespace(reader)?;
676    let mut token = String::new();
677    loop {
678        let (consume, token_ended) = {
679            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
680            if buf.is_empty() {
681                break;
682            }
683            let consume = buf
684                .iter()
685                .take_while(|byte| is_json_number_byte(**byte))
686                .count();
687            // decode-audit: impossible — every byte accepted above is ASCII.
688            token.push_str(
689                std::str::from_utf8(&buf[..consume]).expect("JSON number bytes are ASCII"),
690            );
691            (consume, consume < buf.len())
692        };
693        reader.consume(consume);
694        if token_ended {
695            break;
696        }
697    }
698    Ok(token)
699}
700
701fn is_json_number_byte(byte: u8) -> bool {
702    byte.is_ascii_digit() || matches!(byte, b'-' | b'+' | b'.' | b'e' | b'E')
703}
704
705/// Consume the byte [`consume_leading_whitespace`] just reported, along with the
706/// whitespace it looked past.
707///
708/// That function only peeks — it leaves the whitespace queued so serde keeps its
709/// line and column context — so consuming one byte would eat a space rather than
710/// the delimiter. Getting this wrong made a pretty-printed JSON array report a
711/// phantom malformed record, because the `]` survived the consume and then
712/// looked like content after the closing bracket.
713fn consume_peeked<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
714    skip_ascii_whitespace(reader)?;
715    reader.consume(1);
716    Ok(())
717}
718
719/// Advance the reader past any ASCII whitespace at its current position.
720fn skip_ascii_whitespace<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
721    loop {
722        let (consume, done) = {
723            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
724            if buf.is_empty() {
725                return Ok(());
726            }
727            let consume = buf
728                .iter()
729                .take_while(|byte| byte.is_ascii_whitespace())
730                .count();
731            (consume, consume < buf.len())
732        };
733        reader.consume(consume);
734        if done {
735            return Ok(());
736        }
737    }
738}
739
740fn json_value_kind(value: &Value) -> &'static str {
741    match value {
742        Value::Null => "null",
743        Value::Bool(_) => "boolean",
744        Value::Number(_) => "number",
745        Value::String(_) => "string",
746        Value::Array(_) => "array",
747        Value::Object(_) => "object",
748    }
749}
750
751/// Build a strict-mode error for valid JSON that is not a row object. This is a
752/// distinct category from a syntax failure: the document parsed, it just does
753/// not hold records. `position` is 1-based over the records scanned so far.
754fn non_object_record_error(kind: &str, position: usize) -> DataProfilerError {
755    // No source: the document parsed cleanly, it simply does not hold records,
756    // so there is no decoder error underneath to retain.
757    DataProfilerError::json_parsing_error(&format!(
758        "non-object JSON record at position {position}: \
759         expected an object with fields to profile, found {kind}"
760    ))
761}
762
763/// Build a strict-mode parse error from a serde failure. The serde message
764/// carries line/column context but never the record contents, so it is safe to
765/// surface directly.
766fn malformed_record_error(err: serde_json::Error) -> DataProfilerError {
767    let message = format!("malformed JSON record: {err}");
768    DataProfilerError::json_parsing_with_source(message, err)
769}
770
771/// Build a strict-mode parse error for a JSONL record, located by its physical
772/// line.
773///
774/// Each line is parsed on its own, so the decoder's own line number is always 1
775/// and would be misleading; the column it reports is within the line and so is
776/// also the column in the file. The record's text is never included. Matches the
777/// phrasing the Python bytes reader uses for the same failure.
778fn malformed_jsonl_record_error(line: usize, err: serde_json::Error) -> DataProfilerError {
779    let message = format!(
780        "malformed JSON record on line {line}, column {}: a JSONL record must be one complete JSON value on one line",
781        err.column()
782    );
783    DataProfilerError::json_parsing_with_source(message, err)
784}
785
786fn malformed_array_error(message: &str) -> DataProfilerError {
787    // Callers here hold only a message, not the decoder error, so there is
788    // nothing to retain.
789    DataProfilerError::json_parsing_error(&format!("malformed JSON array: {message}"))
790}
791
792/// Build a strict-mode error for a standard JSON document that is not exactly
793/// one value.
794///
795/// The most common cause is JSONL read with the JSON grammar — several objects
796/// back to back parse as one value plus trailing characters — so the message
797/// names the option that would read it correctly.
798fn json_document_error(err: serde_json::Error) -> DataProfilerError {
799    let message = format!(
800        // One source line on purpose: rustfmt may join a `\`-continued
801        // literal and leave its indentation inside the string, which silently
802        // corrupted this very message once.
803        "malformed JSON document: {err}. A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
804    );
805    DataProfilerError::json_parsing_with_source(message, err)
806}
807
808fn drain_to_end<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
809    loop {
810        let available = reader.fill_buf().map_err(DataProfilerError::from)?.len();
811        if available == 0 {
812            return Ok(());
813        }
814        reader.consume(available);
815    }
816}
817
818fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
819    loop {
820        let mut bytes_to_consume = 0;
821        let first_non_whitespace = {
822            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
823            if buf.is_empty() {
824                return Ok(None);
825            }
826
827            let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
828            if first_non_whitespace.is_none() {
829                bytes_to_consume = buf.len();
830            }
831            first_non_whitespace
832        };
833
834        if first_non_whitespace.is_some() {
835            return Ok(first_non_whitespace);
836        }
837
838        reader.consume(bytes_to_consume);
839    }
840}
841
842/// Convert a JSON [`Value`] to a flat string for column storage.
843fn json_value_to_string(value: &Value) -> String {
844    match value {
845        Value::Null => String::new(),
846        Value::Bool(boolean) => boolean.to_string(),
847        Value::Number(number) => number.to_string(),
848        Value::String(string) => string.to_string(),
849        // decode-audit: impossible — serializing an in-memory Value back to a
850        // string cannot fail; a panic here beats folding "" (= null) into stats.
851        Value::Array(_) | Value::Object(_) => {
852            serde_json::to_string(value).expect("re-serializing a parsed JSON value cannot fail")
853        }
854    }
855}
856
857/// Feed a JSON object's fields into a [`StreamingColumnCollection`].
858///
859/// Columns are registered in the order the fields appear in each record, so the
860/// resulting column order is the first record's field order with later-only
861/// fields appended where they were first seen.
862///
863/// `container_counts` tallies, per column, the values that were an object or an
864/// array. It is what [`profile_builder::mark_container_columns`] later reads
865/// to decide which columns hold nothing but containers.
866fn feed_json_object(
867    obj: &JsonObject,
868    prior_rows: usize,
869    known_columns: &mut Vec<String>,
870    known_columns_set: &mut HashSet<String>,
871    column_stats: &mut StreamingColumnCollection,
872    container_counts: &mut HashMap<String, usize>,
873) {
874    for (key, value) in obj {
875        if known_columns_set.insert(key.clone()) {
876            known_columns.push(key.clone());
877            column_stats.init_column_with_missing(key, prior_rows);
878        }
879        if matches!(value, Value::Array(_) | Value::Object(_)) {
880            *container_counts.entry(key.clone()).or_default() += 1;
881        }
882    }
883
884    let values: Vec<String> = known_columns
885        .iter()
886        .map(|column| {
887            // decode-audit: no-data — a key absent from this object is a
888            // missing field, and "" is the profiler's textual null.
889            obj.get(column)
890                .map(json_value_to_string)
891                .unwrap_or_default()
892        })
893        .collect();
894
895    column_stats.process_record(known_columns, values);
896}
897
898/// Analyze JSON/JSONL data from a buffered reader using streaming statistics.
899///
900/// Returns `(column_profiles, streaming_stats, rows_read, malformed_lines, detected_format)`.
901///
902/// # Column order
903///
904/// Columns are returned in source order — the first record's field order, with
905/// fields that only appear in later records appended in first-seen order. This
906/// matches CSV (header order) and Parquet (schema order), so the same logical
907/// dataset profiles to the same column order in every format.
908///
909/// # Examples
910///
911/// The second record lists its fields in a different order and adds one. The
912/// first record still decides the order, and the new field is appended:
913///
914/// ```
915/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
916/// use std::io::Cursor;
917///
918/// use dataprof_json::{JsonParserConfig, analyze_json_from_reader};
919///
920/// let lines = "{\"id\": 1, \"city\": \"Rome\"}\n\
921///              {\"city\": \"Milan\", \"id\": 2, \"region\": \"Lombardy\"}\n";
922///
923/// let (profiles, _stats, rows_read, malformed_lines, format) =
924///     analyze_json_from_reader(Cursor::new(lines), &JsonParserConfig::jsonl())?;
925///
926/// let names: Vec<&str> = profiles.iter().map(|profile| profile.name.as_str()).collect();
927/// assert_eq!(names, ["id", "city", "region"]);
928/// assert_eq!(rows_read, 2);
929/// assert_eq!(malformed_lines, 0);
930/// assert_eq!(format, dataprof_core::FileFormat::Jsonl);
931///
932/// // The first record has no `region`, which counts as a missing value.
933/// let region = &profiles[2];
934/// assert_eq!(region.total_count, 2);
935/// assert_eq!(region.null_count, 1);
936/// # Ok(())
937/// # }
938/// ```
939pub fn analyze_json_from_reader<R: BufRead>(
940    reader: R,
941    config: &JsonParserConfig,
942) -> Result<
943    (
944        Vec<ColumnProfile>,
945        StreamingColumnCollection,
946        usize,
947        usize,
948        FileFormat,
949    ),
950    DataProfilerError,
951> {
952    analyze_json_from_reader_with_hints(reader, config, &SemanticHints::default())
953}
954
955pub fn analyze_json_from_reader_with_hints<R: BufRead>(
956    reader: R,
957    config: &JsonParserConfig,
958    semantic_hints: &SemanticHints,
959) -> Result<
960    (
961        Vec<ColumnProfile>,
962        StreamingColumnCollection,
963        usize,
964        usize,
965        FileFormat,
966    ),
967    DataProfilerError,
968> {
969    let options = AnalysisOptions::default().with_semantic_hints(semantic_hints.clone());
970    let (profiles, stats, rows_read, malformed_lines, format, _truncated) =
971        analyze_json_from_reader_full(reader, config, &options)?;
972    Ok((profiles, stats, rows_read, malformed_lines, format))
973}
974
975/// As [`analyze_json_from_reader_with_hints`], but also reports whether the scan
976/// stopped early. Kept private so the public tuple stays stable.
977#[allow(clippy::type_complexity)]
978fn analyze_json_from_reader_full<R: BufRead>(
979    reader: R,
980    config: &JsonParserConfig,
981    options: &AnalysisOptions,
982) -> Result<
983    (
984        Vec<ColumnProfile>,
985        StreamingColumnCollection,
986        usize,
987        usize,
988        FileFormat,
989        bool,
990    ),
991    DataProfilerError,
992> {
993    let semantic_hints = options.semantic_hints();
994    let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
995    let mut known_columns = Vec::new();
996    let mut known_columns_set = HashSet::new();
997    let mut container_counts = HashMap::new();
998    let mut rows_seen = 0;
999
1000    let summary = scan_json_from_reader(reader, config, |obj| {
1001        feed_json_object(
1002            obj,
1003            rows_seen,
1004            &mut known_columns,
1005            &mut known_columns_set,
1006            &mut column_stats,
1007            &mut container_counts,
1008        );
1009        rows_seen += 1;
1010    })?;
1011
1012    if let Some(indices) = options.column_indices(&column_stats.column_names())? {
1013        let available = column_stats.column_names();
1014        let selected = indices
1015            .into_iter()
1016            .map(|index| available[index].clone())
1017            .collect::<Vec<_>>();
1018        column_stats.retain_columns(&selected);
1019    }
1020
1021    let profiles = profile_builder::mark_container_columns(
1022        profile_builder::profiles_from_streaming_with_hints(
1023            &column_stats,
1024            !options.include_statistics(),
1025            !options.include_patterns(),
1026            options.locale(),
1027            semantic_hints,
1028        ),
1029        &container_counts,
1030    );
1031
1032    Ok((
1033        profiles,
1034        column_stats,
1035        summary.rows_read,
1036        summary.malformed_lines,
1037        summary.format,
1038        summary.truncated,
1039    ))
1040}
1041
1042/// Analyze a JSON or JSONL file, returning a full [`ProfileReport`].
1043///
1044/// With no explicit [`JsonParserConfig::format`] the file extension decides:
1045/// `.json` reads as a standard document, `.jsonl` and `.ndjson` as JSON Lines,
1046/// and any other name falls back to sniffing the first byte.
1047///
1048/// # Examples
1049///
1050/// ```
1051/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
1052/// use std::io::Write;
1053///
1054/// use dataprof_json::{JsonParserConfig, analyze_json_file};
1055///
1056/// let mut file = tempfile::NamedTempFile::with_suffix(".jsonl")?;
1057/// writeln!(file, "{{\"id\": 1, \"score\": 9.5}}")?;
1058/// writeln!(file, "{{\"id\": 2, \"score\": 7.0}}")?;
1059/// file.flush()?;
1060///
1061/// let report = analyze_json_file(file.path(), &JsonParserConfig::default())?;
1062///
1063/// assert_eq!(report.execution.rows_processed, 2);
1064/// assert_eq!(report.execution.columns_detected, 2);
1065/// assert!(report.execution.truncation_reason.is_none());
1066/// # Ok(())
1067/// # }
1068/// ```
1069pub fn analyze_json_file(
1070    file_path: &Path,
1071    config: &JsonParserConfig,
1072) -> Result<ProfileReport, DataProfilerError> {
1073    analyze_json_file_with_dimensions(file_path, config, None)
1074}
1075
1076/// Like [`analyze_json_file`] but only computes the requested quality dimensions.
1077pub fn analyze_json_file_with_dimensions(
1078    file_path: &Path,
1079    config: &JsonParserConfig,
1080    quality_dimensions: Option<&[QualityDimension]>,
1081) -> Result<ProfileReport, DataProfilerError> {
1082    analyze_json_file_with_dimensions_and_hints(
1083        file_path,
1084        config,
1085        quality_dimensions,
1086        &SemanticHints::default(),
1087    )
1088}
1089
1090pub fn analyze_json_file_with_dimensions_and_hints(
1091    file_path: &Path,
1092    config: &JsonParserConfig,
1093    quality_dimensions: Option<&[QualityDimension]>,
1094    semantic_hints: &SemanticHints,
1095) -> Result<ProfileReport, DataProfilerError> {
1096    let options = AnalysisOptions::default()
1097        .with_quality_dimensions(quality_dimensions.map(<[_]>::to_vec))
1098        .with_semantic_hints(semantic_hints.clone());
1099    analyze_json_file_with_options(file_path, config, &options)
1100}
1101
1102/// Analyze a JSON or JSONL file, honouring the caller's full analysis selection.
1103///
1104/// This is the entry point that carries metric packs and locale as well as
1105/// dimensions and hints, so a JSON profile reports exactly the analysis the
1106/// caller asked for — the same selection the CSV engines apply.
1107pub fn analyze_json_file_with_options(
1108    file_path: &Path,
1109    config: &JsonParserConfig,
1110    options: &AnalysisOptions,
1111) -> Result<ProfileReport, DataProfilerError> {
1112    // An explicit grammar wins; otherwise the file name gets a say before
1113    // content sniffing, which cannot tell a JSON object from a JSONL stream.
1114    let mut config = config.clone();
1115    if config.format.is_none() {
1116        config.format = grammar_for_extension(file_path);
1117    }
1118    let config = &config;
1119    let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
1120    let start = std::time::Instant::now();
1121
1122    let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
1123    let buf_reader = std::io::BufReader::new(file);
1124
1125    let (column_profiles, column_stats, rows_read, malformed_lines, format, truncated) =
1126        analyze_json_from_reader_full(buf_reader, config, options)?;
1127    let read_as_json_document = format == FileFormat::Json;
1128
1129    let file_source = DataSource::File {
1130        path: file_path.display().to_string(),
1131        format,
1132        size_bytes: metadata.len(),
1133        modified_at: None,
1134        parquet_metadata: None,
1135    };
1136
1137    if rows_read == 0 {
1138        if malformed_lines > 0 {
1139            // Nothing parsed under the JSON grammar most often means the source
1140            // is JSONL, so say so here too: under the default skip policy this
1141            // is the only error the caller sees.
1142            let hint = if read_as_json_document {
1143                ". A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
1144            } else {
1145                ""
1146            };
1147            // Aggregate over many discarded records: no single decoder error
1148            // is the cause, so none is retained.
1149            return Err(DataProfilerError::json_parsing_error(&format!(
1150                "No valid JSON records found in file \
1151                 (every record was malformed or not a JSON object){hint}"
1152            )));
1153        }
1154        // A row cap that stopped the scan before the first record still has to
1155        // be disclosed. Without this the report reads exactly like a complete
1156        // profile of an empty source, which is the one thing a truncated scan
1157        // must never be mistaken for.
1158        let mut execution =
1159            ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("json");
1160        if truncated && let Some(max) = config.max_rows {
1161            execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1162        }
1163        return Ok(ReportAssembler::new(file_source, execution)
1164            .columns(column_profiles)
1165            .with_quality_data(HashMap::new())
1166            .with_analysis_options(options)
1167            .build());
1168    }
1169
1170    let sample_columns = profile_builder::quality_check_samples(&column_stats);
1171    let scan_time_ms = start.elapsed().as_millis();
1172    let num_columns = column_profiles.len();
1173
1174    let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms)
1175        .with_engine("json")
1176        // Tolerant scans surface skipped malformed records here so callers can
1177        // tell a partial profile from a clean one.
1178        .with_error_count(malformed_lines);
1179    // `truncated` is set only when a record still remained at the cap, so a source
1180    // holding exactly `max_rows` records is reported as fully read, not cut short.
1181    if truncated && let Some(max) = config.max_rows {
1182        execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1183    }
1184
1185    Ok(ReportAssembler::new(file_source, execution)
1186        .columns(column_profiles)
1187        .with_quality_data(sample_columns)
1188        .with_row_duplicates(column_stats.row_duplicate_summary())
1189        .with_row_completeness(column_stats.row_completeness_summary())
1190        .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
1191        .with_analysis_options(options)
1192        .build())
1193}
1194
1195/// The grammar a file name advertises, if it advertises one.
1196///
1197/// Content sniffing resolves a leading `[` on its own, but not a leading `{`:
1198/// that is the first byte of both a JSON object document and a JSONL file. The
1199/// extension is the evidence that separates them, and a `.json` file holding one
1200/// pretty-printed object is an ordinary thing to profile. `None` means the name
1201/// says nothing and the content decides.
1202fn grammar_for_extension(file_path: &Path) -> Option<JsonFormat> {
1203    let extension = file_path.extension().and_then(|ext| ext.to_str())?;
1204    if extension.eq_ignore_ascii_case("json") {
1205        Some(JsonFormat::Json)
1206    } else if extension.eq_ignore_ascii_case("jsonl") || extension.eq_ignore_ascii_case("ndjson") {
1207        Some(JsonFormat::Jsonl)
1208    } else {
1209        None
1210    }
1211}
1212
1213fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
1214    if error.kind() == std::io::ErrorKind::NotFound {
1215        DataProfilerError::FileNotFound {
1216            path: file_path.display().to_string(),
1217        }
1218    } else {
1219        DataProfilerError::from(error)
1220    }
1221}
1222
1223#[cfg(test)]
1224mod tests {
1225    use super::*;
1226    use std::io::{Cursor, Write};
1227    use tempfile::NamedTempFile;
1228
1229    /// A temp file with no extension, so `analyze_json_file` falls back to the
1230    /// JSONL grammar the way an unnamed stream does.
1231    fn write_file(content: &str) -> NamedTempFile {
1232        let mut file = NamedTempFile::new().unwrap();
1233        write!(file, "{}", content).unwrap();
1234        file.flush().unwrap();
1235        file
1236    }
1237
1238    /// A temp file named `.json`, so `analyze_json_file` reads it as a standard
1239    /// JSON document — the grammar the extension advertises.
1240    fn write_json_file(content: &str) -> NamedTempFile {
1241        let mut file = NamedTempFile::with_suffix(".json").unwrap();
1242        write!(file, "{}", content).unwrap();
1243        file.flush().unwrap();
1244        file
1245    }
1246
1247    fn write_bytes(content: &[u8]) -> NamedTempFile {
1248        let mut file = NamedTempFile::new().unwrap();
1249        file.write_all(content).unwrap();
1250        file.flush().unwrap();
1251        file
1252    }
1253
1254    #[test]
1255    fn test_scan_jsonl_detects_format_and_rows() {
1256        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1257        let mut rows = 0;
1258
1259        let summary = scan_json_from_reader(
1260            Cursor::new(data.as_ref()),
1261            &JsonParserConfig::default(),
1262            |_| {
1263                rows += 1;
1264            },
1265        )
1266        .unwrap();
1267
1268        assert_eq!(summary.format, FileFormat::Jsonl);
1269        assert_eq!(summary.rows_read, 3);
1270        assert_eq!(summary.malformed_lines, 0);
1271        assert_eq!(rows, 3);
1272    }
1273
1274    #[test]
1275    fn test_scan_json_array_detects_format_and_rows() {
1276        let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
1277        let mut keys = Vec::new();
1278
1279        let summary = scan_json_from_reader(
1280            Cursor::new(data.as_ref()),
1281            &JsonParserConfig::default(),
1282            |obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
1283        )
1284        .unwrap();
1285
1286        assert_eq!(summary.format, FileFormat::Json);
1287        assert_eq!(summary.rows_read, 2);
1288        assert_eq!(keys.len(), 2);
1289        assert_eq!(keys[0], vec!["name".to_string()]);
1290    }
1291
1292    #[test]
1293    fn test_scan_accepts_one_leading_utf8_bom_for_json_and_jsonl() {
1294        for (data, config, expected_format) in [
1295            (
1296                b"\xEF\xBB\xBF[{\"x\":1},{\"x\":2}]".as_slice(),
1297                JsonParserConfig::default(),
1298                FileFormat::Json,
1299            ),
1300            (
1301                b"\xEF\xBB\xBF{\"x\":1}\n{\"x\":2}\n".as_slice(),
1302                JsonParserConfig::default(),
1303                FileFormat::Jsonl,
1304            ),
1305        ] {
1306            let mut rows = 0;
1307            let summary = scan_json_from_reader(Cursor::new(data), &config, |_| rows += 1)
1308                .expect("leading UTF-8 BOM should be accepted");
1309
1310            assert_eq!(rows, 2);
1311            assert_eq!(summary.rows_read, 2);
1312            assert_eq!(summary.malformed_lines, 0);
1313            assert_eq!(summary.format, expected_format);
1314        }
1315    }
1316
1317    #[test]
1318    fn test_scan_does_not_strip_nonleading_or_second_utf8_bom() {
1319        let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1320
1321        for data in [
1322            b" \xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1323            b"\xEF\xBB\xBF\xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1324            b"{\"x\":1}\n\xEF\xBB\xBF{\"x\":2}\n".as_slice(),
1325        ] {
1326            scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1327                .expect_err("only the BOM at absolute byte offset zero may be stripped");
1328        }
1329    }
1330
1331    #[test]
1332    fn test_bom_only_input_matches_empty_input_policy() {
1333        for config in [JsonParserConfig::default(), JsonParserConfig::jsonl()] {
1334            let empty = scan_json_from_reader(Cursor::new(b""), &config, |_| {}).unwrap();
1335            let bom =
1336                scan_json_from_reader(Cursor::new(b"\xEF\xBB\xBF".as_slice()), &config, |_| {})
1337                    .unwrap();
1338            assert_eq!(bom, empty);
1339        }
1340    }
1341
1342    #[test]
1343    fn test_json_array_container_grammar_obeys_error_policy() {
1344        let cases = [
1345            (br#"[{"x":1}"#.as_ref(), 1, "missing closing bracket"),
1346            (br#"[{"x":1},"#.as_ref(), 1, "comma followed by EOF"),
1347            (br#"[{"x":1} {"x":2}]"#.as_ref(), 1, "missing comma"),
1348            (br#"[{"x":1},,{"x":2}]"#.as_ref(), 1, "doubled comma"),
1349            (br#"[,{"x":1}]"#.as_ref(), 0, "leading comma"),
1350            (br#"[{"x":1},]"#.as_ref(), 1, "trailing comma"),
1351            (br#"[{"x":1}] trailing"#.as_ref(), 1, "trailing content"),
1352            (
1353                br#"[{"x":1}][{"x":2}]"#.as_ref(),
1354                1,
1355                "second top-level value",
1356            ),
1357        ];
1358
1359        for (data, expected_rows, case) in cases {
1360            let skip = JsonParserConfig::json_document();
1361            let summary =
1362                scan_json_from_reader(Cursor::new(data), &skip, |_| {}).expect("tolerant scan");
1363            assert_eq!(summary.rows_read, expected_rows, "{case}");
1364            assert_eq!(summary.malformed_lines, 1, "{case}");
1365
1366            let strict =
1367                JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict);
1368            let err = scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1369                .expect_err("strict mode must reject invalid array grammar");
1370            assert!(
1371                err.to_string()
1372                    .to_lowercase()
1373                    .contains("malformed json array"),
1374                "{case}: {err}"
1375            );
1376        }
1377    }
1378
1379    #[test]
1380    fn test_json_array_max_rows_does_not_validate_unread_values() {
1381        let data = br#"[{"x":1},not-valid-json"#;
1382        let config = JsonParserConfig::json_document().with_max_rows(1);
1383        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1384
1385        assert_eq!(summary.rows_read, 1);
1386        assert_eq!(summary.malformed_lines, 0);
1387        assert!(summary.truncated);
1388    }
1389
1390    #[test]
1391    fn test_scan_respects_max_rows() {
1392        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
1393        let config = JsonParserConfig::default().with_max_rows(2);
1394        let mut rows = 0;
1395
1396        let summary =
1397            scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
1398
1399        assert_eq!(summary.rows_read, 2);
1400        assert_eq!(rows, 2);
1401    }
1402
1403    #[test]
1404    fn test_scan_skips_malformed_jsonl_lines() {
1405        let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
1406        let mut rows = 0;
1407
1408        let summary = scan_json_from_reader(
1409            Cursor::new(data.as_ref()),
1410            &JsonParserConfig::jsonl(),
1411            |_| {
1412                rows += 1;
1413            },
1414        )
1415        .unwrap();
1416
1417        assert_eq!(summary.format, FileFormat::Jsonl);
1418        assert_eq!(summary.rows_read, 2);
1419        assert_eq!(summary.malformed_lines, 1);
1420        assert_eq!(rows, 2);
1421    }
1422
1423    /// A pretty-printed root object is one record under the JSON grammar and a
1424    /// malformed record under JSONL (#486). It cannot be both: whitespace is
1425    /// insignificant in a JSON document and is the record delimiter in JSONL, so
1426    /// reading a multi-line object as JSONL would mean each of its lines is a
1427    /// record — none of which is valid JSON on its own.
1428    #[test]
1429    fn test_pretty_printed_root_object_is_one_row_under_the_json_grammar() {
1430        let data = br#"{
1431    "type": "FeatureCollection",
1432    "features": [
1433        {"id": 1},
1434        {"id": 2}
1435  ]
1436}"#;
1437        let mut object_field_counts = Vec::new();
1438
1439        let summary = scan_json_from_reader(
1440            Cursor::new(data.as_ref()),
1441            &JsonParserConfig::json_document(),
1442            |obj| object_field_counts.push(obj.len()),
1443        )
1444        .unwrap();
1445
1446        assert_eq!(summary.format, FileFormat::Json);
1447        assert_eq!(summary.rows_read, 1);
1448        assert_eq!(summary.malformed_lines, 0);
1449        assert_eq!(object_field_counts, vec![2]);
1450    }
1451
1452    #[test]
1453    fn test_pretty_printed_root_object_is_not_valid_jsonl() {
1454        let data = br#"{
1455    "type": "FeatureCollection"
1456}"#;
1457        let err = scan_json_from_reader(
1458            Cursor::new(data.as_ref()),
1459            &JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict),
1460            |_| {},
1461        )
1462        .expect_err("a record spanning lines is not JSONL");
1463
1464        assert!(
1465            err.to_string().contains("malformed JSON record"),
1466            "{err}, expected the malformed category"
1467        );
1468    }
1469
1470    /// Every JSON value that is not an object, so none of them can be a row.
1471    /// Numbers are listed twice on purpose: they are the one value serde reads
1472    /// from a reader by peeking one byte past the end.
1473    const NON_OBJECT_VALUES: &[(&str, &str)] = &[
1474        ("null", "null"),
1475        ("boolean", "true"),
1476        ("number", "42"),
1477        ("number", "-1.5e3"),
1478        ("string", r#""text""#),
1479        ("array", "[1, 2]"),
1480    ];
1481
1482    #[test]
1483    fn test_jsonl_counts_non_object_records_and_keeps_scanning() {
1484        for (kind, value) in NON_OBJECT_VALUES {
1485            let data = format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n");
1486            let mut rows = 0;
1487
1488            let summary = scan_json_from_reader(
1489                Cursor::new(data.as_bytes()),
1490                &JsonParserConfig::jsonl(),
1491                |_| rows += 1,
1492            )
1493            .unwrap();
1494
1495            assert_eq!(summary.rows_read, 2, "{kind}: record after it was lost");
1496            assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1497            assert_eq!(rows, 2, "{kind}");
1498        }
1499    }
1500
1501    #[test]
1502    fn test_json_array_counts_non_object_elements_and_keeps_scanning() {
1503        for (kind, value) in NON_OBJECT_VALUES {
1504            let data = format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]");
1505            let mut rows = 0;
1506
1507            let summary = scan_json_from_reader(
1508                Cursor::new(data.as_bytes()),
1509                &JsonParserConfig::json_document(),
1510                |_| rows += 1,
1511            )
1512            .unwrap();
1513
1514            assert_eq!(summary.rows_read, 2, "{kind}: element after it was lost");
1515            assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1516            assert_eq!(rows, 2, "{kind}");
1517        }
1518    }
1519
1520    #[test]
1521    fn test_strict_policy_rejects_the_first_non_object_record() {
1522        for (kind, value) in NON_OBJECT_VALUES {
1523            for (config, data) in [
1524                (
1525                    JsonParserConfig::jsonl(),
1526                    format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n"),
1527                ),
1528                (
1529                    JsonParserConfig::json_document(),
1530                    format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]"),
1531                ),
1532            ] {
1533                let config = config.with_error_policy(JsonErrorPolicy::Strict);
1534                let err = scan_json_from_reader(Cursor::new(data.as_bytes()), &config, |_| {})
1535                    .expect_err("{kind}: strict mode must reject a non-object record");
1536
1537                let message = err.to_string();
1538                assert!(message.contains("non-object JSON record"), "{message}");
1539                assert!(message.contains("at position 2"), "{message}");
1540                assert!(message.contains(&format!("found {kind}")), "{message}");
1541            }
1542        }
1543    }
1544
1545    #[test]
1546    fn test_number_element_does_not_swallow_the_array_delimiter() {
1547        // serde ends a number by peeking one byte past it and drops that byte
1548        // with the deserializer; here it is the `,` the array scanner needs.
1549        let data = br#"[{"id":1}, 1, 2, 3, {"id":2}, 4, {"id":3}]"#;
1550        let mut rows = 0;
1551
1552        let summary = scan_json_from_reader(
1553            Cursor::new(data.as_ref()),
1554            &JsonParserConfig::json_document(),
1555            |_| rows += 1,
1556        )
1557        .unwrap();
1558
1559        assert_eq!(summary.rows_read, 3);
1560        assert_eq!(summary.malformed_lines, 4);
1561        assert_eq!(rows, 3);
1562    }
1563
1564    #[test]
1565    fn test_number_shaped_garbage_stays_a_malformed_record() {
1566        // `1.2.3` is read by the byte-wise number scanner but is not a number,
1567        // so it must not be reported as a valid non-object record.
1568        let data = b"{\"id\":1}\n1.2.3\n";
1569        let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1570        let err = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {})
1571            .expect_err("number-shaped garbage should be malformed");
1572
1573        assert!(
1574            err.to_string().contains("malformed JSON record"),
1575            "{err}, expected the malformed category"
1576        );
1577    }
1578
1579    #[test]
1580    fn test_input_of_only_non_object_records_fails() {
1581        let file = write_file("\"just a string\"\n1\n[2]\n");
1582        let err = analyze_json_file(file.path(), &JsonParserConfig::jsonl())
1583            .expect_err("nothing profileable should not return a clean empty profile");
1584
1585        assert!(
1586            err.to_string().contains("No valid JSON records found"),
1587            "{err}"
1588        );
1589    }
1590
1591    #[test]
1592    /// The JSON grammar accepts both shapes that carry records, so a bare object
1593    /// is one row rather than an error (#486).
1594    fn test_json_grammar_reads_a_bare_object_as_one_row() {
1595        let data = br#"{"x":1}"#;
1596        let mut rows = 0;
1597        let summary = scan_json_from_reader(
1598            Cursor::new(data.as_ref()),
1599            &JsonParserConfig::json_document(),
1600            |_| rows += 1,
1601        )
1602        .unwrap();
1603
1604        assert_eq!(summary.rows_read, 1);
1605        assert_eq!(summary.malformed_lines, 0);
1606        assert_eq!(rows, 1);
1607    }
1608
1609    #[test]
1610    /// Several objects back to back are not a JSON document. The error names
1611    /// the option that reads them, because this is what a JSONL file looks like
1612    /// when it is read with the wrong grammar.
1613    fn test_json_grammar_rejects_concatenated_objects() {
1614        let data = br#"{"x":1}{"x":2}"#;
1615        let err = scan_json_from_reader(
1616            Cursor::new(data.as_ref()),
1617            &JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict),
1618            |_| {},
1619        )
1620        .expect_err("two objects are not one JSON document");
1621
1622        let message = err.to_string();
1623        assert!(message.contains("malformed JSON document"), "{message}");
1624        assert!(message.contains("format=\"jsonl\""), "{message}");
1625    }
1626
1627    #[test]
1628    fn test_analyze_json_from_reader_jsonl_streaming() {
1629        let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
1630        let cursor = Cursor::new(data.as_ref());
1631        let config = JsonParserConfig::default();
1632
1633        let (profiles, _stats, rows, _malformed, format) =
1634            analyze_json_from_reader(cursor, &config).unwrap();
1635        assert_eq!(format, FileFormat::Jsonl);
1636        assert_eq!(rows, 3);
1637        assert_eq!(profiles.len(), 2);
1638    }
1639
1640    #[test]
1641    fn test_analyze_json_from_reader_json_array() {
1642        let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
1643        let cursor = Cursor::new(data.as_ref());
1644        let config = JsonParserConfig::default();
1645
1646        let (profiles, _stats, rows, _malformed, format) =
1647            analyze_json_from_reader(cursor, &config).unwrap();
1648        assert_eq!(format, FileFormat::Json);
1649        assert_eq!(rows, 2);
1650        assert_eq!(profiles.len(), 2);
1651    }
1652
1653    #[test]
1654    fn test_analyze_json_from_reader_max_rows() {
1655        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
1656        let cursor = Cursor::new(data.as_ref());
1657        let config = JsonParserConfig::default().with_max_rows(3);
1658
1659        let (_profiles, _stats, rows, _malformed, _format) =
1660            analyze_json_from_reader(cursor, &config).unwrap();
1661        assert_eq!(rows, 3);
1662    }
1663
1664    #[test]
1665    fn test_analyze_json_from_reader_missing_fields() {
1666        let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
1667        let cursor = Cursor::new(data.as_ref());
1668        let config = JsonParserConfig::jsonl();
1669
1670        let (profiles, _stats, rows, _malformed, _format) =
1671            analyze_json_from_reader(cursor, &config).unwrap();
1672        assert_eq!(rows, 2);
1673
1674        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1675        assert_eq!(col_b.total_count, 2);
1676    }
1677
1678    #[test]
1679    fn test_identical_records_count_as_duplicates_across_a_late_key() {
1680        // Records 1 and 3 are byte-identical; `b` is only discovered between
1681        // them. Signing a row against whatever columns happened to be known at
1682        // the time would make the two look different and report no duplicate.
1683        let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"x\"}\n";
1684        let cursor = Cursor::new(data.as_ref());
1685        let config = JsonParserConfig::jsonl();
1686
1687        let (_profiles, stats, rows, _malformed, _format) =
1688            analyze_json_from_reader(cursor, &config).unwrap();
1689        assert_eq!(rows, 3);
1690
1691        let summary = stats
1692            .row_duplicate_summary()
1693            .expect("three rows were observed");
1694        assert_eq!(summary.rows_checked, 3);
1695        assert_eq!(summary.duplicate_rows, 1);
1696        assert!(!summary.approximate);
1697    }
1698
1699    #[test]
1700    fn test_a_late_key_does_not_invent_duplicates() {
1701        // The counterpart guard: collapsing the trailing absent field must not
1702        // merge rows that differ in the columns they do share.
1703        let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"z\"}\n";
1704        let cursor = Cursor::new(data.as_ref());
1705        let config = JsonParserConfig::jsonl();
1706
1707        let (_profiles, stats, _rows, _malformed, _format) =
1708            analyze_json_from_reader(cursor, &config).unwrap();
1709
1710        let summary = stats
1711            .row_duplicate_summary()
1712            .expect("three rows were observed");
1713        assert_eq!(summary.duplicate_rows, 0);
1714    }
1715
1716    #[test]
1717    fn test_analyze_json_file_quality_report() {
1718        let file = write_file(r#"[{"x":1},{"x":2}]"#);
1719        let config = JsonParserConfig::default();
1720        let report = analyze_json_file(file.path(), &config).unwrap();
1721
1722        assert_eq!(report.execution.rows_processed, 2);
1723        assert_eq!(report.column_profiles.len(), 1);
1724        assert!(report.quality_score().unwrap() >= 0.0);
1725    }
1726
1727    #[test]
1728    fn test_jsonl_skips_malformed_lines() {
1729        let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
1730        let cursor = Cursor::new(data.as_ref());
1731        let config = JsonParserConfig::jsonl();
1732
1733        let (profiles, _stats, rows, _malformed, format) =
1734            analyze_json_from_reader(cursor, &config).unwrap();
1735        assert_eq!(format, FileFormat::Jsonl);
1736        assert_eq!(rows, 2);
1737        assert_eq!(profiles[0].total_count, 2);
1738    }
1739
1740    #[test]
1741    fn test_strict_policy_errors_on_malformed_jsonl() {
1742        // Malformed in first / middle / last position all abort under Strict.
1743        for data in [
1744            b"not-json\n{\"x\":1}\n{\"x\":2}\n".as_ref(),
1745            b"{\"x\":1}\nnot-json\n{\"x\":2}\n".as_ref(),
1746            b"{\"x\":1}\n{\"x\":2}\nnot-json\n".as_ref(),
1747        ] {
1748            let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1749            let err = scan_json_from_reader(Cursor::new(data), &config, |_| {})
1750                .expect_err("strict mode must reject malformed records");
1751            let message = err.to_string().to_lowercase();
1752            assert!(message.contains("malformed json record"), "got: {message}");
1753            // The offending record text must never be echoed back.
1754            assert!(
1755                !err.to_string().contains("not-json"),
1756                "leaked record: {err}"
1757            );
1758        }
1759    }
1760
1761    #[test]
1762    fn test_tolerant_policy_counts_skipped_records() {
1763        let data = b"{\"x\":1}\nnot-json\n{\"x\":2}\n";
1764        let config = JsonParserConfig::jsonl(); // default Skip
1765        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1766        assert_eq!(summary.rows_read, 2);
1767        assert_eq!(summary.malformed_lines, 1);
1768    }
1769
1770    #[test]
1771    fn test_blank_lines_are_not_counted_as_malformed() {
1772        let data = b"{\"x\":1}\n\n   \n{\"x\":2}\n";
1773        for policy in [JsonErrorPolicy::Skip, JsonErrorPolicy::Strict] {
1774            let config = JsonParserConfig::jsonl().with_error_policy(policy);
1775            let summary =
1776                scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1777            assert_eq!(summary.rows_read, 2, "policy {policy:?}");
1778            assert_eq!(summary.malformed_lines, 0, "policy {policy:?}");
1779        }
1780    }
1781
1782    #[test]
1783    fn test_truncated_final_record_obeys_error_policy() {
1784        let data = b"{\"x\":1}\n{\"x\":2";
1785
1786        let skip = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Skip);
1787        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &skip, |_| {}).unwrap();
1788        assert_eq!(summary.rows_read, 1);
1789        assert_eq!(summary.malformed_lines, 1);
1790
1791        let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1792        let err = scan_json_from_reader(Cursor::new(data.as_ref()), &strict, |_| {})
1793            .expect_err("strict mode must reject an incomplete trailing record");
1794        assert!(
1795            err.to_string()
1796                .to_lowercase()
1797                .contains("malformed json record")
1798        );
1799    }
1800
1801    #[test]
1802    fn test_all_malformed_file_fails() {
1803        let file = write_file("not-json\nalso-bad\n");
1804        let config = JsonParserConfig::jsonl();
1805        let err = analyze_json_file(file.path(), &config)
1806            .expect_err("a file with no valid records must fail");
1807        let message = err.to_string().to_lowercase();
1808        assert!(message.contains("no valid json records"), "got: {message}");
1809    }
1810
1811    #[test]
1812    fn test_tolerant_file_reports_error_count() {
1813        let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1814        let config = JsonParserConfig::jsonl();
1815        let report = analyze_json_file(file.path(), &config).unwrap();
1816        assert_eq!(report.execution.rows_processed, 2);
1817        assert_eq!(report.execution.error_count, 1);
1818    }
1819
1820    #[test]
1821    fn test_strict_file_errors_on_malformed() {
1822        let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1823        let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1824        let err = analyze_json_file(file.path(), &config)
1825            .expect_err("strict mode must reject the malformed record");
1826        assert!(
1827            err.to_string()
1828                .to_lowercase()
1829                .contains("malformed json record")
1830        );
1831    }
1832
1833    #[test]
1834    fn test_analyze_json_with_large_leading_whitespace() {
1835        let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
1836        let cursor = Cursor::new(data.into_bytes());
1837        let config = JsonParserConfig::default();
1838
1839        let (_profiles, _stats, rows, malformed, format) =
1840            analyze_json_from_reader(cursor, &config).unwrap();
1841        assert_eq!(format, FileFormat::Json);
1842        assert_eq!(rows, 1);
1843        assert_eq!(malformed, 0);
1844    }
1845
1846    #[test]
1847    fn test_analyze_json_array() {
1848        let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
1849        let config = JsonParserConfig::default();
1850        let report = analyze_json_file(json.path(), &config).unwrap();
1851        let profiles = &report.column_profiles;
1852
1853        assert_eq!(profiles.len(), 2);
1854        let names: Vec<&str> = profiles
1855            .iter()
1856            .map(|profile| profile.name.as_str())
1857            .collect();
1858        assert!(names.contains(&"name"));
1859        assert!(names.contains(&"age"));
1860
1861        let age = profiles
1862            .iter()
1863            .find(|profile| profile.name == "age")
1864            .unwrap();
1865        assert_eq!(age.total_count, 2);
1866        assert_eq!(age.null_count, 0);
1867    }
1868
1869    #[test]
1870    fn test_analyze_jsonl() {
1871        let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
1872        let config = JsonParserConfig::default();
1873        let report = analyze_json_file(jsonl.path(), &config).unwrap();
1874        let profiles = &report.column_profiles;
1875
1876        assert_eq!(profiles.len(), 1);
1877        assert_eq!(profiles[0].name, "x");
1878        assert_eq!(profiles[0].total_count, 3);
1879    }
1880
1881    #[test]
1882    fn test_analyze_json_with_nulls() {
1883        let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
1884        let config = JsonParserConfig::default();
1885        let report = analyze_json_file(json.path(), &config).unwrap();
1886        let profiles = &report.column_profiles;
1887
1888        let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
1889        assert_eq!(col_a.null_count, 1);
1890
1891        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1892        assert_eq!(col_b.null_count, 1);
1893    }
1894
1895    #[test]
1896    fn test_analyze_json_with_missing_fields() {
1897        let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
1898        let config = JsonParserConfig::default();
1899        let report = analyze_json_file(json.path(), &config).unwrap();
1900        let profiles = &report.column_profiles;
1901
1902        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1903        assert_eq!(col_b.total_count, 2);
1904    }
1905
1906    /// Deliberately non-alphabetical field order: sorting would give
1907    /// `active, amount, date, id`, so any re-sort is visible.
1908    const UNSORTED_FIELDS: &str = r#"{"id":1,"amount":12.5,"active":true,"date":"2026-07-23"}"#;
1909    const UNSORTED_ORDER: [&str; 4] = ["id", "amount", "active", "date"];
1910
1911    fn profile_names(report: &ProfileReport) -> Vec<&str> {
1912        report
1913            .column_profiles
1914            .iter()
1915            .map(|profile| profile.name.as_str())
1916            .collect()
1917    }
1918
1919    #[test]
1920    fn test_json_column_order_follows_source_not_alphabet() {
1921        let array = write_file(&format!("[{UNSORTED_FIELDS},{UNSORTED_FIELDS}]"));
1922        let jsonl = write_file(&format!("{UNSORTED_FIELDS}\n{UNSORTED_FIELDS}\n"));
1923
1924        for file in [&array, &jsonl] {
1925            let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1926            assert_eq!(profile_names(&report), UNSORTED_ORDER);
1927        }
1928    }
1929
1930    #[test]
1931    fn test_json_column_order_appends_later_keys_in_first_seen_order() {
1932        // Each record introduces two keys in reverse-alphabetical order, so
1933        // sorting would reshuffle both the leading pair and the appended pair.
1934        let data = b"{\"zulu\":1,\"mike\":2}\n{\"zulu\":3,\"mike\":4,\"delta\":5,\"alpha\":6}\n";
1935        let file = write_bytes(data);
1936        let report = analyze_json_file(file.path(), &JsonParserConfig::jsonl()).unwrap();
1937
1938        assert_eq!(profile_names(&report), ["zulu", "mike", "delta", "alpha"]);
1939    }
1940
1941    #[test]
1942    fn test_json_column_order_survives_report_serialization() {
1943        let file = write_file(&format!("[{UNSORTED_FIELDS}]"));
1944        let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1945
1946        let json = serde_json::to_string(&report).unwrap();
1947        let restored: Value = serde_json::from_str(&json).unwrap();
1948        let names: Vec<&str> = restored["column_profiles"]
1949            .as_array()
1950            .unwrap()
1951            .iter()
1952            .map(|profile| profile["name"].as_str().unwrap())
1953            .collect();
1954
1955        assert_eq!(names, UNSORTED_ORDER);
1956    }
1957
1958    #[test]
1959    fn test_analyze_json_backfills_columns_discovered_after_first_row() {
1960        let json = write_file(r#"[{"a":1},{"a":2,"b":3}]"#);
1961        let config = JsonParserConfig::default();
1962        let report = analyze_json_file(json.path(), &config).unwrap();
1963        let col_b = report
1964            .column_profiles
1965            .iter()
1966            .find(|profile| profile.name == "b")
1967            .unwrap();
1968
1969        assert_eq!(col_b.total_count, 2);
1970        assert_eq!(col_b.null_count, 1);
1971        assert_eq!(col_b.unique_count, Some(1));
1972    }
1973
1974    #[test]
1975    fn test_analyze_json_empty_array() {
1976        let json = write_file("[]");
1977        let config = JsonParserConfig::default();
1978        let report = analyze_json_file(json.path(), &config).unwrap();
1979        assert!(report.column_profiles.is_empty());
1980        // An empty dataset has nothing to assess: no score is fabricated.
1981        assert_eq!(report.quality_score(), None);
1982    }
1983
1984    #[test]
1985    fn test_analyze_json_malformed_returns_error() {
1986        let json = write_file("this is entirely invalid json");
1987        let config = JsonParserConfig::default();
1988        let err = analyze_json_file(json.path(), &config)
1989            .expect_err("malformed JSON should return an error");
1990
1991        let message = err.to_string().to_lowercase();
1992        assert!(message.contains("malformed") && message.contains("json"));
1993    }
1994
1995    #[test]
1996    fn test_analyze_json_file_detects_format() {
1997        let json_array = write_file(r#"[{"x":1}]"#);
1998        let config = JsonParserConfig::default();
1999        let report = analyze_json_file(json_array.path(), &config).unwrap();
2000        assert!(matches!(
2001            report.data_source,
2002            DataSource::File {
2003                format: FileFormat::Json,
2004                ..
2005            }
2006        ));
2007
2008        let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
2009        let report = analyze_json_file(jsonl.path(), &config).unwrap();
2010        assert!(matches!(
2011            report.data_source,
2012            DataSource::File {
2013                format: FileFormat::Jsonl,
2014                ..
2015            }
2016        ));
2017    }
2018
2019    #[test]
2020    fn test_bom_prefixed_file_preserves_source_size() {
2021        let data = b"\xEF\xBB\xBF[{\"x\":1}]";
2022        let json = write_bytes(data);
2023        let report = analyze_json_file(json.path(), &JsonParserConfig::json_document()).unwrap();
2024
2025        assert_eq!(report.execution.rows_processed, 1);
2026        assert!(matches!(
2027            report.data_source,
2028            DataSource::File {
2029                format: FileFormat::Json,
2030                size_bytes,
2031                ..
2032            } if size_bytes == data.len() as u64
2033        ));
2034    }
2035
2036    #[test]
2037    fn test_analyze_json_file_empty() {
2038        let json = write_file("");
2039        let config = JsonParserConfig::default();
2040        let report = analyze_json_file(json.path(), &config).unwrap();
2041        assert_eq!(report.execution.rows_processed, 0);
2042        assert!(report.column_profiles.is_empty());
2043    }
2044
2045    #[test]
2046    fn test_analyze_json_boolean_and_nested() {
2047        let json =
2048            write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
2049        let config = JsonParserConfig::default();
2050        let report = analyze_json_file(json.path(), &config).unwrap();
2051        let profiles = &report.column_profiles;
2052
2053        let flag = profiles
2054            .iter()
2055            .find(|profile| profile.name == "flag")
2056            .unwrap();
2057        assert_eq!(flag.total_count, 2);
2058
2059        let nested = profiles
2060            .iter()
2061            .find(|profile| profile.name == "nested")
2062            .unwrap();
2063        assert_eq!(nested.total_count, 2);
2064    }
2065
2066    #[test]
2067    fn test_single_root_object_compact_yields_one_row() {
2068        let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
2069        let cursor = Cursor::new(data.as_ref());
2070        let config = JsonParserConfig::default();
2071
2072        let (profiles, _stats, rows, malformed, _format) =
2073            analyze_json_from_reader(cursor, &config).unwrap();
2074        assert_eq!(rows, 1);
2075        assert_eq!(malformed, 0);
2076        assert_eq!(profiles.len(), 2);
2077
2078        let names: Vec<&str> = profiles
2079            .iter()
2080            .map(|profile| profile.name.as_str())
2081            .collect();
2082        assert!(names.contains(&"type"));
2083        assert!(names.contains(&"features"));
2084    }
2085
2086    #[test]
2087    fn test_jsonl_multi_object_still_works() {
2088        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
2089        let cursor = Cursor::new(data.as_ref());
2090        let config = JsonParserConfig::default();
2091
2092        let (_profiles, _stats, rows, _malformed, format) =
2093            analyze_json_from_reader(cursor, &config).unwrap();
2094        assert_eq!(format, FileFormat::Jsonl);
2095        assert_eq!(rows, 3);
2096    }
2097
2098    #[test]
2099    fn test_single_root_object_via_analyze_json_file() {
2100        let file =
2101            write_json_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
2102        let config = JsonParserConfig::default();
2103        let report = analyze_json_file(file.path(), &config).unwrap();
2104
2105        assert_eq!(report.execution.rows_processed, 1);
2106        assert_eq!(report.column_profiles.len(), 2);
2107    }
2108
2109    #[test]
2110    fn test_single_root_object_pretty_printed_yields_one_row() {
2111        let data = br#"{
2112  "type": "FeatureCollection",
2113  "features": [
2114    {"id": 1},
2115    {"id": 2}
2116  ]
2117}"#;
2118        let cursor = Cursor::new(data.as_ref());
2119        // A reader has no file name, so the grammar cannot be inferred and is
2120        // named explicitly; `analyze_json_file` takes it from the extension.
2121        let config = JsonParserConfig::json_document();
2122
2123        let (profiles, _stats, rows, malformed, format) =
2124            analyze_json_from_reader(cursor, &config).unwrap();
2125        assert_eq!(format, FileFormat::Json);
2126        assert_eq!(rows, 1);
2127        assert_eq!(malformed, 0);
2128        assert_eq!(profiles.len(), 2);
2129    }
2130
2131    #[test]
2132    fn test_single_root_object_pretty_printed_via_analyze_json_file() {
2133        let file = write_json_file(
2134            "{\n  \"type\": \"FeatureCollection\",\n  \"features\": [\n    {\"id\": 1},\n    {\"id\": 2}\n  ]\n}\n",
2135        );
2136        let config = JsonParserConfig::default();
2137        let report = analyze_json_file(file.path(), &config).unwrap();
2138
2139        assert_eq!(report.execution.rows_processed, 1);
2140        assert_eq!(report.column_profiles.len(), 2);
2141    }
2142}