Skip to main content

dataprof_json/
lib.rs

1//! JSON and JSONL scanning and profiling for `dataprof`.
2//!
3//! This crate is an implementation detail of the `dataprof` facade, which
4//! re-exports [`JsonParserConfig`], [`analyze_json_from_reader`], and
5//! [`analyze_json_file`]. Depend on `dataprof` unless you need JSON support
6//! without the rest of the workspace.
7//!
8//! Two entry shapes are offered. [`scan_json_from_reader`] hands each record to
9//! a callback and reports what it read; [`analyze_json_from_reader`] and
10//! [`analyze_json_file`] build column profiles and a full report on top of it.
11//!
12//! ```
13//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
14//! use std::io::Cursor;
15//!
16//! use dataprof_json::{JsonParserConfig, analyze_json_from_reader};
17//!
18//! let jsonl = "{\"id\": 1, \"city\": \"Rome\"}\n{\"id\": 2, \"city\": \"Milan\"}\n";
19//! let (profiles, _stats, rows_read, malformed_lines, _format) =
20//!     analyze_json_from_reader(Cursor::new(jsonl), &JsonParserConfig::jsonl())?;
21//!
22//! assert_eq!(rows_read, 2);
23//! assert_eq!(malformed_lines, 0);
24//! assert_eq!(profiles[0].name, "id");
25//! # Ok(())
26//! # }
27//! ```
28
29use std::collections::{HashMap, HashSet};
30use std::io::{BufRead, Read};
31use std::path::Path;
32
33pub use dataprof_core::JsonErrorPolicy;
34
35#[doc(hidden)]
36pub mod record;
37use dataprof_core::{
38    AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat,
39    TruncationReason, Utf8BomReader,
40};
41use dataprof_runtime::{
42    ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
43};
44use record::{
45    JsonRecord, drain_to_end, json_document_error, json_value_kind, json_value_text,
46    malformed_array_error, non_object_record_error, object_record, read_json_record,
47    read_jsonl_line, rest_is_white_space, skip_ascii_whitespace, skip_leading_white_space,
48    wide_integer_record_error,
49};
50use serde_json::Value;
51
52/// Which grammar a JSON source is read with.
53///
54/// The two are deliberately distinct: whitespace is insignificant inside a
55/// standard JSON document and load-bearing in JSONL, so one input cannot be
56/// valid under both readings and mean the same thing. Callers pick with
57/// `format="json"` / `format="jsonl"`, or by file extension.
58#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub enum JsonFormat {
60    /// A standard JSON document, in either of the two shapes that carry
61    /// records: an array of objects (`[{...}, {...}]`), or a single object
62    /// (`{...}`) as one record. Values may span lines and be pretty-printed,
63    /// and exactly one document must fill the whole input.
64    Json,
65    /// JSON Lines — one record per physical line. A record may not span lines,
66    /// and a line may not hold more than one value.
67    Jsonl,
68}
69
70/// Configuration for JSON/JSONL parsing and scanning.
71///
72/// The default detects the grammar from the input and reads every record,
73/// skipping malformed ones. The two format constructors say which grammar to
74/// read with, which is what a caller who knows should do: detection cannot tell
75/// a JSON object document from a JSONL file, since both open with `{`.
76///
77/// # Examples
78///
79/// The same two records under each grammar:
80///
81/// ```
82/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
83/// use std::io::Cursor;
84///
85/// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
86///
87/// let document = r#"[{"id": 1}, {"id": 2}]"#;
88/// let lines = "{\"id\": 1}\n{\"id\": 2}\n";
89///
90/// let from_document = scan_json_from_reader(
91///     Cursor::new(document),
92///     &JsonParserConfig::json_document(),
93///     |_| {},
94/// )?;
95/// let from_lines =
96///     scan_json_from_reader(Cursor::new(lines), &JsonParserConfig::jsonl(), |_| {})?;
97///
98/// assert_eq!(from_document.rows_read, 2);
99/// assert_eq!(from_lines.rows_read, 2);
100/// # Ok(())
101/// # }
102/// ```
103#[derive(Debug, Clone, Default)]
104pub struct JsonParserConfig {
105    /// The grammar to read with. `None` detects it from the first byte, which
106    /// resolves `[` to [`JsonFormat::Json`] and everything else to
107    /// [`JsonFormat::Jsonl`] — a leading `{` opens both a JSON object document
108    /// and a JSONL file, so callers that know which they have (from the file
109    /// extension or an explicit `format=`) should say so rather than rely on
110    /// detection.
111    pub format: Option<JsonFormat>,
112    /// Maximum rows to process (None = all rows).
113    pub max_rows: Option<usize>,
114    /// How to react to a malformed record (default: skip).
115    pub error_policy: JsonErrorPolicy,
116}
117
118impl JsonParserConfig {
119    /// Set the maximum number of rows to process.
120    ///
121    /// The cap counts profileable records, not lines: blank lines and records
122    /// skipped as malformed do not consume it.
123    ///
124    /// A cap is only *truncation* when a record still remained once it was
125    /// reached, so a source holding exactly `max_rows` records reads as
126    /// complete. [`JsonScanSummary::truncated`] carries that distinction, and
127    /// [`analyze_json_file`] turns it into a [`TruncationReason::MaxRows`] on
128    /// the report.
129    ///
130    /// # Examples
131    ///
132    /// ```
133    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
134    /// use std::io::Cursor;
135    ///
136    /// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
137    ///
138    /// let lines = "{\"id\": 1}\n{\"id\": 2}\n{\"id\": 3}\n";
139    ///
140    /// // Two of three records: a third remained, so the scan was cut short.
141    /// let capped = JsonParserConfig::jsonl().with_max_rows(2);
142    /// let summary = scan_json_from_reader(Cursor::new(lines), &capped, |_| {})?;
143    /// assert_eq!(summary.rows_read, 2);
144    /// assert!(summary.truncated);
145    ///
146    /// // A cap the source never reaches past is a complete read, not a
147    /// // truncated one.
148    /// let exact = JsonParserConfig::jsonl().with_max_rows(3);
149    /// let summary = scan_json_from_reader(Cursor::new(lines), &exact, |_| {})?;
150    /// assert_eq!(summary.rows_read, 3);
151    /// assert!(!summary.truncated);
152    /// # Ok(())
153    /// # }
154    /// ```
155    pub fn with_max_rows(mut self, max_rows: usize) -> Self {
156        self.max_rows = Some(max_rows);
157        self
158    }
159
160    /// Set how malformed records are handled.
161    ///
162    /// Under the default [`JsonErrorPolicy::Skip`] a malformed record is
163    /// counted in [`JsonScanSummary::malformed_lines`] and scanning continues,
164    /// so a partial profile never looks like a clean one. Under
165    /// [`JsonErrorPolicy::Strict`] the first one fails the scan.
166    ///
167    /// # Examples
168    ///
169    /// ```
170    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
171    /// use std::io::Cursor;
172    ///
173    /// use dataprof_json::{JsonErrorPolicy, JsonParserConfig, scan_json_from_reader};
174    ///
175    /// let lines = "{\"id\": 1}\nnot json\n{\"id\": 2}\n";
176    ///
177    /// let skipping = JsonParserConfig::jsonl();
178    /// let summary = scan_json_from_reader(Cursor::new(lines), &skipping, |_| {})?;
179    /// assert_eq!(summary.rows_read, 2);
180    /// assert_eq!(summary.malformed_lines, 1);
181    ///
182    /// let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
183    /// assert!(scan_json_from_reader(Cursor::new(lines), &strict, |_| {}).is_err());
184    /// # Ok(())
185    /// # }
186    /// ```
187    pub fn with_error_policy(mut self, policy: JsonErrorPolicy) -> Self {
188        self.error_policy = policy;
189        self
190    }
191
192    /// Read as JSON Lines: one record per physical line.
193    pub fn jsonl() -> Self {
194        Self {
195            format: Some(JsonFormat::Jsonl),
196            ..Default::default()
197        }
198    }
199
200    /// Read as a standard JSON document: an array of records, or a single
201    /// object as one record.
202    pub fn json_document() -> Self {
203        Self {
204            format: Some(JsonFormat::Json),
205            ..Default::default()
206        }
207    }
208
209    /// Set the grammar explicitly.
210    pub fn with_format(mut self, format: JsonFormat) -> Self {
211        self.format = Some(format);
212        self
213    }
214}
215
216/// Borrowed JSON object callback payload.
217///
218/// The workspace enables `serde_json/preserve_order`, so iterating this map
219/// yields fields in the order they appear in the source document rather than
220/// alphabetically. That is what makes the column-ordering contract on
221/// [`analyze_json_from_reader`] possible.
222pub type JsonObject = serde_json::Map<String, Value>;
223
224/// Summary of a JSON/JSONL scan.
225#[derive(Debug, Clone, PartialEq, Eq)]
226#[non_exhaustive]
227pub struct JsonScanSummary {
228    pub rows_read: usize,
229    /// Records that were read but not profiled: malformed JSON, and valid JSON
230    /// values that are not objects and therefore carry no fields. Both are
231    /// counted so a tolerant scan never looks like a clean one.
232    pub malformed_lines: usize,
233    pub format: FileFormat,
234    /// The scan stopped at `max_rows` while records still remained. A source
235    /// holding exactly `max_rows` records was read in full, so this stays false.
236    pub truncated: bool,
237    /// Under the skip policy, the array scanner stopped at an element it could
238    /// not read past and left input after it unread. Whether that input held
239    /// records is unknown, so the source was not exhausted: reporting it as
240    /// read in full let a drained remainder pass as a complete profile (#1106).
241    pub unread_after_fault: bool,
242}
243
244/// Scan JSON or JSONL input and invoke `on_object` for each object record.
245///
246/// - **JSONL**: one record per physical line. A record may not span lines and a
247///   line may not hold more than one value, so a concatenation whose delimiter
248///   was lost is reported rather than read as several clean records. Blank lines
249///   are separators: neither records nor errors.
250/// - **JSON**: one standard document. An array of objects streams element by
251///   element without buffering the whole input; a single object is read whole
252///   and is one record, which is why it may be pretty-printed across lines.
253///
254/// # Record policy
255///
256/// Only JSON objects are profileable records — they are the only JSON value
257/// with named fields to turn into columns. A record that is valid JSON but not
258/// an object (a scalar, an array, `null`) is never silently discarded: under
259/// [`JsonErrorPolicy::Skip`] it is counted in
260/// [`JsonScanSummary::malformed_lines`] and scanning continues with the next
261/// record, and under [`JsonErrorPolicy::Strict`] the first one fails the scan.
262///
263/// # Examples
264///
265/// Reading JSONL, keeping one field per record:
266///
267/// ```
268/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
269/// use std::io::Cursor;
270///
271/// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
272///
273/// let lines = "{\"city\": \"Rome\", \"pop\": 2800000}\n\
274///              {\"city\": \"Milan\", \"pop\": 1400000}\n";
275///
276/// let mut cities = Vec::new();
277/// let summary = scan_json_from_reader(
278///     Cursor::new(lines),
279///     &JsonParserConfig::jsonl(),
280///     |object| {
281///         if let Some(city) = object.get("city").and_then(|value| value.as_str()) {
282///             cities.push(city.to_string());
283///         }
284///     },
285/// )?;
286///
287/// assert_eq!(cities, ["Rome", "Milan"]);
288/// assert_eq!(summary.rows_read, 2);
289/// assert!(!summary.truncated);
290/// # Ok(())
291/// # }
292/// ```
293///
294/// The same records as a JSON array. The array streams element by element, and
295/// a record may be pretty-printed across lines because whitespace carries no
296/// meaning under this grammar:
297///
298/// ```
299/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
300/// use std::io::Cursor;
301///
302/// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
303///
304/// let document = r#"[
305///     {"city": "Rome", "pop": 2800000},
306///     {"city": "Milan", "pop": 1400000}
307/// ]"#;
308///
309/// let mut populations = Vec::new();
310/// let summary = scan_json_from_reader(
311///     Cursor::new(document),
312///     &JsonParserConfig::json_document(),
313///     |object| {
314///         // A field that is absent, or present but not a number, stays
315///         // `None`. Reading it as 0 instead would turn a missing value into
316///         // a real measurement, which is the one thing a profile must never
317///         // do.
318///         populations.push(object.get("pop").and_then(|value| value.as_u64()));
319///     },
320/// )?;
321///
322/// assert_eq!(populations, [Some(2_800_000), Some(1_400_000)]);
323/// assert_eq!(summary.rows_read, 2);
324/// assert_eq!(summary.malformed_lines, 0);
325/// # Ok(())
326/// # }
327/// ```
328pub fn scan_json_from_reader<R, F>(
329    reader: R,
330    config: &JsonParserConfig,
331    mut on_object: F,
332) -> Result<JsonScanSummary, DataProfilerError>
333where
334    R: BufRead,
335    F: FnMut(&JsonObject),
336{
337    let mut reader = Utf8BomReader::new(reader).map_err(DataProfilerError::from)?;
338    // The Unicode White_Space before a document is skipped, so detection
339    // and the array branch see past it as the object and JSONL readers'
340    // `trim` does (#1146). An explicit JSONL source is read from its first
341    // byte, as before, so its line numbers count every line.
342    let leading = match config.format {
343        Some(JsonFormat::Jsonl) => None,
344        _ => Some(skip_leading_white_space(&mut reader)?),
345    };
346    let first = leading.as_ref().and_then(|leading| leading.first);
347    // decode-audit: no-data — an explicit JSONL source skips nothing, so no
348    // character was held back to chain in front of the reader.
349    let pending = leading.map(|leading| leading.pending).unwrap_or_default();
350    let mut reader = std::io::Cursor::new(pending).chain(reader);
351    let format = match config.format {
352        Some(JsonFormat::Json) => FileFormat::Json,
353        Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
354        // A leading `{` is the first byte of both a JSON object document and a
355        // JSONL file, so detection can only resolve the array case and has to
356        // fall back to JSONL. Callers that know better pass `format`.
357        None if first == Some(b'[') => FileFormat::Json,
358        None => FileFormat::Jsonl,
359    };
360
361    let mut rows_read = 0;
362    let mut malformed_lines = 0;
363    let mut unread_after_fault = false;
364    let mut truncated = false;
365
366    match format {
367        // JSONL is line-delimited: exactly one record per non-blank physical
368        // line. Reading it as a whitespace-delimited stream of values instead
369        // made `{"a":1}{"b":2}` — a concatenation with the delimiter lost —
370        // profile as two clean rows, which is precisely the corruption a
371        // profiler exists to surface.
372        FileFormat::Jsonl => {
373            let mut line = String::new();
374            let mut line_number = 0;
375            loop {
376                if let Some(max) = config.max_rows
377                    && rows_read >= max
378                {
379                    // Reaching the cap is not truncation unless a record
380                    // actually remains: a file with exactly `max_rows` records
381                    // was read in full. Blank lines are not records.
382                    truncated = next_record_line(&mut reader, &mut line, &mut line_number)?;
383                    break;
384                }
385
386                if !next_record_line(&mut reader, &mut line, &mut line_number)? {
387                    break;
388                }
389
390                match read_jsonl_line(&line) {
391                    JsonRecord::Object(obj) => {
392                        on_object(&obj);
393                        rows_read += 1;
394                    }
395                    JsonRecord::NonObject(kind) => {
396                        if config.error_policy == JsonErrorPolicy::Strict {
397                            return Err(non_object_record_error(
398                                kind,
399                                rows_read + malformed_lines + 1,
400                            ));
401                        }
402                        malformed_lines += 1;
403                    }
404                    JsonRecord::Malformed(err) => {
405                        if config.error_policy == JsonErrorPolicy::Strict {
406                            return Err(record::malformed_jsonl_record_error(
407                                line_number,
408                                &line,
409                                err,
410                            ));
411                        }
412                        malformed_lines += 1;
413                    }
414                    JsonRecord::WideInteger => {
415                        if config.error_policy == JsonErrorPolicy::Strict {
416                            return Err(wide_integer_record_error(&format!(
417                                "on line {line_number}"
418                            )));
419                        }
420                        malformed_lines += 1;
421                    }
422                }
423            }
424        }
425        FileFormat::Json => {
426            // Which of the two record-carrying shapes this document is. The
427            // first byte after the white space decides, and is left unconsumed
428            // so the object branch still sees the value it opens.
429            let found_array = first == Some(b'[');
430
431            if !found_array {
432                // The other shape a standard JSON document takes: a single
433                // object, which is one record. It may be pretty-printed across
434                // lines, which is why this is not the JSONL grammar.
435                let (object_rows, object_malformed, object_truncated) =
436                    scan_single_json_object(&mut reader, config, &mut on_object)?;
437                rows_read += object_rows;
438                malformed_lines += object_malformed;
439                truncated |= object_truncated;
440            } else {
441                // The array loop starts after the opening bracket.
442                consume_peeked(&mut reader)?;
443
444                let mut expect_value = true;
445                let mut allow_end = true;
446                let mut array_closed = false;
447                let mut drain_remainder = false;
448
449                loop {
450                    let Some(next) = consume_leading_whitespace(&mut reader)? else {
451                        if config.error_policy == JsonErrorPolicy::Strict {
452                            return Err(malformed_array_error(
453                                "unexpected end of input before closing ']'",
454                            ));
455                        }
456                        malformed_lines += 1;
457                        drain_remainder = true;
458                        break;
459                    };
460
461                    if expect_value {
462                        if next == b']' {
463                            if !allow_end {
464                                if config.error_policy == JsonErrorPolicy::Strict {
465                                    return Err(malformed_array_error(
466                                        "trailing comma before closing ']'",
467                                    ));
468                                }
469                                malformed_lines += 1;
470                                drain_remainder = true;
471                                break;
472                            }
473                            consume_peeked(&mut reader)?;
474                            array_closed = true;
475                            break;
476                        }
477
478                        if let Some(max) = config.max_rows
479                            && rows_read >= max
480                        {
481                            // A value remains unread. Do not validate the unread
482                            // suffix of an intentionally bounded scan.
483                            truncated = true;
484                            break;
485                        }
486
487                        if next == b',' {
488                            if config.error_policy == JsonErrorPolicy::Strict {
489                                return Err(malformed_array_error(
490                                    "unexpected comma where an array value was required",
491                                ));
492                            }
493                            malformed_lines += 1;
494                            drain_remainder = true;
495                            break;
496                        }
497
498                        match read_json_record(&mut reader, next)? {
499                            JsonRecord::Object(obj) => {
500                                on_object(&obj);
501                                rows_read += 1;
502                            }
503                            JsonRecord::NonObject(kind) => {
504                                if config.error_policy == JsonErrorPolicy::Strict {
505                                    return Err(non_object_record_error(
506                                        kind,
507                                        rows_read + malformed_lines + 1,
508                                    ));
509                                }
510                                // The element was consumed in full, so the array
511                                // grammar is still intact and the objects after
512                                // it are still profileable.
513                                malformed_lines += 1;
514                            }
515                            JsonRecord::Malformed(err) => {
516                                if config.error_policy == JsonErrorPolicy::Strict {
517                                    return Err(malformed_record_error(err));
518                                }
519                                malformed_lines += 1;
520                                drain_remainder = true;
521                                break;
522                            }
523                            JsonRecord::WideInteger => {
524                                if config.error_policy == JsonErrorPolicy::Strict {
525                                    return Err(wide_integer_record_error(&format!(
526                                        "at position {}",
527                                        rows_read + malformed_lines + 1
528                                    )));
529                                }
530                                // Read in full: the array grammar is intact.
531                                malformed_lines += 1;
532                            }
533                        }
534                        expect_value = false;
535                    } else {
536                        match next {
537                            b',' => {
538                                consume_peeked(&mut reader)?;
539                                expect_value = true;
540                                allow_end = false;
541                            }
542                            b']' => {
543                                consume_peeked(&mut reader)?;
544                                array_closed = true;
545                                break;
546                            }
547                            _ => {
548                                if config.error_policy == JsonErrorPolicy::Strict {
549                                    return Err(malformed_array_error(
550                                        "expected ',' or ']' after an array value",
551                                    ));
552                                }
553                                malformed_lines += 1;
554                                drain_remainder = true;
555                                break;
556                            }
557                        }
558                    }
559                }
560
561                if drain_remainder {
562                    // Something left unread after the fault means the read
563                    // stopped short, whatever that input held (#1106).
564                    unread_after_fault = !rest_is_white_space(&mut reader)?.0;
565                    drain_to_end(&mut reader)?;
566                } else if array_closed && !rest_is_white_space(&mut reader)?.0 {
567                    if config.error_policy == JsonErrorPolicy::Strict {
568                        return Err(malformed_array_error(
569                            "non-whitespace content follows the closing ']'",
570                        ));
571                    }
572                    malformed_lines += 1;
573                    drain_to_end(&mut reader)?;
574                }
575            }
576        }
577        _ => unreachable!("json scanner only returns json or jsonl formats"),
578    }
579
580    Ok(JsonScanSummary {
581        rows_read,
582        malformed_lines,
583        format,
584        truncated,
585        unread_after_fault,
586    })
587}
588
589/// Read the next line that holds a record into `line`, skipping blank ones.
590///
591/// Returns whether a record line was found. Blank lines are separators, not
592/// records: they are skipped without counting as rows or as errors. `line_number`
593/// counts every physical line read, blank ones included, so a diagnostic points
594/// at the line the user would open the file to.
595fn next_record_line<R: BufRead>(
596    reader: &mut R,
597    line: &mut String,
598    line_number: &mut usize,
599) -> Result<bool, DataProfilerError> {
600    loop {
601        line.clear();
602        let read = reader.read_line(line).map_err(DataProfilerError::from)?;
603        if read == 0 {
604            return Ok(false);
605        }
606        *line_number += 1;
607        if !line.trim().is_empty() {
608            return Ok(true);
609        }
610    }
611}
612
613/// Read a whole-input single JSON object as one record.
614///
615/// Returns `(rows_read, malformed, truncated)`. Anything that is not exactly one object
616/// filling the input is an error under [`JsonErrorPolicy::Strict`] and a counted
617/// malformed record otherwise, so a truncated or concatenated document never
618/// profiles as clean.
619fn scan_single_json_object<R, F>(
620    reader: &mut R,
621    config: &JsonParserConfig,
622    on_object: &mut F,
623) -> Result<(usize, usize, bool), DataProfilerError>
624where
625    R: BufRead,
626    F: FnMut(&JsonObject),
627{
628    let mut text = String::new();
629    reader
630        .read_to_string(&mut text)
631        .map_err(DataProfilerError::from)?;
632
633    if text.trim().is_empty() {
634        return Ok((0, 0, false));
635    }
636
637    // A row cap of zero asks for no records, so the document is not read. The
638    // record it holds is left unread, which is a truncation and has to be
639    // reported as one — an unread record and an empty source must not produce
640    // the same report.
641    if config.max_rows == Some(0) {
642        return Ok((0, 0, true));
643    }
644
645    match serde_json::from_str::<Value>(text.trim()) {
646        Ok(Value::Object(obj)) => match object_record(obj, text.as_bytes()) {
647            JsonRecord::Object(obj) => {
648                on_object(&obj);
649                Ok((1, 0, false))
650            }
651            _ => {
652                if config.error_policy == JsonErrorPolicy::Strict {
653                    return Err(wide_integer_record_error("at position 1"));
654                }
655                Ok((0, 1, false))
656            }
657        },
658        Ok(value) => {
659            if config.error_policy == JsonErrorPolicy::Strict {
660                return Err(non_object_record_error(json_value_kind(&value), 1));
661            }
662            Ok((0, 1, false))
663        }
664        Err(err) => {
665            if config.error_policy == JsonErrorPolicy::Strict {
666                return Err(json_document_error(err));
667            }
668            Ok((0, 1, false))
669        }
670    }
671}
672
673/// Consume the byte [`consume_leading_whitespace`] just reported, along with the
674/// whitespace it looked past.
675///
676/// That function only peeks — it leaves the whitespace queued so serde keeps its
677/// line and column context — so consuming one byte would eat a space rather than
678/// the delimiter. Getting this wrong made a pretty-printed JSON array report a
679/// phantom malformed record, because the `]` survived the consume and then
680/// looked like content after the closing bracket.
681fn consume_peeked<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
682    skip_ascii_whitespace(reader)?;
683    reader.consume(1);
684    Ok(())
685}
686
687/// Build a strict-mode parse error from a serde failure. The serde message
688/// carries line/column context but never the record contents, so it is safe to
689/// surface directly.
690fn malformed_record_error(err: serde_json::Error) -> DataProfilerError {
691    let message = format!("malformed JSON record: {err}");
692    DataProfilerError::json_parsing_with_source(message, err)
693}
694
695fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
696    loop {
697        let mut bytes_to_consume = 0;
698        let first_non_whitespace = {
699            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
700            if buf.is_empty() {
701                return Ok(None);
702            }
703
704            let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
705            if first_non_whitespace.is_none() {
706                bytes_to_consume = buf.len();
707            }
708            first_non_whitespace
709        };
710
711        if first_non_whitespace.is_some() {
712            return Ok(first_non_whitespace);
713        }
714
715        reader.consume(bytes_to_consume);
716    }
717}
718
719/// Feed a JSON object's fields into a [`StreamingColumnCollection`].
720///
721/// Columns are registered in the order the fields appear in each record, so the
722/// resulting column order is the first record's field order with later-only
723/// fields appended where they were first seen.
724///
725/// `container_counts` tallies, per column, the values that were an object or an
726/// array. It is what [`profile_builder::mark_container_columns`] later reads
727/// to decide which columns hold nothing but containers.
728fn feed_json_object(
729    obj: &JsonObject,
730    prior_rows: usize,
731    known_columns: &mut Vec<String>,
732    known_columns_set: &mut HashSet<String>,
733    column_stats: &mut StreamingColumnCollection,
734    container_counts: &mut HashMap<String, usize>,
735) {
736    for (key, value) in obj {
737        if known_columns_set.insert(key.clone()) {
738            known_columns.push(key.clone());
739            column_stats.init_column_with_missing(key, prior_rows);
740        }
741        if matches!(value, Value::Array(_) | Value::Object(_)) {
742            *container_counts.entry(key.clone()).or_default() += 1;
743        }
744    }
745
746    let values: Vec<String> = known_columns
747        .iter()
748        .map(|column| {
749            // decode-audit: no-data — a key absent from this object is a
750            // missing field, and "" is the profiler's textual null.
751            obj.get(column).map(json_value_text).unwrap_or_default()
752        })
753        .collect();
754    let containers: Vec<usize> = known_columns
755        .iter()
756        .enumerate()
757        .filter(|(_, column)| matches!(obj.get(*column), Some(Value::Array(_) | Value::Object(_))))
758        .map(|(position, _)| position)
759        .collect();
760
761    column_stats.process_record_with_containers(known_columns, values, &containers);
762}
763
764/// Analyze JSON/JSONL data from a buffered reader using streaming statistics.
765///
766/// Returns `(column_profiles, streaming_stats, rows_read, malformed_lines, detected_format)`.
767///
768/// # Column order
769///
770/// Columns are returned in source order — the first record's field order, with
771/// fields that only appear in later records appended in first-seen order. This
772/// matches CSV (header order) and Parquet (schema order), so the same logical
773/// dataset profiles to the same column order in every format.
774///
775/// # Examples
776///
777/// The second record lists its fields in a different order and adds one. The
778/// first record still decides the order, and the new field is appended:
779///
780/// ```
781/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
782/// use std::io::Cursor;
783///
784/// use dataprof_json::{JsonParserConfig, analyze_json_from_reader};
785///
786/// let lines = "{\"id\": 1, \"city\": \"Rome\"}\n\
787///              {\"city\": \"Milan\", \"id\": 2, \"region\": \"Lombardy\"}\n";
788///
789/// let (profiles, _stats, rows_read, malformed_lines, format) =
790///     analyze_json_from_reader(Cursor::new(lines), &JsonParserConfig::jsonl())?;
791///
792/// let names: Vec<&str> = profiles.iter().map(|profile| profile.name.as_str()).collect();
793/// assert_eq!(names, ["id", "city", "region"]);
794/// assert_eq!(rows_read, 2);
795/// assert_eq!(malformed_lines, 0);
796/// assert_eq!(format, dataprof_core::FileFormat::Jsonl);
797///
798/// // The first record has no `region`, which counts as a missing value.
799/// let region = &profiles[2];
800/// assert_eq!(region.total_count, 2);
801/// assert_eq!(region.null_count, 1);
802/// # Ok(())
803/// # }
804/// ```
805pub fn analyze_json_from_reader<R: BufRead>(
806    reader: R,
807    config: &JsonParserConfig,
808) -> Result<
809    (
810        Vec<ColumnProfile>,
811        StreamingColumnCollection,
812        usize,
813        usize,
814        FileFormat,
815    ),
816    DataProfilerError,
817> {
818    let options = AnalysisOptions::default();
819    let (profiles, stats, rows_read, malformed_lines, format, _truncated, _unread) =
820        analyze_json_from_reader_full(reader, config, &options)?;
821    Ok((profiles, stats, rows_read, malformed_lines, format))
822}
823
824/// As [`analyze_json_from_reader`], but with the caller's analysis options, and
825/// also reports whether the scan stopped early at a row cap, and whether it
826/// left input unread after a fault. Kept private so the public tuple stays
827/// stable.
828#[allow(clippy::type_complexity)]
829fn analyze_json_from_reader_full<R: BufRead>(
830    reader: R,
831    config: &JsonParserConfig,
832    options: &AnalysisOptions,
833) -> Result<
834    (
835        Vec<ColumnProfile>,
836        StreamingColumnCollection,
837        usize,
838        usize,
839        FileFormat,
840        bool,
841        bool,
842    ),
843    DataProfilerError,
844> {
845    let semantic_hints = options.semantic_hints();
846    let mut column_stats = StreamingColumnCollection::new()
847        .with_semantic_hints(semantic_hints)
848        .with_duplicate_rows_exact_limit(options.duplicate_rows_exact_limit());
849    let mut known_columns = Vec::new();
850    let mut known_columns_set = HashSet::new();
851    let mut container_counts = HashMap::new();
852    let mut rows_seen = 0;
853
854    let summary = scan_json_from_reader(reader, config, |obj| {
855        feed_json_object(
856            obj,
857            rows_seen,
858            &mut known_columns,
859            &mut known_columns_set,
860            &mut column_stats,
861            &mut container_counts,
862        );
863        rows_seen += 1;
864    })?;
865
866    if let Some(indices) = options.column_indices(&column_stats.column_names())? {
867        let available = column_stats.column_names();
868        let selected = indices
869            .into_iter()
870            .map(|index| available[index].clone())
871            .collect::<Vec<_>>();
872        column_stats.retain_columns(&selected);
873    }
874
875    let profiles = profile_builder::mark_container_columns(
876        profile_builder::profiles_from_streaming_with_hints(
877            &column_stats,
878            !options.include_statistics(),
879            !options.include_patterns(),
880            options.locale(),
881            semantic_hints,
882        ),
883        &container_counts,
884    );
885
886    Ok((
887        profiles,
888        column_stats,
889        summary.rows_read,
890        summary.malformed_lines,
891        summary.format,
892        summary.truncated,
893        summary.unread_after_fault,
894    ))
895}
896
897/// Analyze a JSON or JSONL file, returning a full [`ProfileReport`].
898///
899/// With no explicit [`JsonParserConfig::format`] the file extension decides:
900/// `.json` reads as a standard document, `.jsonl` and `.ndjson` as JSON Lines,
901/// and any other name falls back to sniffing the first byte.
902///
903/// # Examples
904///
905/// ```
906/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
907/// use std::io::Write;
908///
909/// use dataprof_json::{JsonParserConfig, analyze_json_file};
910///
911/// let mut file = tempfile::NamedTempFile::with_suffix(".jsonl")?;
912/// writeln!(file, "{{\"id\": 1, \"score\": 9.5}}")?;
913/// writeln!(file, "{{\"id\": 2, \"score\": 7.0}}")?;
914/// file.flush()?;
915///
916/// let report = analyze_json_file(file.path(), &JsonParserConfig::default())?;
917///
918/// assert_eq!(report.execution.rows_processed, 2);
919/// assert_eq!(report.execution.columns_detected, 2);
920/// assert!(report.execution.truncation_reason.is_none());
921/// # Ok(())
922/// # }
923/// ```
924pub fn analyze_json_file(
925    file_path: &Path,
926    config: &JsonParserConfig,
927) -> Result<ProfileReport, DataProfilerError> {
928    analyze_json_file_with_options(file_path, config, &AnalysisOptions::default())
929}
930
931/// Analyze a JSON or JSONL file, honouring the caller's full analysis selection.
932///
933/// This is the entry point that carries metric packs and locale as well as
934/// dimensions and hints, so a JSON profile reports exactly the analysis the
935/// caller asked for — the same selection the CSV engines apply.
936pub fn analyze_json_file_with_options(
937    file_path: &Path,
938    config: &JsonParserConfig,
939    options: &AnalysisOptions,
940) -> Result<ProfileReport, DataProfilerError> {
941    // An explicit grammar wins; otherwise the file name gets a say before
942    // content sniffing, which cannot tell a JSON object from a JSONL stream.
943    let mut config = config.clone();
944    if config.format.is_none() {
945        config.format = grammar_for_extension(file_path);
946    }
947    let config = &config;
948    let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
949    dataprof_core::binary_input::reject_binary_input(file_path)?;
950    let start = std::time::Instant::now();
951
952    let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
953    let buf_reader = std::io::BufReader::new(file);
954
955    let (
956        column_profiles,
957        column_stats,
958        rows_read,
959        malformed_lines,
960        format,
961        truncated,
962        unread_after_fault,
963    ) = analyze_json_from_reader_full(buf_reader, config, options)?;
964    let read_as_json_document = format == FileFormat::Json;
965
966    let file_source = DataSource::File {
967        path: file_path.display().to_string(),
968        format,
969        size_bytes: metadata.len(),
970        modified_at: None,
971        parquet_metadata: None,
972        csv_metadata: None,
973    };
974
975    if rows_read == 0 {
976        if malformed_lines > 0 {
977            // Nothing parsed under the JSON grammar most often means the source
978            // is JSONL, so say so here too: under the default skip policy this
979            // is the only error the caller sees.
980            let hint = if read_as_json_document {
981                ". A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
982            } else {
983                ""
984            };
985            // Aggregate over many discarded records: no single decoder error
986            // is the cause, so none is retained.
987            return Err(DataProfilerError::json_parsing_error(&format!(
988                "No valid JSON records found in file \
989                 (every record was malformed, not a JSON object, or held an integer outside the 64-bit range){hint}"
990            )));
991        }
992        // A row cap that stopped the scan before the first record still has to
993        // be disclosed. Without this the report reads exactly like a complete
994        // profile of an empty source, which is the one thing a truncated scan
995        // must never be mistaken for.
996        let mut execution =
997            ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("json");
998        if truncated && let Some(max) = config.max_rows {
999            execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1000        }
1001        return Ok(ReportAssembler::new(file_source, execution)
1002            .columns(column_profiles)
1003            .with_quality_data(HashMap::new())
1004            .with_analysis_options(options)
1005            .build());
1006    }
1007
1008    let sample_columns = profile_builder::quality_check_samples(&column_stats);
1009    let scan_time_ms = start.elapsed().as_millis();
1010    let num_columns = column_profiles.len();
1011
1012    let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms)
1013        .with_engine("json")
1014        // Tolerant scans surface skipped malformed records here so callers can
1015        // tell a partial profile from a clean one.
1016        .with_error_count(malformed_lines);
1017    // `truncated` is set only when a record still remained at the cap, so a source
1018    // holding exactly `max_rows` records is reported as fully read, not cut short.
1019    if truncated && let Some(max) = config.max_rows {
1020        execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1021    }
1022    if unread_after_fault {
1023        execution = execution.with_source_exhausted(false);
1024    }
1025
1026    Ok(ReportAssembler::new(file_source, execution)
1027        .columns(column_profiles)
1028        .with_quality_data(sample_columns)
1029        .with_row_duplicates(column_stats.row_duplicate_summary())
1030        .with_row_completeness(column_stats.row_completeness_summary())
1031        .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
1032        .with_analysis_options(options)
1033        .build())
1034}
1035
1036/// The grammar a file name advertises, if it advertises one.
1037///
1038/// Content sniffing resolves a leading `[` on its own, but not a leading `{`:
1039/// that is the first byte of both a JSON object document and a JSONL file. The
1040/// extension is the evidence that separates them, and a `.json` file holding one
1041/// pretty-printed object is an ordinary thing to profile. `None` means the name
1042/// says nothing and the content decides.
1043fn grammar_for_extension(file_path: &Path) -> Option<JsonFormat> {
1044    let extension = file_path.extension().and_then(|ext| ext.to_str())?;
1045    if extension.eq_ignore_ascii_case("json") {
1046        Some(JsonFormat::Json)
1047    } else if extension.eq_ignore_ascii_case("jsonl") || extension.eq_ignore_ascii_case("ndjson") {
1048        Some(JsonFormat::Jsonl)
1049    } else {
1050        None
1051    }
1052}
1053
1054fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
1055    if error.kind() == std::io::ErrorKind::NotFound {
1056        DataProfilerError::FileNotFound {
1057            path: file_path.display().to_string(),
1058        }
1059    } else {
1060        DataProfilerError::from(error)
1061    }
1062}
1063
1064#[cfg(test)]
1065mod tests {
1066    use super::*;
1067    use std::io::{Cursor, Write};
1068    use tempfile::NamedTempFile;
1069
1070    /// A temp file with no extension, so `analyze_json_file` falls back to the
1071    /// JSONL grammar the way an unnamed stream does.
1072    fn write_file(content: &str) -> NamedTempFile {
1073        let mut file = NamedTempFile::new().unwrap();
1074        write!(file, "{}", content).unwrap();
1075        file.flush().unwrap();
1076        file
1077    }
1078
1079    /// A temp file named `.json`, so `analyze_json_file` reads it as a standard
1080    /// JSON document — the grammar the extension advertises.
1081    fn write_json_file(content: &str) -> NamedTempFile {
1082        let mut file = NamedTempFile::with_suffix(".json").unwrap();
1083        write!(file, "{}", content).unwrap();
1084        file.flush().unwrap();
1085        file
1086    }
1087
1088    fn write_bytes(content: &[u8]) -> NamedTempFile {
1089        let mut file = NamedTempFile::new().unwrap();
1090        file.write_all(content).unwrap();
1091        file.flush().unwrap();
1092        file
1093    }
1094
1095    #[test]
1096    fn test_scan_jsonl_detects_format_and_rows() {
1097        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1098        let mut rows = 0;
1099
1100        let summary = scan_json_from_reader(
1101            Cursor::new(data.as_ref()),
1102            &JsonParserConfig::default(),
1103            |_| {
1104                rows += 1;
1105            },
1106        )
1107        .unwrap();
1108
1109        assert_eq!(summary.format, FileFormat::Jsonl);
1110        assert_eq!(summary.rows_read, 3);
1111        assert_eq!(summary.malformed_lines, 0);
1112        assert_eq!(rows, 3);
1113    }
1114
1115    #[test]
1116    fn test_scan_json_array_detects_format_and_rows() {
1117        let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
1118        let mut keys = Vec::new();
1119
1120        let summary = scan_json_from_reader(
1121            Cursor::new(data.as_ref()),
1122            &JsonParserConfig::default(),
1123            |obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
1124        )
1125        .unwrap();
1126
1127        assert_eq!(summary.format, FileFormat::Json);
1128        assert_eq!(summary.rows_read, 2);
1129        assert_eq!(keys.len(), 2);
1130        assert_eq!(keys[0], vec!["name".to_string()]);
1131    }
1132
1133    #[test]
1134    fn test_scan_accepts_one_leading_utf8_bom_for_json_and_jsonl() {
1135        for (data, config, expected_format) in [
1136            (
1137                b"\xEF\xBB\xBF[{\"x\":1},{\"x\":2}]".as_slice(),
1138                JsonParserConfig::default(),
1139                FileFormat::Json,
1140            ),
1141            (
1142                b"\xEF\xBB\xBF{\"x\":1}\n{\"x\":2}\n".as_slice(),
1143                JsonParserConfig::default(),
1144                FileFormat::Jsonl,
1145            ),
1146        ] {
1147            let mut rows = 0;
1148            let summary = scan_json_from_reader(Cursor::new(data), &config, |_| rows += 1)
1149                .expect("leading UTF-8 BOM should be accepted");
1150
1151            assert_eq!(rows, 2);
1152            assert_eq!(summary.rows_read, 2);
1153            assert_eq!(summary.malformed_lines, 0);
1154            assert_eq!(summary.format, expected_format);
1155        }
1156    }
1157
1158    #[test]
1159    fn test_scan_does_not_strip_nonleading_or_second_utf8_bom() {
1160        let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1161
1162        for data in [
1163            b" \xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1164            b"\xEF\xBB\xBF\xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1165            b"{\"x\":1}\n\xEF\xBB\xBF{\"x\":2}\n".as_slice(),
1166        ] {
1167            scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1168                .expect_err("only the BOM at absolute byte offset zero may be stripped");
1169        }
1170    }
1171
1172    #[test]
1173    fn test_bom_only_input_matches_empty_input_policy() {
1174        for config in [JsonParserConfig::default(), JsonParserConfig::jsonl()] {
1175            let empty = scan_json_from_reader(Cursor::new(b""), &config, |_| {}).unwrap();
1176            let bom =
1177                scan_json_from_reader(Cursor::new(b"\xEF\xBB\xBF".as_slice()), &config, |_| {})
1178                    .unwrap();
1179            assert_eq!(bom, empty);
1180        }
1181    }
1182
1183    #[test]
1184    fn test_json_array_container_grammar_obeys_error_policy() {
1185        let cases = [
1186            (br#"[{"x":1}"#.as_ref(), 1, "missing closing bracket"),
1187            (br#"[{"x":1},"#.as_ref(), 1, "comma followed by EOF"),
1188            (br#"[{"x":1} {"x":2}]"#.as_ref(), 1, "missing comma"),
1189            (br#"[{"x":1},,{"x":2}]"#.as_ref(), 1, "doubled comma"),
1190            (br#"[,{"x":1}]"#.as_ref(), 0, "leading comma"),
1191            (br#"[{"x":1},]"#.as_ref(), 1, "trailing comma"),
1192            (br#"[{"x":1}] trailing"#.as_ref(), 1, "trailing content"),
1193            (
1194                br#"[{"x":1}][{"x":2}]"#.as_ref(),
1195                1,
1196                "second top-level value",
1197            ),
1198        ];
1199
1200        for (data, expected_rows, case) in cases {
1201            let skip = JsonParserConfig::json_document();
1202            let summary =
1203                scan_json_from_reader(Cursor::new(data), &skip, |_| {}).expect("tolerant scan");
1204            assert_eq!(summary.rows_read, expected_rows, "{case}");
1205            assert_eq!(summary.malformed_lines, 1, "{case}");
1206
1207            let strict =
1208                JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict);
1209            let err = scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1210                .expect_err("strict mode must reject invalid array grammar");
1211            assert!(
1212                err.to_string()
1213                    .to_lowercase()
1214                    .contains("malformed json array"),
1215                "{case}: {err}"
1216            );
1217        }
1218    }
1219
1220    #[test]
1221    fn test_json_array_max_rows_does_not_validate_unread_values() {
1222        let data = br#"[{"x":1},not-valid-json"#;
1223        let config = JsonParserConfig::json_document().with_max_rows(1);
1224        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1225
1226        assert_eq!(summary.rows_read, 1);
1227        assert_eq!(summary.malformed_lines, 0);
1228        assert!(summary.truncated);
1229    }
1230
1231    #[test]
1232    fn test_scan_respects_max_rows() {
1233        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
1234        let config = JsonParserConfig::default().with_max_rows(2);
1235        let mut rows = 0;
1236
1237        let summary =
1238            scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
1239
1240        assert_eq!(summary.rows_read, 2);
1241        assert_eq!(rows, 2);
1242    }
1243
1244    #[test]
1245    fn test_scan_skips_malformed_jsonl_lines() {
1246        let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
1247        let mut rows = 0;
1248
1249        let summary = scan_json_from_reader(
1250            Cursor::new(data.as_ref()),
1251            &JsonParserConfig::jsonl(),
1252            |_| {
1253                rows += 1;
1254            },
1255        )
1256        .unwrap();
1257
1258        assert_eq!(summary.format, FileFormat::Jsonl);
1259        assert_eq!(summary.rows_read, 2);
1260        assert_eq!(summary.malformed_lines, 1);
1261        assert_eq!(rows, 2);
1262    }
1263
1264    /// A pretty-printed root object is one record under the JSON grammar and a
1265    /// malformed record under JSONL (#486). It cannot be both: whitespace is
1266    /// insignificant in a JSON document and is the record delimiter in JSONL, so
1267    /// reading a multi-line object as JSONL would mean each of its lines is a
1268    /// record — none of which is valid JSON on its own.
1269    #[test]
1270    fn test_pretty_printed_root_object_is_one_row_under_the_json_grammar() {
1271        let data = br#"{
1272    "type": "FeatureCollection",
1273    "features": [
1274        {"id": 1},
1275        {"id": 2}
1276  ]
1277}"#;
1278        let mut object_field_counts = Vec::new();
1279
1280        let summary = scan_json_from_reader(
1281            Cursor::new(data.as_ref()),
1282            &JsonParserConfig::json_document(),
1283            |obj| object_field_counts.push(obj.len()),
1284        )
1285        .unwrap();
1286
1287        assert_eq!(summary.format, FileFormat::Json);
1288        assert_eq!(summary.rows_read, 1);
1289        assert_eq!(summary.malformed_lines, 0);
1290        assert_eq!(object_field_counts, vec![2]);
1291    }
1292
1293    #[test]
1294    fn test_pretty_printed_root_object_is_not_valid_jsonl() {
1295        let data = br#"{
1296    "type": "FeatureCollection"
1297}"#;
1298        let err = scan_json_from_reader(
1299            Cursor::new(data.as_ref()),
1300            &JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict),
1301            |_| {},
1302        )
1303        .expect_err("a record spanning lines is not JSONL");
1304
1305        assert!(
1306            err.to_string().contains("malformed JSON record"),
1307            "{err}, expected the malformed category"
1308        );
1309    }
1310
1311    /// Every JSON value that is not an object, so none of them can be a row.
1312    /// Numbers are listed twice on purpose: they are the one value serde reads
1313    /// from a reader by peeking one byte past the end.
1314    const NON_OBJECT_VALUES: &[(&str, &str)] = &[
1315        ("null", "null"),
1316        ("boolean", "true"),
1317        ("number", "42"),
1318        ("number", "-1.5e3"),
1319        ("string", r#""text""#),
1320        ("array", "[1, 2]"),
1321    ];
1322
1323    #[test]
1324    fn test_jsonl_counts_non_object_records_and_keeps_scanning() {
1325        for (kind, value) in NON_OBJECT_VALUES {
1326            let data = format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n");
1327            let mut rows = 0;
1328
1329            let summary = scan_json_from_reader(
1330                Cursor::new(data.as_bytes()),
1331                &JsonParserConfig::jsonl(),
1332                |_| rows += 1,
1333            )
1334            .unwrap();
1335
1336            assert_eq!(summary.rows_read, 2, "{kind}: record after it was lost");
1337            assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1338            assert_eq!(rows, 2, "{kind}");
1339        }
1340    }
1341
1342    #[test]
1343    fn test_json_array_counts_non_object_elements_and_keeps_scanning() {
1344        for (kind, value) in NON_OBJECT_VALUES {
1345            let data = format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]");
1346            let mut rows = 0;
1347
1348            let summary = scan_json_from_reader(
1349                Cursor::new(data.as_bytes()),
1350                &JsonParserConfig::json_document(),
1351                |_| rows += 1,
1352            )
1353            .unwrap();
1354
1355            assert_eq!(summary.rows_read, 2, "{kind}: element after it was lost");
1356            assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1357            assert_eq!(rows, 2, "{kind}");
1358        }
1359    }
1360
1361    #[test]
1362    fn test_strict_policy_rejects_the_first_non_object_record() {
1363        for (kind, value) in NON_OBJECT_VALUES {
1364            for (config, data) in [
1365                (
1366                    JsonParserConfig::jsonl(),
1367                    format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n"),
1368                ),
1369                (
1370                    JsonParserConfig::json_document(),
1371                    format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]"),
1372                ),
1373            ] {
1374                let config = config.with_error_policy(JsonErrorPolicy::Strict);
1375                let err = scan_json_from_reader(Cursor::new(data.as_bytes()), &config, |_| {})
1376                    .expect_err("{kind}: strict mode must reject a non-object record");
1377
1378                let message = err.to_string();
1379                assert!(message.contains("non-object JSON record"), "{message}");
1380                assert!(message.contains("at position 2"), "{message}");
1381                assert!(message.contains(&format!("found {kind}")), "{message}");
1382            }
1383        }
1384    }
1385
1386    #[test]
1387    fn test_number_element_does_not_swallow_the_array_delimiter() {
1388        // serde ends a number by peeking one byte past it and drops that byte
1389        // with the deserializer; here it is the `,` the array scanner needs.
1390        let data = br#"[{"id":1}, 1, 2, 3, {"id":2}, 4, {"id":3}]"#;
1391        let mut rows = 0;
1392
1393        let summary = scan_json_from_reader(
1394            Cursor::new(data.as_ref()),
1395            &JsonParserConfig::json_document(),
1396            |_| rows += 1,
1397        )
1398        .unwrap();
1399
1400        assert_eq!(summary.rows_read, 3);
1401        assert_eq!(summary.malformed_lines, 4);
1402        assert_eq!(rows, 3);
1403    }
1404
1405    #[test]
1406    fn test_number_shaped_garbage_stays_a_malformed_record() {
1407        // `1.2.3` is read by the byte-wise number scanner but is not a number,
1408        // so it must not be reported as a valid non-object record.
1409        let data = b"{\"id\":1}\n1.2.3\n";
1410        let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1411        let err = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {})
1412            .expect_err("number-shaped garbage should be malformed");
1413
1414        assert!(
1415            err.to_string().contains("malformed JSON record"),
1416            "{err}, expected the malformed category"
1417        );
1418    }
1419
1420    #[test]
1421    fn test_input_of_only_non_object_records_fails() {
1422        let file = write_file("\"just a string\"\n1\n[2]\n");
1423        let err = analyze_json_file(file.path(), &JsonParserConfig::jsonl())
1424            .expect_err("nothing profileable should not return a clean empty profile");
1425
1426        assert!(
1427            err.to_string().contains("No valid JSON records found"),
1428            "{err}"
1429        );
1430    }
1431
1432    #[test]
1433    /// The JSON grammar accepts both shapes that carry records, so a bare object
1434    /// is one row rather than an error (#486).
1435    fn test_json_grammar_reads_a_bare_object_as_one_row() {
1436        let data = br#"{"x":1}"#;
1437        let mut rows = 0;
1438        let summary = scan_json_from_reader(
1439            Cursor::new(data.as_ref()),
1440            &JsonParserConfig::json_document(),
1441            |_| rows += 1,
1442        )
1443        .unwrap();
1444
1445        assert_eq!(summary.rows_read, 1);
1446        assert_eq!(summary.malformed_lines, 0);
1447        assert_eq!(rows, 1);
1448    }
1449
1450    #[test]
1451    /// Several objects back to back are not a JSON document. The error names
1452    /// the option that reads them, because this is what a JSONL file looks like
1453    /// when it is read with the wrong grammar.
1454    fn test_json_grammar_rejects_concatenated_objects() {
1455        let data = br#"{"x":1}{"x":2}"#;
1456        let err = scan_json_from_reader(
1457            Cursor::new(data.as_ref()),
1458            &JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict),
1459            |_| {},
1460        )
1461        .expect_err("two objects are not one JSON document");
1462
1463        let message = err.to_string();
1464        assert!(message.contains("malformed JSON document"), "{message}");
1465        assert!(message.contains("format=\"jsonl\""), "{message}");
1466    }
1467
1468    #[test]
1469    fn test_analyze_json_from_reader_jsonl_streaming() {
1470        let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
1471        let cursor = Cursor::new(data.as_ref());
1472        let config = JsonParserConfig::default();
1473
1474        let (profiles, _stats, rows, _malformed, format) =
1475            analyze_json_from_reader(cursor, &config).unwrap();
1476        assert_eq!(format, FileFormat::Jsonl);
1477        assert_eq!(rows, 3);
1478        assert_eq!(profiles.len(), 2);
1479    }
1480
1481    #[test]
1482    fn test_analyze_json_from_reader_json_array() {
1483        let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
1484        let cursor = Cursor::new(data.as_ref());
1485        let config = JsonParserConfig::default();
1486
1487        let (profiles, _stats, rows, _malformed, format) =
1488            analyze_json_from_reader(cursor, &config).unwrap();
1489        assert_eq!(format, FileFormat::Json);
1490        assert_eq!(rows, 2);
1491        assert_eq!(profiles.len(), 2);
1492    }
1493
1494    #[test]
1495    fn test_integer_negative_zero_profiles_as_integer_zero_in_every_shape() {
1496        for (data, config) in [
1497            (
1498                &b"[{\"v\":0},{\"v\":-0}]"[..],
1499                JsonParserConfig::json_document(),
1500            ),
1501            (&b"{\"v\":0}\n{\"v\":-0}\n"[..], JsonParserConfig::jsonl()),
1502            (&b"{\"v\":-0}"[..], JsonParserConfig::json_document()),
1503        ] {
1504            let (profiles, _stats, _rows, _malformed, _format) =
1505                analyze_json_from_reader(Cursor::new(data), &config).unwrap();
1506            assert_eq!(
1507                profiles[0].data_type,
1508                dataprof_core::DataType::Integer,
1509                "{data:?}"
1510            );
1511            assert_eq!(profiles[0].unique_count, Some(1), "{data:?}");
1512        }
1513    }
1514
1515    #[test]
1516    fn test_analyze_json_from_reader_max_rows() {
1517        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
1518        let cursor = Cursor::new(data.as_ref());
1519        let config = JsonParserConfig::default().with_max_rows(3);
1520
1521        let (_profiles, _stats, rows, _malformed, _format) =
1522            analyze_json_from_reader(cursor, &config).unwrap();
1523        assert_eq!(rows, 3);
1524    }
1525
1526    #[test]
1527    fn test_analyze_json_from_reader_missing_fields() {
1528        let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
1529        let cursor = Cursor::new(data.as_ref());
1530        let config = JsonParserConfig::jsonl();
1531
1532        let (profiles, _stats, rows, _malformed, _format) =
1533            analyze_json_from_reader(cursor, &config).unwrap();
1534        assert_eq!(rows, 2);
1535
1536        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1537        assert_eq!(col_b.total_count, 2);
1538    }
1539
1540    #[test]
1541    fn test_identical_records_count_as_duplicates_across_a_late_key() {
1542        // Records 1 and 3 are byte-identical; `b` is only discovered between
1543        // them. Signing a row against whatever columns happened to be known at
1544        // the time would make the two look different and report no duplicate.
1545        let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"x\"}\n";
1546        let cursor = Cursor::new(data.as_ref());
1547        let config = JsonParserConfig::jsonl();
1548
1549        let (_profiles, stats, rows, _malformed, _format) =
1550            analyze_json_from_reader(cursor, &config).unwrap();
1551        assert_eq!(rows, 3);
1552
1553        let summary = stats
1554            .row_duplicate_summary()
1555            .expect("three rows were observed");
1556        assert_eq!(summary.rows_checked, 3);
1557        assert_eq!(summary.duplicate_rows, 1);
1558        assert!(!summary.approximate);
1559    }
1560
1561    #[test]
1562    fn test_a_late_key_does_not_invent_duplicates() {
1563        // The counterpart guard: collapsing the trailing absent field must not
1564        // merge rows that differ in the columns they do share.
1565        let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"z\"}\n";
1566        let cursor = Cursor::new(data.as_ref());
1567        let config = JsonParserConfig::jsonl();
1568
1569        let (_profiles, stats, _rows, _malformed, _format) =
1570            analyze_json_from_reader(cursor, &config).unwrap();
1571
1572        let summary = stats
1573            .row_duplicate_summary()
1574            .expect("three rows were observed");
1575        assert_eq!(summary.duplicate_rows, 0);
1576    }
1577
1578    #[test]
1579    fn test_analyze_json_file_quality_report() {
1580        let file = write_file(r#"[{"x":1},{"x":2}]"#);
1581        let config = JsonParserConfig::default();
1582        let report = analyze_json_file(file.path(), &config).unwrap();
1583
1584        assert_eq!(report.execution.rows_processed, 2);
1585        assert_eq!(report.column_profiles.len(), 1);
1586        assert!(report.quality_score().unwrap() >= 0.0);
1587    }
1588
1589    #[test]
1590    fn test_jsonl_skips_malformed_lines() {
1591        let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
1592        let cursor = Cursor::new(data.as_ref());
1593        let config = JsonParserConfig::jsonl();
1594
1595        let (profiles, _stats, rows, _malformed, format) =
1596            analyze_json_from_reader(cursor, &config).unwrap();
1597        assert_eq!(format, FileFormat::Jsonl);
1598        assert_eq!(rows, 2);
1599        assert_eq!(profiles[0].total_count, 2);
1600    }
1601
1602    #[test]
1603    fn test_strict_policy_errors_on_malformed_jsonl() {
1604        // Malformed in first / middle / last position all abort under Strict.
1605        for data in [
1606            b"not-json\n{\"x\":1}\n{\"x\":2}\n".as_ref(),
1607            b"{\"x\":1}\nnot-json\n{\"x\":2}\n".as_ref(),
1608            b"{\"x\":1}\n{\"x\":2}\nnot-json\n".as_ref(),
1609        ] {
1610            let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1611            let err = scan_json_from_reader(Cursor::new(data), &config, |_| {})
1612                .expect_err("strict mode must reject malformed records");
1613            let message = err.to_string().to_lowercase();
1614            assert!(message.contains("malformed json record"), "got: {message}");
1615            // The offending record text must never be echoed back.
1616            assert!(
1617                !err.to_string().contains("not-json"),
1618                "leaked record: {err}"
1619            );
1620        }
1621    }
1622
1623    #[test]
1624    fn test_tolerant_policy_counts_skipped_records() {
1625        let data = b"{\"x\":1}\nnot-json\n{\"x\":2}\n";
1626        let config = JsonParserConfig::jsonl(); // default Skip
1627        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1628        assert_eq!(summary.rows_read, 2);
1629        assert_eq!(summary.malformed_lines, 1);
1630    }
1631
1632    #[test]
1633    fn test_blank_lines_are_not_counted_as_malformed() {
1634        let data = b"{\"x\":1}\n\n   \n{\"x\":2}\n";
1635        for policy in [JsonErrorPolicy::Skip, JsonErrorPolicy::Strict] {
1636            let config = JsonParserConfig::jsonl().with_error_policy(policy);
1637            let summary =
1638                scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1639            assert_eq!(summary.rows_read, 2, "policy {policy:?}");
1640            assert_eq!(summary.malformed_lines, 0, "policy {policy:?}");
1641        }
1642    }
1643
1644    #[test]
1645    fn test_truncated_final_record_obeys_error_policy() {
1646        let data = b"{\"x\":1}\n{\"x\":2";
1647
1648        let skip = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Skip);
1649        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &skip, |_| {}).unwrap();
1650        assert_eq!(summary.rows_read, 1);
1651        assert_eq!(summary.malformed_lines, 1);
1652
1653        let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1654        let err = scan_json_from_reader(Cursor::new(data.as_ref()), &strict, |_| {})
1655            .expect_err("strict mode must reject an incomplete trailing record");
1656        assert!(
1657            err.to_string()
1658                .to_lowercase()
1659                .contains("malformed json record")
1660        );
1661    }
1662
1663    #[test]
1664    fn test_all_malformed_file_fails() {
1665        let file = write_file("not-json\nalso-bad\n");
1666        let config = JsonParserConfig::jsonl();
1667        let err = analyze_json_file(file.path(), &config)
1668            .expect_err("a file with no valid records must fail");
1669        let message = err.to_string().to_lowercase();
1670        assert!(message.contains("no valid json records"), "got: {message}");
1671    }
1672
1673    #[test]
1674    fn test_tolerant_file_reports_error_count() {
1675        let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1676        let config = JsonParserConfig::jsonl();
1677        let report = analyze_json_file(file.path(), &config).unwrap();
1678        assert_eq!(report.execution.rows_processed, 2);
1679        assert_eq!(report.execution.error_count, 1);
1680    }
1681
1682    #[test]
1683    fn test_strict_file_errors_on_malformed() {
1684        let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1685        let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1686        let err = analyze_json_file(file.path(), &config)
1687            .expect_err("strict mode must reject the malformed record");
1688        assert!(
1689            err.to_string()
1690                .to_lowercase()
1691                .contains("malformed json record")
1692        );
1693    }
1694
1695    #[test]
1696    fn test_analyze_json_with_large_leading_whitespace() {
1697        let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
1698        let cursor = Cursor::new(data.into_bytes());
1699        let config = JsonParserConfig::default();
1700
1701        let (_profiles, _stats, rows, malformed, format) =
1702            analyze_json_from_reader(cursor, &config).unwrap();
1703        assert_eq!(format, FileFormat::Json);
1704        assert_eq!(rows, 1);
1705        assert_eq!(malformed, 0);
1706    }
1707
1708    #[test]
1709    fn test_analyze_json_array() {
1710        let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
1711        let config = JsonParserConfig::default();
1712        let report = analyze_json_file(json.path(), &config).unwrap();
1713        let profiles = &report.column_profiles;
1714
1715        assert_eq!(profiles.len(), 2);
1716        let names: Vec<&str> = profiles
1717            .iter()
1718            .map(|profile| profile.name.as_str())
1719            .collect();
1720        assert!(names.contains(&"name"));
1721        assert!(names.contains(&"age"));
1722
1723        let age = profiles
1724            .iter()
1725            .find(|profile| profile.name == "age")
1726            .unwrap();
1727        assert_eq!(age.total_count, 2);
1728        assert_eq!(age.null_count, 0);
1729    }
1730
1731    #[test]
1732    fn test_analyze_jsonl() {
1733        let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
1734        let config = JsonParserConfig::default();
1735        let report = analyze_json_file(jsonl.path(), &config).unwrap();
1736        let profiles = &report.column_profiles;
1737
1738        assert_eq!(profiles.len(), 1);
1739        assert_eq!(profiles[0].name, "x");
1740        assert_eq!(profiles[0].total_count, 3);
1741    }
1742
1743    #[test]
1744    fn test_analyze_json_with_nulls() {
1745        let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
1746        let config = JsonParserConfig::default();
1747        let report = analyze_json_file(json.path(), &config).unwrap();
1748        let profiles = &report.column_profiles;
1749
1750        let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
1751        assert_eq!(col_a.null_count, 1);
1752
1753        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1754        assert_eq!(col_b.null_count, 1);
1755    }
1756
1757    #[test]
1758    fn test_analyze_json_with_missing_fields() {
1759        let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
1760        let config = JsonParserConfig::default();
1761        let report = analyze_json_file(json.path(), &config).unwrap();
1762        let profiles = &report.column_profiles;
1763
1764        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1765        assert_eq!(col_b.total_count, 2);
1766    }
1767
1768    /// Deliberately non-alphabetical field order: sorting would give
1769    /// `active, amount, date, id`, so any re-sort is visible.
1770    const UNSORTED_FIELDS: &str = r#"{"id":1,"amount":12.5,"active":true,"date":"2026-07-23"}"#;
1771    const UNSORTED_ORDER: [&str; 4] = ["id", "amount", "active", "date"];
1772
1773    fn profile_names(report: &ProfileReport) -> Vec<&str> {
1774        report
1775            .column_profiles
1776            .iter()
1777            .map(|profile| profile.name.as_str())
1778            .collect()
1779    }
1780
1781    #[test]
1782    fn test_json_column_order_follows_source_not_alphabet() {
1783        let array = write_file(&format!("[{UNSORTED_FIELDS},{UNSORTED_FIELDS}]"));
1784        let jsonl = write_file(&format!("{UNSORTED_FIELDS}\n{UNSORTED_FIELDS}\n"));
1785
1786        for file in [&array, &jsonl] {
1787            let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1788            assert_eq!(profile_names(&report), UNSORTED_ORDER);
1789        }
1790    }
1791
1792    #[test]
1793    fn test_json_column_order_appends_later_keys_in_first_seen_order() {
1794        // Each record introduces two keys in reverse-alphabetical order, so
1795        // sorting would reshuffle both the leading pair and the appended pair.
1796        let data = b"{\"zulu\":1,\"mike\":2}\n{\"zulu\":3,\"mike\":4,\"delta\":5,\"alpha\":6}\n";
1797        let file = write_bytes(data);
1798        let report = analyze_json_file(file.path(), &JsonParserConfig::jsonl()).unwrap();
1799
1800        assert_eq!(profile_names(&report), ["zulu", "mike", "delta", "alpha"]);
1801    }
1802
1803    #[test]
1804    fn test_json_column_order_survives_report_serialization() {
1805        let file = write_file(&format!("[{UNSORTED_FIELDS}]"));
1806        let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1807
1808        let json = serde_json::to_string(&report).unwrap();
1809        let restored: Value = serde_json::from_str(&json).unwrap();
1810        let names: Vec<&str> = restored["column_profiles"]
1811            .as_array()
1812            .unwrap()
1813            .iter()
1814            .map(|profile| profile["name"].as_str().unwrap())
1815            .collect();
1816
1817        assert_eq!(names, UNSORTED_ORDER);
1818    }
1819
1820    #[test]
1821    fn test_analyze_json_backfills_columns_discovered_after_first_row() {
1822        let json = write_file(r#"[{"a":1},{"a":2,"b":3}]"#);
1823        let config = JsonParserConfig::default();
1824        let report = analyze_json_file(json.path(), &config).unwrap();
1825        let col_b = report
1826            .column_profiles
1827            .iter()
1828            .find(|profile| profile.name == "b")
1829            .unwrap();
1830
1831        assert_eq!(col_b.total_count, 2);
1832        assert_eq!(col_b.null_count, 1);
1833        assert_eq!(col_b.unique_count, Some(1));
1834    }
1835
1836    #[test]
1837    fn test_analyze_json_empty_array() {
1838        let json = write_file("[]");
1839        let config = JsonParserConfig::default();
1840        let report = analyze_json_file(json.path(), &config).unwrap();
1841        assert!(report.column_profiles.is_empty());
1842        // An empty dataset has nothing to assess: no score is fabricated.
1843        assert_eq!(report.quality_score(), None);
1844    }
1845
1846    #[test]
1847    fn test_analyze_json_malformed_returns_error() {
1848        let json = write_file("this is entirely invalid json");
1849        let config = JsonParserConfig::default();
1850        let err = analyze_json_file(json.path(), &config)
1851            .expect_err("malformed JSON should return an error");
1852
1853        let message = err.to_string().to_lowercase();
1854        assert!(message.contains("malformed") && message.contains("json"));
1855    }
1856
1857    #[test]
1858    fn test_analyze_json_file_detects_format() {
1859        let json_array = write_file(r#"[{"x":1}]"#);
1860        let config = JsonParserConfig::default();
1861        let report = analyze_json_file(json_array.path(), &config).unwrap();
1862        assert!(matches!(
1863            report.data_source,
1864            DataSource::File {
1865                format: FileFormat::Json,
1866                ..
1867            }
1868        ));
1869
1870        let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
1871        let report = analyze_json_file(jsonl.path(), &config).unwrap();
1872        assert!(matches!(
1873            report.data_source,
1874            DataSource::File {
1875                format: FileFormat::Jsonl,
1876                ..
1877            }
1878        ));
1879    }
1880
1881    #[test]
1882    fn test_bom_prefixed_file_preserves_source_size() {
1883        let data = b"\xEF\xBB\xBF[{\"x\":1}]";
1884        let json = write_bytes(data);
1885        let report = analyze_json_file(json.path(), &JsonParserConfig::json_document()).unwrap();
1886
1887        assert_eq!(report.execution.rows_processed, 1);
1888        assert!(matches!(
1889            report.data_source,
1890            DataSource::File {
1891                format: FileFormat::Json,
1892                size_bytes,
1893                ..
1894            } if size_bytes == data.len() as u64
1895        ));
1896    }
1897
1898    #[test]
1899    fn test_analyze_json_file_empty() {
1900        let json = write_file("");
1901        let config = JsonParserConfig::default();
1902        let report = analyze_json_file(json.path(), &config).unwrap();
1903        assert_eq!(report.execution.rows_processed, 0);
1904        assert!(report.column_profiles.is_empty());
1905    }
1906
1907    #[test]
1908    fn test_analyze_json_boolean_and_nested() {
1909        let json =
1910            write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
1911        let config = JsonParserConfig::default();
1912        let report = analyze_json_file(json.path(), &config).unwrap();
1913        let profiles = &report.column_profiles;
1914
1915        let flag = profiles
1916            .iter()
1917            .find(|profile| profile.name == "flag")
1918            .unwrap();
1919        assert_eq!(flag.total_count, 2);
1920
1921        let nested = profiles
1922            .iter()
1923            .find(|profile| profile.name == "nested")
1924            .unwrap();
1925        assert_eq!(nested.total_count, 2);
1926    }
1927
1928    #[test]
1929    fn test_single_root_object_compact_yields_one_row() {
1930        let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
1931        let cursor = Cursor::new(data.as_ref());
1932        let config = JsonParserConfig::default();
1933
1934        let (profiles, _stats, rows, malformed, _format) =
1935            analyze_json_from_reader(cursor, &config).unwrap();
1936        assert_eq!(rows, 1);
1937        assert_eq!(malformed, 0);
1938        assert_eq!(profiles.len(), 2);
1939
1940        let names: Vec<&str> = profiles
1941            .iter()
1942            .map(|profile| profile.name.as_str())
1943            .collect();
1944        assert!(names.contains(&"type"));
1945        assert!(names.contains(&"features"));
1946    }
1947
1948    #[test]
1949    fn test_jsonl_multi_object_still_works() {
1950        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1951        let cursor = Cursor::new(data.as_ref());
1952        let config = JsonParserConfig::default();
1953
1954        let (_profiles, _stats, rows, _malformed, format) =
1955            analyze_json_from_reader(cursor, &config).unwrap();
1956        assert_eq!(format, FileFormat::Jsonl);
1957        assert_eq!(rows, 3);
1958    }
1959
1960    #[test]
1961    fn test_single_root_object_via_analyze_json_file() {
1962        let file =
1963            write_json_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
1964        let config = JsonParserConfig::default();
1965        let report = analyze_json_file(file.path(), &config).unwrap();
1966
1967        assert_eq!(report.execution.rows_processed, 1);
1968        assert_eq!(report.column_profiles.len(), 2);
1969    }
1970
1971    #[test]
1972    fn test_single_root_object_pretty_printed_yields_one_row() {
1973        let data = br#"{
1974  "type": "FeatureCollection",
1975  "features": [
1976    {"id": 1},
1977    {"id": 2}
1978  ]
1979}"#;
1980        let cursor = Cursor::new(data.as_ref());
1981        // A reader has no file name, so the grammar cannot be inferred and is
1982        // named explicitly; `analyze_json_file` takes it from the extension.
1983        let config = JsonParserConfig::json_document();
1984
1985        let (profiles, _stats, rows, malformed, format) =
1986            analyze_json_from_reader(cursor, &config).unwrap();
1987        assert_eq!(format, FileFormat::Json);
1988        assert_eq!(rows, 1);
1989        assert_eq!(malformed, 0);
1990        assert_eq!(profiles.len(), 2);
1991    }
1992
1993    #[test]
1994    fn test_single_root_object_pretty_printed_via_analyze_json_file() {
1995        let file = write_json_file(
1996            "{\n  \"type\": \"FeatureCollection\",\n  \"features\": [\n    {\"id\": 1},\n    {\"id\": 2}\n  ]\n}\n",
1997        );
1998        let config = JsonParserConfig::default();
1999        let report = analyze_json_file(file.path(), &config).unwrap();
2000
2001        assert_eq!(report.execution.rows_processed, 1);
2002        assert_eq!(report.column_profiles.len(), 2);
2003    }
2004}