Skip to main content

dataprof_json/
lib.rs

1//! JSON and JSONL scanning and profiling for `dataprof`.
2//!
3//! This crate is an implementation detail of the `dataprof` facade, which
4//! re-exports [`JsonParserConfig`], [`analyze_json_from_reader`], and
5//! [`analyze_json_file`]. Depend on `dataprof` unless you need JSON support
6//! without the rest of the workspace.
7//!
8//! Two entry shapes are offered. [`scan_json_from_reader`] hands each record to
9//! a callback and reports what it read; [`analyze_json_from_reader`] and
10//! [`analyze_json_file`] build column profiles and a full report on top of it.
11//!
12//! ```
13//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
14//! use std::io::Cursor;
15//!
16//! use dataprof_json::{JsonParserConfig, analyze_json_from_reader};
17//!
18//! let jsonl = "{\"id\": 1, \"city\": \"Rome\"}\n{\"id\": 2, \"city\": \"Milan\"}\n";
19//! let (profiles, _stats, rows_read, malformed_lines, _format) =
20//!     analyze_json_from_reader(Cursor::new(jsonl), &JsonParserConfig::jsonl())?;
21//!
22//! assert_eq!(rows_read, 2);
23//! assert_eq!(malformed_lines, 0);
24//! assert_eq!(profiles[0].name, "id");
25//! # Ok(())
26//! # }
27//! ```
28
29use std::collections::{HashMap, HashSet};
30use std::io::BufRead;
31use std::path::Path;
32
33pub use dataprof_core::JsonErrorPolicy;
34
35#[doc(hidden)]
36pub mod record;
37use dataprof_core::{
38    AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat,
39    TruncationReason, Utf8BomReader,
40};
41use dataprof_runtime::{
42    ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
43};
44use record::{
45    JsonRecord, drain_to_end, json_document_error, json_value_kind, json_value_text,
46    malformed_array_error, non_object_record_error, object_record, read_json_record,
47    read_jsonl_line, skip_ascii_whitespace, wide_integer_record_error,
48};
49use serde_json::Value;
50
51/// Which grammar a JSON source is read with.
52///
53/// The two are deliberately distinct: whitespace is insignificant inside a
54/// standard JSON document and load-bearing in JSONL, so one input cannot be
55/// valid under both readings and mean the same thing. Callers pick with
56/// `format="json"` / `format="jsonl"`, or by file extension.
57#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58pub enum JsonFormat {
59    /// A standard JSON document, in either of the two shapes that carry
60    /// records: an array of objects (`[{...}, {...}]`), or a single object
61    /// (`{...}`) as one record. Values may span lines and be pretty-printed,
62    /// and exactly one document must fill the whole input.
63    Json,
64    /// JSON Lines — one record per physical line. A record may not span lines,
65    /// and a line may not hold more than one value.
66    Jsonl,
67}
68
69/// Configuration for JSON/JSONL parsing and scanning.
70///
71/// The default detects the grammar from the input and reads every record,
72/// skipping malformed ones. The two format constructors say which grammar to
73/// read with, which is what a caller who knows should do: detection cannot tell
74/// a JSON object document from a JSONL file, since both open with `{`.
75///
76/// # Examples
77///
78/// The same two records under each grammar:
79///
80/// ```
81/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
82/// use std::io::Cursor;
83///
84/// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
85///
86/// let document = r#"[{"id": 1}, {"id": 2}]"#;
87/// let lines = "{\"id\": 1}\n{\"id\": 2}\n";
88///
89/// let from_document = scan_json_from_reader(
90///     Cursor::new(document),
91///     &JsonParserConfig::json_document(),
92///     |_| {},
93/// )?;
94/// let from_lines =
95///     scan_json_from_reader(Cursor::new(lines), &JsonParserConfig::jsonl(), |_| {})?;
96///
97/// assert_eq!(from_document.rows_read, 2);
98/// assert_eq!(from_lines.rows_read, 2);
99/// # Ok(())
100/// # }
101/// ```
102#[derive(Debug, Clone, Default)]
103pub struct JsonParserConfig {
104    /// The grammar to read with. `None` detects it from the first byte, which
105    /// resolves `[` to [`JsonFormat::Json`] and everything else to
106    /// [`JsonFormat::Jsonl`] — a leading `{` opens both a JSON object document
107    /// and a JSONL file, so callers that know which they have (from the file
108    /// extension or an explicit `format=`) should say so rather than rely on
109    /// detection.
110    pub format: Option<JsonFormat>,
111    /// Maximum rows to process (None = all rows).
112    pub max_rows: Option<usize>,
113    /// How to react to a malformed record (default: skip).
114    pub error_policy: JsonErrorPolicy,
115}
116
117impl JsonParserConfig {
118    /// Set the maximum number of rows to process.
119    ///
120    /// The cap counts profileable records, not lines: blank lines and records
121    /// skipped as malformed do not consume it.
122    ///
123    /// A cap is only *truncation* when a record still remained once it was
124    /// reached, so a source holding exactly `max_rows` records reads as
125    /// complete. [`JsonScanSummary::truncated`] carries that distinction, and
126    /// [`analyze_json_file`] turns it into a [`TruncationReason::MaxRows`] on
127    /// the report.
128    ///
129    /// # Examples
130    ///
131    /// ```
132    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
133    /// use std::io::Cursor;
134    ///
135    /// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
136    ///
137    /// let lines = "{\"id\": 1}\n{\"id\": 2}\n{\"id\": 3}\n";
138    ///
139    /// // Two of three records: a third remained, so the scan was cut short.
140    /// let capped = JsonParserConfig::jsonl().with_max_rows(2);
141    /// let summary = scan_json_from_reader(Cursor::new(lines), &capped, |_| {})?;
142    /// assert_eq!(summary.rows_read, 2);
143    /// assert!(summary.truncated);
144    ///
145    /// // A cap the source never reaches past is a complete read, not a
146    /// // truncated one.
147    /// let exact = JsonParserConfig::jsonl().with_max_rows(3);
148    /// let summary = scan_json_from_reader(Cursor::new(lines), &exact, |_| {})?;
149    /// assert_eq!(summary.rows_read, 3);
150    /// assert!(!summary.truncated);
151    /// # Ok(())
152    /// # }
153    /// ```
154    pub fn with_max_rows(mut self, max_rows: usize) -> Self {
155        self.max_rows = Some(max_rows);
156        self
157    }
158
159    /// Set how malformed records are handled.
160    ///
161    /// Under the default [`JsonErrorPolicy::Skip`] a malformed record is
162    /// counted in [`JsonScanSummary::malformed_lines`] and scanning continues,
163    /// so a partial profile never looks like a clean one. Under
164    /// [`JsonErrorPolicy::Strict`] the first one fails the scan.
165    ///
166    /// # Examples
167    ///
168    /// ```
169    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
170    /// use std::io::Cursor;
171    ///
172    /// use dataprof_json::{JsonErrorPolicy, JsonParserConfig, scan_json_from_reader};
173    ///
174    /// let lines = "{\"id\": 1}\nnot json\n{\"id\": 2}\n";
175    ///
176    /// let skipping = JsonParserConfig::jsonl();
177    /// let summary = scan_json_from_reader(Cursor::new(lines), &skipping, |_| {})?;
178    /// assert_eq!(summary.rows_read, 2);
179    /// assert_eq!(summary.malformed_lines, 1);
180    ///
181    /// let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
182    /// assert!(scan_json_from_reader(Cursor::new(lines), &strict, |_| {}).is_err());
183    /// # Ok(())
184    /// # }
185    /// ```
186    pub fn with_error_policy(mut self, policy: JsonErrorPolicy) -> Self {
187        self.error_policy = policy;
188        self
189    }
190
191    /// Read as JSON Lines: one record per physical line.
192    pub fn jsonl() -> Self {
193        Self {
194            format: Some(JsonFormat::Jsonl),
195            ..Default::default()
196        }
197    }
198
199    /// Read as a standard JSON document: an array of records, or a single
200    /// object as one record.
201    pub fn json_document() -> Self {
202        Self {
203            format: Some(JsonFormat::Json),
204            ..Default::default()
205        }
206    }
207
208    /// Set the grammar explicitly.
209    pub fn with_format(mut self, format: JsonFormat) -> Self {
210        self.format = Some(format);
211        self
212    }
213}
214
215/// Borrowed JSON object callback payload.
216///
217/// The workspace enables `serde_json/preserve_order`, so iterating this map
218/// yields fields in the order they appear in the source document rather than
219/// alphabetically. That is what makes the column-ordering contract on
220/// [`analyze_json_from_reader`] possible.
221pub type JsonObject = serde_json::Map<String, Value>;
222
223/// Summary of a JSON/JSONL scan.
224#[derive(Debug, Clone, PartialEq, Eq)]
225#[non_exhaustive]
226pub struct JsonScanSummary {
227    pub rows_read: usize,
228    /// Records that were read but not profiled: malformed JSON, and valid JSON
229    /// values that are not objects and therefore carry no fields. Both are
230    /// counted so a tolerant scan never looks like a clean one.
231    pub malformed_lines: usize,
232    pub format: FileFormat,
233    /// The scan stopped at `max_rows` while records still remained. A source
234    /// holding exactly `max_rows` records was read in full, so this stays false.
235    pub truncated: bool,
236    /// Under the skip policy, the array scanner stopped at an element it could
237    /// not read past and left input after it unread. Whether that input held
238    /// records is unknown, so the source was not exhausted: reporting it as
239    /// read in full let a drained remainder pass as a complete profile (#1106).
240    pub unread_after_fault: bool,
241}
242
243/// Scan JSON or JSONL input and invoke `on_object` for each object record.
244///
245/// - **JSONL**: one record per physical line. A record may not span lines and a
246///   line may not hold more than one value, so a concatenation whose delimiter
247///   was lost is reported rather than read as several clean records. Blank lines
248///   are separators: neither records nor errors.
249/// - **JSON**: one standard document. An array of objects streams element by
250///   element without buffering the whole input; a single object is read whole
251///   and is one record, which is why it may be pretty-printed across lines.
252///
253/// # Record policy
254///
255/// Only JSON objects are profileable records — they are the only JSON value
256/// with named fields to turn into columns. A record that is valid JSON but not
257/// an object (a scalar, an array, `null`) is never silently discarded: under
258/// [`JsonErrorPolicy::Skip`] it is counted in
259/// [`JsonScanSummary::malformed_lines`] and scanning continues with the next
260/// record, and under [`JsonErrorPolicy::Strict`] the first one fails the scan.
261///
262/// # Examples
263///
264/// Reading JSONL, keeping one field per record:
265///
266/// ```
267/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
268/// use std::io::Cursor;
269///
270/// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
271///
272/// let lines = "{\"city\": \"Rome\", \"pop\": 2800000}\n\
273///              {\"city\": \"Milan\", \"pop\": 1400000}\n";
274///
275/// let mut cities = Vec::new();
276/// let summary = scan_json_from_reader(
277///     Cursor::new(lines),
278///     &JsonParserConfig::jsonl(),
279///     |object| {
280///         if let Some(city) = object.get("city").and_then(|value| value.as_str()) {
281///             cities.push(city.to_string());
282///         }
283///     },
284/// )?;
285///
286/// assert_eq!(cities, ["Rome", "Milan"]);
287/// assert_eq!(summary.rows_read, 2);
288/// assert!(!summary.truncated);
289/// # Ok(())
290/// # }
291/// ```
292///
293/// The same records as a JSON array. The array streams element by element, and
294/// a record may be pretty-printed across lines because whitespace carries no
295/// meaning under this grammar:
296///
297/// ```
298/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
299/// use std::io::Cursor;
300///
301/// use dataprof_json::{JsonParserConfig, scan_json_from_reader};
302///
303/// let document = r#"[
304///     {"city": "Rome", "pop": 2800000},
305///     {"city": "Milan", "pop": 1400000}
306/// ]"#;
307///
308/// let mut populations = Vec::new();
309/// let summary = scan_json_from_reader(
310///     Cursor::new(document),
311///     &JsonParserConfig::json_document(),
312///     |object| {
313///         // A field that is absent, or present but not a number, stays
314///         // `None`. Reading it as 0 instead would turn a missing value into
315///         // a real measurement, which is the one thing a profile must never
316///         // do.
317///         populations.push(object.get("pop").and_then(|value| value.as_u64()));
318///     },
319/// )?;
320///
321/// assert_eq!(populations, [Some(2_800_000), Some(1_400_000)]);
322/// assert_eq!(summary.rows_read, 2);
323/// assert_eq!(summary.malformed_lines, 0);
324/// # Ok(())
325/// # }
326/// ```
327pub fn scan_json_from_reader<R, F>(
328    reader: R,
329    config: &JsonParserConfig,
330    mut on_object: F,
331) -> Result<JsonScanSummary, DataProfilerError>
332where
333    R: BufRead,
334    F: FnMut(&JsonObject),
335{
336    let mut reader = Utf8BomReader::new(reader).map_err(DataProfilerError::from)?;
337    let format = match config.format {
338        Some(JsonFormat::Json) => FileFormat::Json,
339        Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
340        // A leading `{` is the first byte of both a JSON object document and a
341        // JSONL file, so detection can only resolve the array case and has to
342        // fall back to JSONL. Callers that know better pass `format`.
343        None => match consume_leading_whitespace(&mut reader)? {
344            Some(b'[') => FileFormat::Json,
345            _ => FileFormat::Jsonl,
346        },
347    };
348
349    let mut rows_read = 0;
350    let mut malformed_lines = 0;
351    let mut unread_after_fault = false;
352    let mut truncated = false;
353
354    match format {
355        // JSONL is line-delimited: exactly one record per non-blank physical
356        // line. Reading it as a whitespace-delimited stream of values instead
357        // made `{"a":1}{"b":2}` — a concatenation with the delimiter lost —
358        // profile as two clean rows, which is precisely the corruption a
359        // profiler exists to surface.
360        FileFormat::Jsonl => {
361            let mut line = String::new();
362            let mut line_number = 0;
363            loop {
364                if let Some(max) = config.max_rows
365                    && rows_read >= max
366                {
367                    // Reaching the cap is not truncation unless a record
368                    // actually remains: a file with exactly `max_rows` records
369                    // was read in full. Blank lines are not records.
370                    truncated = next_record_line(&mut reader, &mut line, &mut line_number)?;
371                    break;
372                }
373
374                if !next_record_line(&mut reader, &mut line, &mut line_number)? {
375                    break;
376                }
377
378                match read_jsonl_line(&line) {
379                    JsonRecord::Object(obj) => {
380                        on_object(&obj);
381                        rows_read += 1;
382                    }
383                    JsonRecord::NonObject(kind) => {
384                        if config.error_policy == JsonErrorPolicy::Strict {
385                            return Err(non_object_record_error(
386                                kind,
387                                rows_read + malformed_lines + 1,
388                            ));
389                        }
390                        malformed_lines += 1;
391                    }
392                    JsonRecord::Malformed(err) => {
393                        if config.error_policy == JsonErrorPolicy::Strict {
394                            return Err(malformed_jsonl_record_error(line_number, err));
395                        }
396                        malformed_lines += 1;
397                    }
398                    JsonRecord::WideInteger => {
399                        if config.error_policy == JsonErrorPolicy::Strict {
400                            return Err(wide_integer_record_error(&format!(
401                                "on line {line_number}"
402                            )));
403                        }
404                        malformed_lines += 1;
405                    }
406                }
407            }
408        }
409        FileFormat::Json => {
410            // Which of the two record-carrying shapes this document is. The
411            // first non-whitespace byte decides, and is left unconsumed so the
412            // object branch still sees the value it opens.
413            let found_array = consume_leading_whitespace(&mut reader)? == Some(b'[');
414
415            if !found_array {
416                // The other shape a standard JSON document takes: a single
417                // object, which is one record. It may be pretty-printed across
418                // lines, which is why this is not the JSONL grammar.
419                let (object_rows, object_malformed, object_truncated) =
420                    scan_single_json_object(&mut reader, config, &mut on_object)?;
421                rows_read += object_rows;
422                malformed_lines += object_malformed;
423                truncated |= object_truncated;
424            } else {
425                // The array loop starts after the opening bracket.
426                consume_peeked(&mut reader)?;
427
428                let mut expect_value = true;
429                let mut allow_end = true;
430                let mut array_closed = false;
431                let mut drain_remainder = false;
432
433                loop {
434                    let Some(next) = consume_leading_whitespace(&mut reader)? else {
435                        if config.error_policy == JsonErrorPolicy::Strict {
436                            return Err(malformed_array_error(
437                                "unexpected end of input before closing ']'",
438                            ));
439                        }
440                        malformed_lines += 1;
441                        drain_remainder = true;
442                        break;
443                    };
444
445                    if expect_value {
446                        if next == b']' {
447                            if !allow_end {
448                                if config.error_policy == JsonErrorPolicy::Strict {
449                                    return Err(malformed_array_error(
450                                        "trailing comma before closing ']'",
451                                    ));
452                                }
453                                malformed_lines += 1;
454                                drain_remainder = true;
455                                break;
456                            }
457                            consume_peeked(&mut reader)?;
458                            array_closed = true;
459                            break;
460                        }
461
462                        if let Some(max) = config.max_rows
463                            && rows_read >= max
464                        {
465                            // A value remains unread. Do not validate the unread
466                            // suffix of an intentionally bounded scan.
467                            truncated = true;
468                            break;
469                        }
470
471                        if next == b',' {
472                            if config.error_policy == JsonErrorPolicy::Strict {
473                                return Err(malformed_array_error(
474                                    "unexpected comma where an array value was required",
475                                ));
476                            }
477                            malformed_lines += 1;
478                            drain_remainder = true;
479                            break;
480                        }
481
482                        match read_json_record(&mut reader, next)? {
483                            JsonRecord::Object(obj) => {
484                                on_object(&obj);
485                                rows_read += 1;
486                            }
487                            JsonRecord::NonObject(kind) => {
488                                if config.error_policy == JsonErrorPolicy::Strict {
489                                    return Err(non_object_record_error(
490                                        kind,
491                                        rows_read + malformed_lines + 1,
492                                    ));
493                                }
494                                // The element was consumed in full, so the array
495                                // grammar is still intact and the objects after
496                                // it are still profileable.
497                                malformed_lines += 1;
498                            }
499                            JsonRecord::Malformed(err) => {
500                                if config.error_policy == JsonErrorPolicy::Strict {
501                                    return Err(malformed_record_error(err));
502                                }
503                                malformed_lines += 1;
504                                drain_remainder = true;
505                                break;
506                            }
507                            JsonRecord::WideInteger => {
508                                if config.error_policy == JsonErrorPolicy::Strict {
509                                    return Err(wide_integer_record_error(&format!(
510                                        "at position {}",
511                                        rows_read + malformed_lines + 1
512                                    )));
513                                }
514                                // Read in full: the array grammar is intact.
515                                malformed_lines += 1;
516                            }
517                        }
518                        expect_value = false;
519                    } else {
520                        match next {
521                            b',' => {
522                                consume_peeked(&mut reader)?;
523                                expect_value = true;
524                                allow_end = false;
525                            }
526                            b']' => {
527                                consume_peeked(&mut reader)?;
528                                array_closed = true;
529                                break;
530                            }
531                            _ => {
532                                if config.error_policy == JsonErrorPolicy::Strict {
533                                    return Err(malformed_array_error(
534                                        "expected ',' or ']' after an array value",
535                                    ));
536                                }
537                                malformed_lines += 1;
538                                drain_remainder = true;
539                                break;
540                            }
541                        }
542                    }
543                }
544
545                if drain_remainder {
546                    // Something left unread after the fault means the read
547                    // stopped short, whatever that input held (#1106).
548                    unread_after_fault = consume_leading_whitespace(&mut reader)?.is_some();
549                    drain_to_end(&mut reader)?;
550                } else if array_closed && consume_leading_whitespace(&mut reader)?.is_some() {
551                    if config.error_policy == JsonErrorPolicy::Strict {
552                        return Err(malformed_array_error(
553                            "non-whitespace content follows the closing ']'",
554                        ));
555                    }
556                    malformed_lines += 1;
557                    drain_to_end(&mut reader)?;
558                }
559            }
560        }
561        _ => unreachable!("json scanner only returns json or jsonl formats"),
562    }
563
564    Ok(JsonScanSummary {
565        rows_read,
566        malformed_lines,
567        format,
568        truncated,
569        unread_after_fault,
570    })
571}
572
573/// Read the next line that holds a record into `line`, skipping blank ones.
574///
575/// Returns whether a record line was found. Blank lines are separators, not
576/// records: they are skipped without counting as rows or as errors. `line_number`
577/// counts every physical line read, blank ones included, so a diagnostic points
578/// at the line the user would open the file to.
579fn next_record_line<R: BufRead>(
580    reader: &mut R,
581    line: &mut String,
582    line_number: &mut usize,
583) -> Result<bool, DataProfilerError> {
584    loop {
585        line.clear();
586        let read = reader.read_line(line).map_err(DataProfilerError::from)?;
587        if read == 0 {
588            return Ok(false);
589        }
590        *line_number += 1;
591        if !line.trim().is_empty() {
592            return Ok(true);
593        }
594    }
595}
596
597/// Read a whole-input single JSON object as one record.
598///
599/// Returns `(rows_read, malformed, truncated)`. Anything that is not exactly one object
600/// filling the input is an error under [`JsonErrorPolicy::Strict`] and a counted
601/// malformed record otherwise, so a truncated or concatenated document never
602/// profiles as clean.
603fn scan_single_json_object<R, F>(
604    reader: &mut R,
605    config: &JsonParserConfig,
606    on_object: &mut F,
607) -> Result<(usize, usize, bool), DataProfilerError>
608where
609    R: BufRead,
610    F: FnMut(&JsonObject),
611{
612    let mut text = String::new();
613    reader
614        .read_to_string(&mut text)
615        .map_err(DataProfilerError::from)?;
616
617    if text.trim().is_empty() {
618        return Ok((0, 0, false));
619    }
620
621    // A row cap of zero asks for no records, so the document is not read. The
622    // record it holds is left unread, which is a truncation and has to be
623    // reported as one — an unread record and an empty source must not produce
624    // the same report.
625    if config.max_rows == Some(0) {
626        return Ok((0, 0, true));
627    }
628
629    match serde_json::from_str::<Value>(text.trim()) {
630        Ok(Value::Object(obj)) => match object_record(obj, text.as_bytes()) {
631            JsonRecord::Object(obj) => {
632                on_object(&obj);
633                Ok((1, 0, false))
634            }
635            _ => {
636                if config.error_policy == JsonErrorPolicy::Strict {
637                    return Err(wide_integer_record_error("at position 1"));
638                }
639                Ok((0, 1, false))
640            }
641        },
642        Ok(value) => {
643            if config.error_policy == JsonErrorPolicy::Strict {
644                return Err(non_object_record_error(json_value_kind(&value), 1));
645            }
646            Ok((0, 1, false))
647        }
648        Err(err) => {
649            if config.error_policy == JsonErrorPolicy::Strict {
650                return Err(json_document_error(err));
651            }
652            Ok((0, 1, false))
653        }
654    }
655}
656
657/// Consume the byte [`consume_leading_whitespace`] just reported, along with the
658/// whitespace it looked past.
659///
660/// That function only peeks — it leaves the whitespace queued so serde keeps its
661/// line and column context — so consuming one byte would eat a space rather than
662/// the delimiter. Getting this wrong made a pretty-printed JSON array report a
663/// phantom malformed record, because the `]` survived the consume and then
664/// looked like content after the closing bracket.
665fn consume_peeked<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
666    skip_ascii_whitespace(reader)?;
667    reader.consume(1);
668    Ok(())
669}
670
671/// Build a strict-mode parse error from a serde failure. The serde message
672/// carries line/column context but never the record contents, so it is safe to
673/// surface directly.
674fn malformed_record_error(err: serde_json::Error) -> DataProfilerError {
675    let message = format!("malformed JSON record: {err}");
676    DataProfilerError::json_parsing_with_source(message, err)
677}
678
679/// Build a strict-mode parse error for a JSONL record, located by its physical
680/// line.
681///
682/// Each line is parsed on its own, so the decoder's own line number is always 1
683/// and would be misleading; the column it reports is within the line and so is
684/// also the column in the file. The record's text is never included. Matches the
685/// phrasing the Python bytes reader uses for the same failure.
686fn malformed_jsonl_record_error(line: usize, err: serde_json::Error) -> DataProfilerError {
687    let message = format!(
688        "malformed JSON record on line {line}, column {}: a JSONL record must be one complete JSON value on one line",
689        err.column()
690    );
691    DataProfilerError::json_parsing_with_source(message, err)
692}
693
694fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
695    loop {
696        let mut bytes_to_consume = 0;
697        let first_non_whitespace = {
698            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
699            if buf.is_empty() {
700                return Ok(None);
701            }
702
703            let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
704            if first_non_whitespace.is_none() {
705                bytes_to_consume = buf.len();
706            }
707            first_non_whitespace
708        };
709
710        if first_non_whitespace.is_some() {
711            return Ok(first_non_whitespace);
712        }
713
714        reader.consume(bytes_to_consume);
715    }
716}
717
718/// Feed a JSON object's fields into a [`StreamingColumnCollection`].
719///
720/// Columns are registered in the order the fields appear in each record, so the
721/// resulting column order is the first record's field order with later-only
722/// fields appended where they were first seen.
723///
724/// `container_counts` tallies, per column, the values that were an object or an
725/// array. It is what [`profile_builder::mark_container_columns`] later reads
726/// to decide which columns hold nothing but containers.
727fn feed_json_object(
728    obj: &JsonObject,
729    prior_rows: usize,
730    known_columns: &mut Vec<String>,
731    known_columns_set: &mut HashSet<String>,
732    column_stats: &mut StreamingColumnCollection,
733    container_counts: &mut HashMap<String, usize>,
734) {
735    for (key, value) in obj {
736        if known_columns_set.insert(key.clone()) {
737            known_columns.push(key.clone());
738            column_stats.init_column_with_missing(key, prior_rows);
739        }
740        if matches!(value, Value::Array(_) | Value::Object(_)) {
741            *container_counts.entry(key.clone()).or_default() += 1;
742        }
743    }
744
745    let values: Vec<String> = known_columns
746        .iter()
747        .map(|column| {
748            // decode-audit: no-data — a key absent from this object is a
749            // missing field, and "" is the profiler's textual null.
750            obj.get(column).map(json_value_text).unwrap_or_default()
751        })
752        .collect();
753    let containers: Vec<usize> = known_columns
754        .iter()
755        .enumerate()
756        .filter(|(_, column)| matches!(obj.get(*column), Some(Value::Array(_) | Value::Object(_))))
757        .map(|(position, _)| position)
758        .collect();
759
760    column_stats.process_record_with_containers(known_columns, values, &containers);
761}
762
763/// Analyze JSON/JSONL data from a buffered reader using streaming statistics.
764///
765/// Returns `(column_profiles, streaming_stats, rows_read, malformed_lines, detected_format)`.
766///
767/// # Column order
768///
769/// Columns are returned in source order — the first record's field order, with
770/// fields that only appear in later records appended in first-seen order. This
771/// matches CSV (header order) and Parquet (schema order), so the same logical
772/// dataset profiles to the same column order in every format.
773///
774/// # Examples
775///
776/// The second record lists its fields in a different order and adds one. The
777/// first record still decides the order, and the new field is appended:
778///
779/// ```
780/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
781/// use std::io::Cursor;
782///
783/// use dataprof_json::{JsonParserConfig, analyze_json_from_reader};
784///
785/// let lines = "{\"id\": 1, \"city\": \"Rome\"}\n\
786///              {\"city\": \"Milan\", \"id\": 2, \"region\": \"Lombardy\"}\n";
787///
788/// let (profiles, _stats, rows_read, malformed_lines, format) =
789///     analyze_json_from_reader(Cursor::new(lines), &JsonParserConfig::jsonl())?;
790///
791/// let names: Vec<&str> = profiles.iter().map(|profile| profile.name.as_str()).collect();
792/// assert_eq!(names, ["id", "city", "region"]);
793/// assert_eq!(rows_read, 2);
794/// assert_eq!(malformed_lines, 0);
795/// assert_eq!(format, dataprof_core::FileFormat::Jsonl);
796///
797/// // The first record has no `region`, which counts as a missing value.
798/// let region = &profiles[2];
799/// assert_eq!(region.total_count, 2);
800/// assert_eq!(region.null_count, 1);
801/// # Ok(())
802/// # }
803/// ```
804pub fn analyze_json_from_reader<R: BufRead>(
805    reader: R,
806    config: &JsonParserConfig,
807) -> Result<
808    (
809        Vec<ColumnProfile>,
810        StreamingColumnCollection,
811        usize,
812        usize,
813        FileFormat,
814    ),
815    DataProfilerError,
816> {
817    let options = AnalysisOptions::default();
818    let (profiles, stats, rows_read, malformed_lines, format, _truncated, _unread) =
819        analyze_json_from_reader_full(reader, config, &options)?;
820    Ok((profiles, stats, rows_read, malformed_lines, format))
821}
822
823/// As [`analyze_json_from_reader`], but with the caller's analysis options, and
824/// also reports whether the scan stopped early at a row cap, and whether it
825/// left input unread after a fault. Kept private so the public tuple stays
826/// stable.
827#[allow(clippy::type_complexity)]
828fn analyze_json_from_reader_full<R: BufRead>(
829    reader: R,
830    config: &JsonParserConfig,
831    options: &AnalysisOptions,
832) -> Result<
833    (
834        Vec<ColumnProfile>,
835        StreamingColumnCollection,
836        usize,
837        usize,
838        FileFormat,
839        bool,
840        bool,
841    ),
842    DataProfilerError,
843> {
844    let semantic_hints = options.semantic_hints();
845    let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
846    let mut known_columns = Vec::new();
847    let mut known_columns_set = HashSet::new();
848    let mut container_counts = HashMap::new();
849    let mut rows_seen = 0;
850
851    let summary = scan_json_from_reader(reader, config, |obj| {
852        feed_json_object(
853            obj,
854            rows_seen,
855            &mut known_columns,
856            &mut known_columns_set,
857            &mut column_stats,
858            &mut container_counts,
859        );
860        rows_seen += 1;
861    })?;
862
863    if let Some(indices) = options.column_indices(&column_stats.column_names())? {
864        let available = column_stats.column_names();
865        let selected = indices
866            .into_iter()
867            .map(|index| available[index].clone())
868            .collect::<Vec<_>>();
869        column_stats.retain_columns(&selected);
870    }
871
872    let profiles = profile_builder::mark_container_columns(
873        profile_builder::profiles_from_streaming_with_hints(
874            &column_stats,
875            !options.include_statistics(),
876            !options.include_patterns(),
877            options.locale(),
878            semantic_hints,
879        ),
880        &container_counts,
881    );
882
883    Ok((
884        profiles,
885        column_stats,
886        summary.rows_read,
887        summary.malformed_lines,
888        summary.format,
889        summary.truncated,
890        summary.unread_after_fault,
891    ))
892}
893
894/// Analyze a JSON or JSONL file, returning a full [`ProfileReport`].
895///
896/// With no explicit [`JsonParserConfig::format`] the file extension decides:
897/// `.json` reads as a standard document, `.jsonl` and `.ndjson` as JSON Lines,
898/// and any other name falls back to sniffing the first byte.
899///
900/// # Examples
901///
902/// ```
903/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
904/// use std::io::Write;
905///
906/// use dataprof_json::{JsonParserConfig, analyze_json_file};
907///
908/// let mut file = tempfile::NamedTempFile::with_suffix(".jsonl")?;
909/// writeln!(file, "{{\"id\": 1, \"score\": 9.5}}")?;
910/// writeln!(file, "{{\"id\": 2, \"score\": 7.0}}")?;
911/// file.flush()?;
912///
913/// let report = analyze_json_file(file.path(), &JsonParserConfig::default())?;
914///
915/// assert_eq!(report.execution.rows_processed, 2);
916/// assert_eq!(report.execution.columns_detected, 2);
917/// assert!(report.execution.truncation_reason.is_none());
918/// # Ok(())
919/// # }
920/// ```
921pub fn analyze_json_file(
922    file_path: &Path,
923    config: &JsonParserConfig,
924) -> Result<ProfileReport, DataProfilerError> {
925    analyze_json_file_with_options(file_path, config, &AnalysisOptions::default())
926}
927
928/// Analyze a JSON or JSONL file, honouring the caller's full analysis selection.
929///
930/// This is the entry point that carries metric packs and locale as well as
931/// dimensions and hints, so a JSON profile reports exactly the analysis the
932/// caller asked for — the same selection the CSV engines apply.
933pub fn analyze_json_file_with_options(
934    file_path: &Path,
935    config: &JsonParserConfig,
936    options: &AnalysisOptions,
937) -> Result<ProfileReport, DataProfilerError> {
938    // An explicit grammar wins; otherwise the file name gets a say before
939    // content sniffing, which cannot tell a JSON object from a JSONL stream.
940    let mut config = config.clone();
941    if config.format.is_none() {
942        config.format = grammar_for_extension(file_path);
943    }
944    let config = &config;
945    let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
946    dataprof_core::binary_input::reject_binary_input(file_path)?;
947    let start = std::time::Instant::now();
948
949    let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
950    let buf_reader = std::io::BufReader::new(file);
951
952    let (
953        column_profiles,
954        column_stats,
955        rows_read,
956        malformed_lines,
957        format,
958        truncated,
959        unread_after_fault,
960    ) = analyze_json_from_reader_full(buf_reader, config, options)?;
961    let read_as_json_document = format == FileFormat::Json;
962
963    let file_source = DataSource::File {
964        path: file_path.display().to_string(),
965        format,
966        size_bytes: metadata.len(),
967        modified_at: None,
968        parquet_metadata: None,
969    };
970
971    if rows_read == 0 {
972        if malformed_lines > 0 {
973            // Nothing parsed under the JSON grammar most often means the source
974            // is JSONL, so say so here too: under the default skip policy this
975            // is the only error the caller sees.
976            let hint = if read_as_json_document {
977                ". A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
978            } else {
979                ""
980            };
981            // Aggregate over many discarded records: no single decoder error
982            // is the cause, so none is retained.
983            return Err(DataProfilerError::json_parsing_error(&format!(
984                "No valid JSON records found in file \
985                 (every record was malformed, not a JSON object, or held an integer outside the 64-bit range){hint}"
986            )));
987        }
988        // A row cap that stopped the scan before the first record still has to
989        // be disclosed. Without this the report reads exactly like a complete
990        // profile of an empty source, which is the one thing a truncated scan
991        // must never be mistaken for.
992        let mut execution =
993            ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("json");
994        if truncated && let Some(max) = config.max_rows {
995            execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
996        }
997        return Ok(ReportAssembler::new(file_source, execution)
998            .columns(column_profiles)
999            .with_quality_data(HashMap::new())
1000            .with_analysis_options(options)
1001            .build());
1002    }
1003
1004    let sample_columns = profile_builder::quality_check_samples(&column_stats);
1005    let scan_time_ms = start.elapsed().as_millis();
1006    let num_columns = column_profiles.len();
1007
1008    let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms)
1009        .with_engine("json")
1010        // Tolerant scans surface skipped malformed records here so callers can
1011        // tell a partial profile from a clean one.
1012        .with_error_count(malformed_lines);
1013    // `truncated` is set only when a record still remained at the cap, so a source
1014    // holding exactly `max_rows` records is reported as fully read, not cut short.
1015    if truncated && let Some(max) = config.max_rows {
1016        execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1017    }
1018    if unread_after_fault {
1019        execution = execution.with_source_exhausted(false);
1020    }
1021
1022    Ok(ReportAssembler::new(file_source, execution)
1023        .columns(column_profiles)
1024        .with_quality_data(sample_columns)
1025        .with_row_duplicates(column_stats.row_duplicate_summary())
1026        .with_row_completeness(column_stats.row_completeness_summary())
1027        .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
1028        .with_analysis_options(options)
1029        .build())
1030}
1031
1032/// The grammar a file name advertises, if it advertises one.
1033///
1034/// Content sniffing resolves a leading `[` on its own, but not a leading `{`:
1035/// that is the first byte of both a JSON object document and a JSONL file. The
1036/// extension is the evidence that separates them, and a `.json` file holding one
1037/// pretty-printed object is an ordinary thing to profile. `None` means the name
1038/// says nothing and the content decides.
1039fn grammar_for_extension(file_path: &Path) -> Option<JsonFormat> {
1040    let extension = file_path.extension().and_then(|ext| ext.to_str())?;
1041    if extension.eq_ignore_ascii_case("json") {
1042        Some(JsonFormat::Json)
1043    } else if extension.eq_ignore_ascii_case("jsonl") || extension.eq_ignore_ascii_case("ndjson") {
1044        Some(JsonFormat::Jsonl)
1045    } else {
1046        None
1047    }
1048}
1049
1050fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
1051    if error.kind() == std::io::ErrorKind::NotFound {
1052        DataProfilerError::FileNotFound {
1053            path: file_path.display().to_string(),
1054        }
1055    } else {
1056        DataProfilerError::from(error)
1057    }
1058}
1059
1060#[cfg(test)]
1061mod tests {
1062    use super::*;
1063    use std::io::{Cursor, Write};
1064    use tempfile::NamedTempFile;
1065
1066    /// A temp file with no extension, so `analyze_json_file` falls back to the
1067    /// JSONL grammar the way an unnamed stream does.
1068    fn write_file(content: &str) -> NamedTempFile {
1069        let mut file = NamedTempFile::new().unwrap();
1070        write!(file, "{}", content).unwrap();
1071        file.flush().unwrap();
1072        file
1073    }
1074
1075    /// A temp file named `.json`, so `analyze_json_file` reads it as a standard
1076    /// JSON document — the grammar the extension advertises.
1077    fn write_json_file(content: &str) -> NamedTempFile {
1078        let mut file = NamedTempFile::with_suffix(".json").unwrap();
1079        write!(file, "{}", content).unwrap();
1080        file.flush().unwrap();
1081        file
1082    }
1083
1084    fn write_bytes(content: &[u8]) -> NamedTempFile {
1085        let mut file = NamedTempFile::new().unwrap();
1086        file.write_all(content).unwrap();
1087        file.flush().unwrap();
1088        file
1089    }
1090
1091    #[test]
1092    fn test_scan_jsonl_detects_format_and_rows() {
1093        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1094        let mut rows = 0;
1095
1096        let summary = scan_json_from_reader(
1097            Cursor::new(data.as_ref()),
1098            &JsonParserConfig::default(),
1099            |_| {
1100                rows += 1;
1101            },
1102        )
1103        .unwrap();
1104
1105        assert_eq!(summary.format, FileFormat::Jsonl);
1106        assert_eq!(summary.rows_read, 3);
1107        assert_eq!(summary.malformed_lines, 0);
1108        assert_eq!(rows, 3);
1109    }
1110
1111    #[test]
1112    fn test_scan_json_array_detects_format_and_rows() {
1113        let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
1114        let mut keys = Vec::new();
1115
1116        let summary = scan_json_from_reader(
1117            Cursor::new(data.as_ref()),
1118            &JsonParserConfig::default(),
1119            |obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
1120        )
1121        .unwrap();
1122
1123        assert_eq!(summary.format, FileFormat::Json);
1124        assert_eq!(summary.rows_read, 2);
1125        assert_eq!(keys.len(), 2);
1126        assert_eq!(keys[0], vec!["name".to_string()]);
1127    }
1128
1129    #[test]
1130    fn test_scan_accepts_one_leading_utf8_bom_for_json_and_jsonl() {
1131        for (data, config, expected_format) in [
1132            (
1133                b"\xEF\xBB\xBF[{\"x\":1},{\"x\":2}]".as_slice(),
1134                JsonParserConfig::default(),
1135                FileFormat::Json,
1136            ),
1137            (
1138                b"\xEF\xBB\xBF{\"x\":1}\n{\"x\":2}\n".as_slice(),
1139                JsonParserConfig::default(),
1140                FileFormat::Jsonl,
1141            ),
1142        ] {
1143            let mut rows = 0;
1144            let summary = scan_json_from_reader(Cursor::new(data), &config, |_| rows += 1)
1145                .expect("leading UTF-8 BOM should be accepted");
1146
1147            assert_eq!(rows, 2);
1148            assert_eq!(summary.rows_read, 2);
1149            assert_eq!(summary.malformed_lines, 0);
1150            assert_eq!(summary.format, expected_format);
1151        }
1152    }
1153
1154    #[test]
1155    fn test_scan_does_not_strip_nonleading_or_second_utf8_bom() {
1156        let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1157
1158        for data in [
1159            b" \xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1160            b"\xEF\xBB\xBF\xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1161            b"{\"x\":1}\n\xEF\xBB\xBF{\"x\":2}\n".as_slice(),
1162        ] {
1163            scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1164                .expect_err("only the BOM at absolute byte offset zero may be stripped");
1165        }
1166    }
1167
1168    #[test]
1169    fn test_bom_only_input_matches_empty_input_policy() {
1170        for config in [JsonParserConfig::default(), JsonParserConfig::jsonl()] {
1171            let empty = scan_json_from_reader(Cursor::new(b""), &config, |_| {}).unwrap();
1172            let bom =
1173                scan_json_from_reader(Cursor::new(b"\xEF\xBB\xBF".as_slice()), &config, |_| {})
1174                    .unwrap();
1175            assert_eq!(bom, empty);
1176        }
1177    }
1178
1179    #[test]
1180    fn test_json_array_container_grammar_obeys_error_policy() {
1181        let cases = [
1182            (br#"[{"x":1}"#.as_ref(), 1, "missing closing bracket"),
1183            (br#"[{"x":1},"#.as_ref(), 1, "comma followed by EOF"),
1184            (br#"[{"x":1} {"x":2}]"#.as_ref(), 1, "missing comma"),
1185            (br#"[{"x":1},,{"x":2}]"#.as_ref(), 1, "doubled comma"),
1186            (br#"[,{"x":1}]"#.as_ref(), 0, "leading comma"),
1187            (br#"[{"x":1},]"#.as_ref(), 1, "trailing comma"),
1188            (br#"[{"x":1}] trailing"#.as_ref(), 1, "trailing content"),
1189            (
1190                br#"[{"x":1}][{"x":2}]"#.as_ref(),
1191                1,
1192                "second top-level value",
1193            ),
1194        ];
1195
1196        for (data, expected_rows, case) in cases {
1197            let skip = JsonParserConfig::json_document();
1198            let summary =
1199                scan_json_from_reader(Cursor::new(data), &skip, |_| {}).expect("tolerant scan");
1200            assert_eq!(summary.rows_read, expected_rows, "{case}");
1201            assert_eq!(summary.malformed_lines, 1, "{case}");
1202
1203            let strict =
1204                JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict);
1205            let err = scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1206                .expect_err("strict mode must reject invalid array grammar");
1207            assert!(
1208                err.to_string()
1209                    .to_lowercase()
1210                    .contains("malformed json array"),
1211                "{case}: {err}"
1212            );
1213        }
1214    }
1215
1216    #[test]
1217    fn test_json_array_max_rows_does_not_validate_unread_values() {
1218        let data = br#"[{"x":1},not-valid-json"#;
1219        let config = JsonParserConfig::json_document().with_max_rows(1);
1220        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1221
1222        assert_eq!(summary.rows_read, 1);
1223        assert_eq!(summary.malformed_lines, 0);
1224        assert!(summary.truncated);
1225    }
1226
1227    #[test]
1228    fn test_scan_respects_max_rows() {
1229        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
1230        let config = JsonParserConfig::default().with_max_rows(2);
1231        let mut rows = 0;
1232
1233        let summary =
1234            scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
1235
1236        assert_eq!(summary.rows_read, 2);
1237        assert_eq!(rows, 2);
1238    }
1239
1240    #[test]
1241    fn test_scan_skips_malformed_jsonl_lines() {
1242        let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
1243        let mut rows = 0;
1244
1245        let summary = scan_json_from_reader(
1246            Cursor::new(data.as_ref()),
1247            &JsonParserConfig::jsonl(),
1248            |_| {
1249                rows += 1;
1250            },
1251        )
1252        .unwrap();
1253
1254        assert_eq!(summary.format, FileFormat::Jsonl);
1255        assert_eq!(summary.rows_read, 2);
1256        assert_eq!(summary.malformed_lines, 1);
1257        assert_eq!(rows, 2);
1258    }
1259
1260    /// A pretty-printed root object is one record under the JSON grammar and a
1261    /// malformed record under JSONL (#486). It cannot be both: whitespace is
1262    /// insignificant in a JSON document and is the record delimiter in JSONL, so
1263    /// reading a multi-line object as JSONL would mean each of its lines is a
1264    /// record — none of which is valid JSON on its own.
1265    #[test]
1266    fn test_pretty_printed_root_object_is_one_row_under_the_json_grammar() {
1267        let data = br#"{
1268    "type": "FeatureCollection",
1269    "features": [
1270        {"id": 1},
1271        {"id": 2}
1272  ]
1273}"#;
1274        let mut object_field_counts = Vec::new();
1275
1276        let summary = scan_json_from_reader(
1277            Cursor::new(data.as_ref()),
1278            &JsonParserConfig::json_document(),
1279            |obj| object_field_counts.push(obj.len()),
1280        )
1281        .unwrap();
1282
1283        assert_eq!(summary.format, FileFormat::Json);
1284        assert_eq!(summary.rows_read, 1);
1285        assert_eq!(summary.malformed_lines, 0);
1286        assert_eq!(object_field_counts, vec![2]);
1287    }
1288
1289    #[test]
1290    fn test_pretty_printed_root_object_is_not_valid_jsonl() {
1291        let data = br#"{
1292    "type": "FeatureCollection"
1293}"#;
1294        let err = scan_json_from_reader(
1295            Cursor::new(data.as_ref()),
1296            &JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict),
1297            |_| {},
1298        )
1299        .expect_err("a record spanning lines is not JSONL");
1300
1301        assert!(
1302            err.to_string().contains("malformed JSON record"),
1303            "{err}, expected the malformed category"
1304        );
1305    }
1306
1307    /// Every JSON value that is not an object, so none of them can be a row.
1308    /// Numbers are listed twice on purpose: they are the one value serde reads
1309    /// from a reader by peeking one byte past the end.
1310    const NON_OBJECT_VALUES: &[(&str, &str)] = &[
1311        ("null", "null"),
1312        ("boolean", "true"),
1313        ("number", "42"),
1314        ("number", "-1.5e3"),
1315        ("string", r#""text""#),
1316        ("array", "[1, 2]"),
1317    ];
1318
1319    #[test]
1320    fn test_jsonl_counts_non_object_records_and_keeps_scanning() {
1321        for (kind, value) in NON_OBJECT_VALUES {
1322            let data = format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n");
1323            let mut rows = 0;
1324
1325            let summary = scan_json_from_reader(
1326                Cursor::new(data.as_bytes()),
1327                &JsonParserConfig::jsonl(),
1328                |_| rows += 1,
1329            )
1330            .unwrap();
1331
1332            assert_eq!(summary.rows_read, 2, "{kind}: record after it was lost");
1333            assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1334            assert_eq!(rows, 2, "{kind}");
1335        }
1336    }
1337
1338    #[test]
1339    fn test_json_array_counts_non_object_elements_and_keeps_scanning() {
1340        for (kind, value) in NON_OBJECT_VALUES {
1341            let data = format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]");
1342            let mut rows = 0;
1343
1344            let summary = scan_json_from_reader(
1345                Cursor::new(data.as_bytes()),
1346                &JsonParserConfig::json_document(),
1347                |_| rows += 1,
1348            )
1349            .unwrap();
1350
1351            assert_eq!(summary.rows_read, 2, "{kind}: element after it was lost");
1352            assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1353            assert_eq!(rows, 2, "{kind}");
1354        }
1355    }
1356
1357    #[test]
1358    fn test_strict_policy_rejects_the_first_non_object_record() {
1359        for (kind, value) in NON_OBJECT_VALUES {
1360            for (config, data) in [
1361                (
1362                    JsonParserConfig::jsonl(),
1363                    format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n"),
1364                ),
1365                (
1366                    JsonParserConfig::json_document(),
1367                    format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]"),
1368                ),
1369            ] {
1370                let config = config.with_error_policy(JsonErrorPolicy::Strict);
1371                let err = scan_json_from_reader(Cursor::new(data.as_bytes()), &config, |_| {})
1372                    .expect_err("{kind}: strict mode must reject a non-object record");
1373
1374                let message = err.to_string();
1375                assert!(message.contains("non-object JSON record"), "{message}");
1376                assert!(message.contains("at position 2"), "{message}");
1377                assert!(message.contains(&format!("found {kind}")), "{message}");
1378            }
1379        }
1380    }
1381
1382    #[test]
1383    fn test_number_element_does_not_swallow_the_array_delimiter() {
1384        // serde ends a number by peeking one byte past it and drops that byte
1385        // with the deserializer; here it is the `,` the array scanner needs.
1386        let data = br#"[{"id":1}, 1, 2, 3, {"id":2}, 4, {"id":3}]"#;
1387        let mut rows = 0;
1388
1389        let summary = scan_json_from_reader(
1390            Cursor::new(data.as_ref()),
1391            &JsonParserConfig::json_document(),
1392            |_| rows += 1,
1393        )
1394        .unwrap();
1395
1396        assert_eq!(summary.rows_read, 3);
1397        assert_eq!(summary.malformed_lines, 4);
1398        assert_eq!(rows, 3);
1399    }
1400
1401    #[test]
1402    fn test_number_shaped_garbage_stays_a_malformed_record() {
1403        // `1.2.3` is read by the byte-wise number scanner but is not a number,
1404        // so it must not be reported as a valid non-object record.
1405        let data = b"{\"id\":1}\n1.2.3\n";
1406        let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1407        let err = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {})
1408            .expect_err("number-shaped garbage should be malformed");
1409
1410        assert!(
1411            err.to_string().contains("malformed JSON record"),
1412            "{err}, expected the malformed category"
1413        );
1414    }
1415
1416    #[test]
1417    fn test_input_of_only_non_object_records_fails() {
1418        let file = write_file("\"just a string\"\n1\n[2]\n");
1419        let err = analyze_json_file(file.path(), &JsonParserConfig::jsonl())
1420            .expect_err("nothing profileable should not return a clean empty profile");
1421
1422        assert!(
1423            err.to_string().contains("No valid JSON records found"),
1424            "{err}"
1425        );
1426    }
1427
1428    #[test]
1429    /// The JSON grammar accepts both shapes that carry records, so a bare object
1430    /// is one row rather than an error (#486).
1431    fn test_json_grammar_reads_a_bare_object_as_one_row() {
1432        let data = br#"{"x":1}"#;
1433        let mut rows = 0;
1434        let summary = scan_json_from_reader(
1435            Cursor::new(data.as_ref()),
1436            &JsonParserConfig::json_document(),
1437            |_| rows += 1,
1438        )
1439        .unwrap();
1440
1441        assert_eq!(summary.rows_read, 1);
1442        assert_eq!(summary.malformed_lines, 0);
1443        assert_eq!(rows, 1);
1444    }
1445
1446    #[test]
1447    /// Several objects back to back are not a JSON document. The error names
1448    /// the option that reads them, because this is what a JSONL file looks like
1449    /// when it is read with the wrong grammar.
1450    fn test_json_grammar_rejects_concatenated_objects() {
1451        let data = br#"{"x":1}{"x":2}"#;
1452        let err = scan_json_from_reader(
1453            Cursor::new(data.as_ref()),
1454            &JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict),
1455            |_| {},
1456        )
1457        .expect_err("two objects are not one JSON document");
1458
1459        let message = err.to_string();
1460        assert!(message.contains("malformed JSON document"), "{message}");
1461        assert!(message.contains("format=\"jsonl\""), "{message}");
1462    }
1463
1464    #[test]
1465    fn test_analyze_json_from_reader_jsonl_streaming() {
1466        let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
1467        let cursor = Cursor::new(data.as_ref());
1468        let config = JsonParserConfig::default();
1469
1470        let (profiles, _stats, rows, _malformed, format) =
1471            analyze_json_from_reader(cursor, &config).unwrap();
1472        assert_eq!(format, FileFormat::Jsonl);
1473        assert_eq!(rows, 3);
1474        assert_eq!(profiles.len(), 2);
1475    }
1476
1477    #[test]
1478    fn test_analyze_json_from_reader_json_array() {
1479        let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
1480        let cursor = Cursor::new(data.as_ref());
1481        let config = JsonParserConfig::default();
1482
1483        let (profiles, _stats, rows, _malformed, format) =
1484            analyze_json_from_reader(cursor, &config).unwrap();
1485        assert_eq!(format, FileFormat::Json);
1486        assert_eq!(rows, 2);
1487        assert_eq!(profiles.len(), 2);
1488    }
1489
1490    #[test]
1491    fn test_integer_negative_zero_profiles_as_integer_zero_in_every_shape() {
1492        for (data, config) in [
1493            (
1494                &b"[{\"v\":0},{\"v\":-0}]"[..],
1495                JsonParserConfig::json_document(),
1496            ),
1497            (&b"{\"v\":0}\n{\"v\":-0}\n"[..], JsonParserConfig::jsonl()),
1498            (&b"{\"v\":-0}"[..], JsonParserConfig::json_document()),
1499        ] {
1500            let (profiles, _stats, _rows, _malformed, _format) =
1501                analyze_json_from_reader(Cursor::new(data), &config).unwrap();
1502            assert_eq!(
1503                profiles[0].data_type,
1504                dataprof_core::DataType::Integer,
1505                "{data:?}"
1506            );
1507            assert_eq!(profiles[0].unique_count, Some(1), "{data:?}");
1508        }
1509    }
1510
1511    #[test]
1512    fn test_analyze_json_from_reader_max_rows() {
1513        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
1514        let cursor = Cursor::new(data.as_ref());
1515        let config = JsonParserConfig::default().with_max_rows(3);
1516
1517        let (_profiles, _stats, rows, _malformed, _format) =
1518            analyze_json_from_reader(cursor, &config).unwrap();
1519        assert_eq!(rows, 3);
1520    }
1521
1522    #[test]
1523    fn test_analyze_json_from_reader_missing_fields() {
1524        let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
1525        let cursor = Cursor::new(data.as_ref());
1526        let config = JsonParserConfig::jsonl();
1527
1528        let (profiles, _stats, rows, _malformed, _format) =
1529            analyze_json_from_reader(cursor, &config).unwrap();
1530        assert_eq!(rows, 2);
1531
1532        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1533        assert_eq!(col_b.total_count, 2);
1534    }
1535
1536    #[test]
1537    fn test_identical_records_count_as_duplicates_across_a_late_key() {
1538        // Records 1 and 3 are byte-identical; `b` is only discovered between
1539        // them. Signing a row against whatever columns happened to be known at
1540        // the time would make the two look different and report no duplicate.
1541        let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"x\"}\n";
1542        let cursor = Cursor::new(data.as_ref());
1543        let config = JsonParserConfig::jsonl();
1544
1545        let (_profiles, stats, rows, _malformed, _format) =
1546            analyze_json_from_reader(cursor, &config).unwrap();
1547        assert_eq!(rows, 3);
1548
1549        let summary = stats
1550            .row_duplicate_summary()
1551            .expect("three rows were observed");
1552        assert_eq!(summary.rows_checked, 3);
1553        assert_eq!(summary.duplicate_rows, 1);
1554        assert!(!summary.approximate);
1555    }
1556
1557    #[test]
1558    fn test_a_late_key_does_not_invent_duplicates() {
1559        // The counterpart guard: collapsing the trailing absent field must not
1560        // merge rows that differ in the columns they do share.
1561        let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"z\"}\n";
1562        let cursor = Cursor::new(data.as_ref());
1563        let config = JsonParserConfig::jsonl();
1564
1565        let (_profiles, stats, _rows, _malformed, _format) =
1566            analyze_json_from_reader(cursor, &config).unwrap();
1567
1568        let summary = stats
1569            .row_duplicate_summary()
1570            .expect("three rows were observed");
1571        assert_eq!(summary.duplicate_rows, 0);
1572    }
1573
1574    #[test]
1575    fn test_analyze_json_file_quality_report() {
1576        let file = write_file(r#"[{"x":1},{"x":2}]"#);
1577        let config = JsonParserConfig::default();
1578        let report = analyze_json_file(file.path(), &config).unwrap();
1579
1580        assert_eq!(report.execution.rows_processed, 2);
1581        assert_eq!(report.column_profiles.len(), 1);
1582        assert!(report.quality_score().unwrap() >= 0.0);
1583    }
1584
1585    #[test]
1586    fn test_jsonl_skips_malformed_lines() {
1587        let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
1588        let cursor = Cursor::new(data.as_ref());
1589        let config = JsonParserConfig::jsonl();
1590
1591        let (profiles, _stats, rows, _malformed, format) =
1592            analyze_json_from_reader(cursor, &config).unwrap();
1593        assert_eq!(format, FileFormat::Jsonl);
1594        assert_eq!(rows, 2);
1595        assert_eq!(profiles[0].total_count, 2);
1596    }
1597
1598    #[test]
1599    fn test_strict_policy_errors_on_malformed_jsonl() {
1600        // Malformed in first / middle / last position all abort under Strict.
1601        for data in [
1602            b"not-json\n{\"x\":1}\n{\"x\":2}\n".as_ref(),
1603            b"{\"x\":1}\nnot-json\n{\"x\":2}\n".as_ref(),
1604            b"{\"x\":1}\n{\"x\":2}\nnot-json\n".as_ref(),
1605        ] {
1606            let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1607            let err = scan_json_from_reader(Cursor::new(data), &config, |_| {})
1608                .expect_err("strict mode must reject malformed records");
1609            let message = err.to_string().to_lowercase();
1610            assert!(message.contains("malformed json record"), "got: {message}");
1611            // The offending record text must never be echoed back.
1612            assert!(
1613                !err.to_string().contains("not-json"),
1614                "leaked record: {err}"
1615            );
1616        }
1617    }
1618
1619    #[test]
1620    fn test_tolerant_policy_counts_skipped_records() {
1621        let data = b"{\"x\":1}\nnot-json\n{\"x\":2}\n";
1622        let config = JsonParserConfig::jsonl(); // default Skip
1623        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1624        assert_eq!(summary.rows_read, 2);
1625        assert_eq!(summary.malformed_lines, 1);
1626    }
1627
1628    #[test]
1629    fn test_blank_lines_are_not_counted_as_malformed() {
1630        let data = b"{\"x\":1}\n\n   \n{\"x\":2}\n";
1631        for policy in [JsonErrorPolicy::Skip, JsonErrorPolicy::Strict] {
1632            let config = JsonParserConfig::jsonl().with_error_policy(policy);
1633            let summary =
1634                scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1635            assert_eq!(summary.rows_read, 2, "policy {policy:?}");
1636            assert_eq!(summary.malformed_lines, 0, "policy {policy:?}");
1637        }
1638    }
1639
1640    #[test]
1641    fn test_truncated_final_record_obeys_error_policy() {
1642        let data = b"{\"x\":1}\n{\"x\":2";
1643
1644        let skip = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Skip);
1645        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &skip, |_| {}).unwrap();
1646        assert_eq!(summary.rows_read, 1);
1647        assert_eq!(summary.malformed_lines, 1);
1648
1649        let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1650        let err = scan_json_from_reader(Cursor::new(data.as_ref()), &strict, |_| {})
1651            .expect_err("strict mode must reject an incomplete trailing record");
1652        assert!(
1653            err.to_string()
1654                .to_lowercase()
1655                .contains("malformed json record")
1656        );
1657    }
1658
1659    #[test]
1660    fn test_all_malformed_file_fails() {
1661        let file = write_file("not-json\nalso-bad\n");
1662        let config = JsonParserConfig::jsonl();
1663        let err = analyze_json_file(file.path(), &config)
1664            .expect_err("a file with no valid records must fail");
1665        let message = err.to_string().to_lowercase();
1666        assert!(message.contains("no valid json records"), "got: {message}");
1667    }
1668
1669    #[test]
1670    fn test_tolerant_file_reports_error_count() {
1671        let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1672        let config = JsonParserConfig::jsonl();
1673        let report = analyze_json_file(file.path(), &config).unwrap();
1674        assert_eq!(report.execution.rows_processed, 2);
1675        assert_eq!(report.execution.error_count, 1);
1676    }
1677
1678    #[test]
1679    fn test_strict_file_errors_on_malformed() {
1680        let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1681        let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1682        let err = analyze_json_file(file.path(), &config)
1683            .expect_err("strict mode must reject the malformed record");
1684        assert!(
1685            err.to_string()
1686                .to_lowercase()
1687                .contains("malformed json record")
1688        );
1689    }
1690
1691    #[test]
1692    fn test_analyze_json_with_large_leading_whitespace() {
1693        let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
1694        let cursor = Cursor::new(data.into_bytes());
1695        let config = JsonParserConfig::default();
1696
1697        let (_profiles, _stats, rows, malformed, format) =
1698            analyze_json_from_reader(cursor, &config).unwrap();
1699        assert_eq!(format, FileFormat::Json);
1700        assert_eq!(rows, 1);
1701        assert_eq!(malformed, 0);
1702    }
1703
1704    #[test]
1705    fn test_analyze_json_array() {
1706        let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
1707        let config = JsonParserConfig::default();
1708        let report = analyze_json_file(json.path(), &config).unwrap();
1709        let profiles = &report.column_profiles;
1710
1711        assert_eq!(profiles.len(), 2);
1712        let names: Vec<&str> = profiles
1713            .iter()
1714            .map(|profile| profile.name.as_str())
1715            .collect();
1716        assert!(names.contains(&"name"));
1717        assert!(names.contains(&"age"));
1718
1719        let age = profiles
1720            .iter()
1721            .find(|profile| profile.name == "age")
1722            .unwrap();
1723        assert_eq!(age.total_count, 2);
1724        assert_eq!(age.null_count, 0);
1725    }
1726
1727    #[test]
1728    fn test_analyze_jsonl() {
1729        let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
1730        let config = JsonParserConfig::default();
1731        let report = analyze_json_file(jsonl.path(), &config).unwrap();
1732        let profiles = &report.column_profiles;
1733
1734        assert_eq!(profiles.len(), 1);
1735        assert_eq!(profiles[0].name, "x");
1736        assert_eq!(profiles[0].total_count, 3);
1737    }
1738
1739    #[test]
1740    fn test_analyze_json_with_nulls() {
1741        let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
1742        let config = JsonParserConfig::default();
1743        let report = analyze_json_file(json.path(), &config).unwrap();
1744        let profiles = &report.column_profiles;
1745
1746        let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
1747        assert_eq!(col_a.null_count, 1);
1748
1749        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1750        assert_eq!(col_b.null_count, 1);
1751    }
1752
1753    #[test]
1754    fn test_analyze_json_with_missing_fields() {
1755        let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
1756        let config = JsonParserConfig::default();
1757        let report = analyze_json_file(json.path(), &config).unwrap();
1758        let profiles = &report.column_profiles;
1759
1760        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1761        assert_eq!(col_b.total_count, 2);
1762    }
1763
1764    /// Deliberately non-alphabetical field order: sorting would give
1765    /// `active, amount, date, id`, so any re-sort is visible.
1766    const UNSORTED_FIELDS: &str = r#"{"id":1,"amount":12.5,"active":true,"date":"2026-07-23"}"#;
1767    const UNSORTED_ORDER: [&str; 4] = ["id", "amount", "active", "date"];
1768
1769    fn profile_names(report: &ProfileReport) -> Vec<&str> {
1770        report
1771            .column_profiles
1772            .iter()
1773            .map(|profile| profile.name.as_str())
1774            .collect()
1775    }
1776
1777    #[test]
1778    fn test_json_column_order_follows_source_not_alphabet() {
1779        let array = write_file(&format!("[{UNSORTED_FIELDS},{UNSORTED_FIELDS}]"));
1780        let jsonl = write_file(&format!("{UNSORTED_FIELDS}\n{UNSORTED_FIELDS}\n"));
1781
1782        for file in [&array, &jsonl] {
1783            let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1784            assert_eq!(profile_names(&report), UNSORTED_ORDER);
1785        }
1786    }
1787
1788    #[test]
1789    fn test_json_column_order_appends_later_keys_in_first_seen_order() {
1790        // Each record introduces two keys in reverse-alphabetical order, so
1791        // sorting would reshuffle both the leading pair and the appended pair.
1792        let data = b"{\"zulu\":1,\"mike\":2}\n{\"zulu\":3,\"mike\":4,\"delta\":5,\"alpha\":6}\n";
1793        let file = write_bytes(data);
1794        let report = analyze_json_file(file.path(), &JsonParserConfig::jsonl()).unwrap();
1795
1796        assert_eq!(profile_names(&report), ["zulu", "mike", "delta", "alpha"]);
1797    }
1798
1799    #[test]
1800    fn test_json_column_order_survives_report_serialization() {
1801        let file = write_file(&format!("[{UNSORTED_FIELDS}]"));
1802        let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1803
1804        let json = serde_json::to_string(&report).unwrap();
1805        let restored: Value = serde_json::from_str(&json).unwrap();
1806        let names: Vec<&str> = restored["column_profiles"]
1807            .as_array()
1808            .unwrap()
1809            .iter()
1810            .map(|profile| profile["name"].as_str().unwrap())
1811            .collect();
1812
1813        assert_eq!(names, UNSORTED_ORDER);
1814    }
1815
1816    #[test]
1817    fn test_analyze_json_backfills_columns_discovered_after_first_row() {
1818        let json = write_file(r#"[{"a":1},{"a":2,"b":3}]"#);
1819        let config = JsonParserConfig::default();
1820        let report = analyze_json_file(json.path(), &config).unwrap();
1821        let col_b = report
1822            .column_profiles
1823            .iter()
1824            .find(|profile| profile.name == "b")
1825            .unwrap();
1826
1827        assert_eq!(col_b.total_count, 2);
1828        assert_eq!(col_b.null_count, 1);
1829        assert_eq!(col_b.unique_count, Some(1));
1830    }
1831
1832    #[test]
1833    fn test_analyze_json_empty_array() {
1834        let json = write_file("[]");
1835        let config = JsonParserConfig::default();
1836        let report = analyze_json_file(json.path(), &config).unwrap();
1837        assert!(report.column_profiles.is_empty());
1838        // An empty dataset has nothing to assess: no score is fabricated.
1839        assert_eq!(report.quality_score(), None);
1840    }
1841
1842    #[test]
1843    fn test_analyze_json_malformed_returns_error() {
1844        let json = write_file("this is entirely invalid json");
1845        let config = JsonParserConfig::default();
1846        let err = analyze_json_file(json.path(), &config)
1847            .expect_err("malformed JSON should return an error");
1848
1849        let message = err.to_string().to_lowercase();
1850        assert!(message.contains("malformed") && message.contains("json"));
1851    }
1852
1853    #[test]
1854    fn test_analyze_json_file_detects_format() {
1855        let json_array = write_file(r#"[{"x":1}]"#);
1856        let config = JsonParserConfig::default();
1857        let report = analyze_json_file(json_array.path(), &config).unwrap();
1858        assert!(matches!(
1859            report.data_source,
1860            DataSource::File {
1861                format: FileFormat::Json,
1862                ..
1863            }
1864        ));
1865
1866        let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
1867        let report = analyze_json_file(jsonl.path(), &config).unwrap();
1868        assert!(matches!(
1869            report.data_source,
1870            DataSource::File {
1871                format: FileFormat::Jsonl,
1872                ..
1873            }
1874        ));
1875    }
1876
1877    #[test]
1878    fn test_bom_prefixed_file_preserves_source_size() {
1879        let data = b"\xEF\xBB\xBF[{\"x\":1}]";
1880        let json = write_bytes(data);
1881        let report = analyze_json_file(json.path(), &JsonParserConfig::json_document()).unwrap();
1882
1883        assert_eq!(report.execution.rows_processed, 1);
1884        assert!(matches!(
1885            report.data_source,
1886            DataSource::File {
1887                format: FileFormat::Json,
1888                size_bytes,
1889                ..
1890            } if size_bytes == data.len() as u64
1891        ));
1892    }
1893
1894    #[test]
1895    fn test_analyze_json_file_empty() {
1896        let json = write_file("");
1897        let config = JsonParserConfig::default();
1898        let report = analyze_json_file(json.path(), &config).unwrap();
1899        assert_eq!(report.execution.rows_processed, 0);
1900        assert!(report.column_profiles.is_empty());
1901    }
1902
1903    #[test]
1904    fn test_analyze_json_boolean_and_nested() {
1905        let json =
1906            write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
1907        let config = JsonParserConfig::default();
1908        let report = analyze_json_file(json.path(), &config).unwrap();
1909        let profiles = &report.column_profiles;
1910
1911        let flag = profiles
1912            .iter()
1913            .find(|profile| profile.name == "flag")
1914            .unwrap();
1915        assert_eq!(flag.total_count, 2);
1916
1917        let nested = profiles
1918            .iter()
1919            .find(|profile| profile.name == "nested")
1920            .unwrap();
1921        assert_eq!(nested.total_count, 2);
1922    }
1923
1924    #[test]
1925    fn test_single_root_object_compact_yields_one_row() {
1926        let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
1927        let cursor = Cursor::new(data.as_ref());
1928        let config = JsonParserConfig::default();
1929
1930        let (profiles, _stats, rows, malformed, _format) =
1931            analyze_json_from_reader(cursor, &config).unwrap();
1932        assert_eq!(rows, 1);
1933        assert_eq!(malformed, 0);
1934        assert_eq!(profiles.len(), 2);
1935
1936        let names: Vec<&str> = profiles
1937            .iter()
1938            .map(|profile| profile.name.as_str())
1939            .collect();
1940        assert!(names.contains(&"type"));
1941        assert!(names.contains(&"features"));
1942    }
1943
1944    #[test]
1945    fn test_jsonl_multi_object_still_works() {
1946        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1947        let cursor = Cursor::new(data.as_ref());
1948        let config = JsonParserConfig::default();
1949
1950        let (_profiles, _stats, rows, _malformed, format) =
1951            analyze_json_from_reader(cursor, &config).unwrap();
1952        assert_eq!(format, FileFormat::Jsonl);
1953        assert_eq!(rows, 3);
1954    }
1955
1956    #[test]
1957    fn test_single_root_object_via_analyze_json_file() {
1958        let file =
1959            write_json_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
1960        let config = JsonParserConfig::default();
1961        let report = analyze_json_file(file.path(), &config).unwrap();
1962
1963        assert_eq!(report.execution.rows_processed, 1);
1964        assert_eq!(report.column_profiles.len(), 2);
1965    }
1966
1967    #[test]
1968    fn test_single_root_object_pretty_printed_yields_one_row() {
1969        let data = br#"{
1970  "type": "FeatureCollection",
1971  "features": [
1972    {"id": 1},
1973    {"id": 2}
1974  ]
1975}"#;
1976        let cursor = Cursor::new(data.as_ref());
1977        // A reader has no file name, so the grammar cannot be inferred and is
1978        // named explicitly; `analyze_json_file` takes it from the extension.
1979        let config = JsonParserConfig::json_document();
1980
1981        let (profiles, _stats, rows, malformed, format) =
1982            analyze_json_from_reader(cursor, &config).unwrap();
1983        assert_eq!(format, FileFormat::Json);
1984        assert_eq!(rows, 1);
1985        assert_eq!(malformed, 0);
1986        assert_eq!(profiles.len(), 2);
1987    }
1988
1989    #[test]
1990    fn test_single_root_object_pretty_printed_via_analyze_json_file() {
1991        let file = write_json_file(
1992            "{\n  \"type\": \"FeatureCollection\",\n  \"features\": [\n    {\"id\": 1},\n    {\"id\": 2}\n  ]\n}\n",
1993        );
1994        let config = JsonParserConfig::default();
1995        let report = analyze_json_file(file.path(), &config).unwrap();
1996
1997        assert_eq!(report.execution.rows_processed, 1);
1998        assert_eq!(report.column_profiles.len(), 2);
1999    }
2000}