Skip to main content

dataprof_json/
lib.rs

1use std::collections::{HashMap, HashSet};
2use std::io::BufRead;
3use std::path::Path;
4
5pub use dataprof_core::JsonErrorPolicy;
6use dataprof_core::{
7    ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat, QualityDimension,
8    SemanticHints, TruncationReason,
9};
10use dataprof_runtime::{
11    ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
12};
13use serde::Deserialize;
14use serde_json::Value;
15
16/// JSON/JSONL format hint.
17#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub enum JsonFormat {
19    /// Standard JSON array of objects (`[{...}, {...}]`).
20    JsonArray,
21    /// JSON Lines — one JSON object per line.
22    Jsonl,
23}
24
25/// Configuration for JSON/JSONL parsing and scanning.
26#[derive(Debug, Clone, Default)]
27pub struct JsonParserConfig {
28    /// Force a specific format (None = auto-detect from content).
29    pub format: Option<JsonFormat>,
30    /// Maximum rows to process (None = all rows).
31    pub max_rows: Option<usize>,
32    /// How to react to a malformed record (default: skip).
33    pub error_policy: JsonErrorPolicy,
34}
35
36impl JsonParserConfig {
37    /// Set the maximum number of rows to process.
38    pub fn with_max_rows(mut self, max_rows: usize) -> Self {
39        self.max_rows = Some(max_rows);
40        self
41    }
42
43    /// Set how malformed records are handled.
44    pub fn with_error_policy(mut self, policy: JsonErrorPolicy) -> Self {
45        self.error_policy = policy;
46        self
47    }
48
49    /// Force JSONL format.
50    pub fn jsonl() -> Self {
51        Self {
52            format: Some(JsonFormat::Jsonl),
53            ..Default::default()
54        }
55    }
56
57    /// Force JSON array format.
58    pub fn json_array() -> Self {
59        Self {
60            format: Some(JsonFormat::JsonArray),
61            ..Default::default()
62        }
63    }
64}
65
66/// Borrowed JSON object callback payload.
67pub type JsonObject = serde_json::Map<String, Value>;
68
69/// Summary of a JSON/JSONL scan.
70#[derive(Debug, Clone, PartialEq, Eq)]
71#[non_exhaustive]
72pub struct JsonScanSummary {
73    pub rows_read: usize,
74    pub malformed_lines: usize,
75    pub format: FileFormat,
76    /// The scan stopped at `max_rows` while records still remained. A source
77    /// holding exactly `max_rows` records was read in full, so this stays false.
78    pub truncated: bool,
79}
80
81/// Scan JSON or JSONL input and invoke `on_object` for each object record.
82///
83/// - **JSONL**: scans one top-level JSON value at a time from the reader.
84/// - **JSON array**: streams array elements without buffering the entire input.
85pub fn scan_json_from_reader<R, F>(
86    mut reader: R,
87    config: &JsonParserConfig,
88    mut on_object: F,
89) -> Result<JsonScanSummary, DataProfilerError>
90where
91    R: BufRead,
92    F: FnMut(&JsonObject),
93{
94    let format = match config.format {
95        Some(JsonFormat::JsonArray) => FileFormat::Json,
96        Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
97        None => match consume_leading_whitespace(&mut reader)? {
98            Some(b'[') => FileFormat::Json,
99            _ => FileFormat::Jsonl,
100        },
101    };
102
103    let mut rows_read = 0;
104    let mut malformed_lines = 0;
105    let mut truncated = false;
106
107    match format {
108        FileFormat::Jsonl => loop {
109            if let Some(max) = config.max_rows
110                && rows_read >= max
111            {
112                // Reaching the cap is not truncation unless a record actually
113                // remains: a file with exactly `max_rows` rows was read in full.
114                truncated = consume_leading_whitespace(&mut reader)?.is_some();
115                break;
116            }
117
118            // `serde_json` classifies both a clean end of input and an
119            // incomplete trailing value as `Category::Eof`. Check whether a
120            // non-whitespace value actually starts here so only the former is
121            // treated as normal exhaustion.
122            if consume_leading_whitespace(&mut reader)?.is_none() {
123                break;
124            }
125
126            let mut deserializer = serde_json::Deserializer::from_reader(&mut reader);
127            let value: Value = match Value::deserialize(&mut deserializer) {
128                Ok(v) => v,
129                Err(err) => {
130                    if config.error_policy == JsonErrorPolicy::Strict {
131                        return Err(malformed_record_error(&err));
132                    }
133                    malformed_lines += 1;
134                    if err.classify() == serde_json::error::Category::Eof {
135                        break;
136                    }
137                    skip_to_next_line(&mut reader)?;
138                    continue;
139                }
140            };
141
142            if let Value::Object(ref obj) = value {
143                on_object(obj);
144                rows_read += 1;
145            }
146        },
147        FileFormat::Json => {
148            let mut found_array = false;
149            loop {
150                let mut consume = 0;
151                {
152                    let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
153                    if buf.is_empty() {
154                        break;
155                    }
156                    for &byte in buf {
157                        consume += 1;
158                        if byte == b'[' {
159                            found_array = true;
160                            break;
161                        } else if !byte.is_ascii_whitespace() {
162                            break;
163                        }
164                    }
165                }
166                reader.consume(consume);
167                if found_array || consume == 0 {
168                    break;
169                }
170            }
171
172            if !found_array {
173                if config.format.is_some() {
174                    return Err(DataProfilerError::JsonParsingError {
175                        message: "Expected JSON array (starts with '[') but input does not match"
176                            .to_string(),
177                    });
178                }
179            } else {
180                let mut expect_value = true;
181                let mut allow_end = true;
182                let mut array_closed = false;
183                let mut drain_remainder = false;
184
185                loop {
186                    let Some(next) = consume_leading_whitespace(&mut reader)? else {
187                        if config.error_policy == JsonErrorPolicy::Strict {
188                            return Err(malformed_array_error(
189                                "unexpected end of input before closing ']'",
190                            ));
191                        }
192                        malformed_lines += 1;
193                        drain_remainder = true;
194                        break;
195                    };
196
197                    if expect_value {
198                        if next == b']' {
199                            if !allow_end {
200                                if config.error_policy == JsonErrorPolicy::Strict {
201                                    return Err(malformed_array_error(
202                                        "trailing comma before closing ']'",
203                                    ));
204                                }
205                                malformed_lines += 1;
206                                drain_remainder = true;
207                                break;
208                            }
209                            reader.consume(1);
210                            array_closed = true;
211                            break;
212                        }
213
214                        if let Some(max) = config.max_rows
215                            && rows_read >= max
216                        {
217                            // A value remains unread. Do not validate the unread
218                            // suffix of an intentionally bounded scan.
219                            truncated = true;
220                            break;
221                        }
222
223                        if next == b',' {
224                            if config.error_policy == JsonErrorPolicy::Strict {
225                                return Err(malformed_array_error(
226                                    "unexpected comma where an array value was required",
227                                ));
228                            }
229                            malformed_lines += 1;
230                            drain_remainder = true;
231                            break;
232                        }
233
234                        let mut deserializer = serde_json::Deserializer::from_reader(&mut reader);
235                        match Value::deserialize(&mut deserializer) {
236                            Ok(Value::Object(obj)) => {
237                                on_object(&obj);
238                                rows_read += 1;
239                            }
240                            Ok(_) => {}
241                            Err(err) => {
242                                if config.error_policy == JsonErrorPolicy::Strict {
243                                    return Err(malformed_record_error(&err));
244                                }
245                                malformed_lines += 1;
246                                drain_remainder = true;
247                                break;
248                            }
249                        }
250                        expect_value = false;
251                    } else {
252                        match next {
253                            b',' => {
254                                reader.consume(1);
255                                expect_value = true;
256                                allow_end = false;
257                            }
258                            b']' => {
259                                reader.consume(1);
260                                array_closed = true;
261                                break;
262                            }
263                            _ => {
264                                if config.error_policy == JsonErrorPolicy::Strict {
265                                    return Err(malformed_array_error(
266                                        "expected ',' or ']' after an array value",
267                                    ));
268                                }
269                                malformed_lines += 1;
270                                drain_remainder = true;
271                                break;
272                            }
273                        }
274                    }
275                }
276
277                if drain_remainder {
278                    drain_to_end(&mut reader)?;
279                } else if array_closed && consume_leading_whitespace(&mut reader)?.is_some() {
280                    if config.error_policy == JsonErrorPolicy::Strict {
281                        return Err(malformed_array_error(
282                            "non-whitespace content follows the closing ']'",
283                        ));
284                    }
285                    malformed_lines += 1;
286                    drain_to_end(&mut reader)?;
287                }
288            }
289        }
290        _ => unreachable!("json scanner only returns json or jsonl formats"),
291    }
292
293    Ok(JsonScanSummary {
294        rows_read,
295        malformed_lines,
296        format,
297        truncated,
298    })
299}
300
301/// Build a strict-mode parse error from a serde failure. The serde message
302/// carries line/column context but never the record contents, so it is safe to
303/// surface directly.
304fn malformed_record_error(err: &serde_json::Error) -> DataProfilerError {
305    DataProfilerError::JsonParsingError {
306        message: format!("malformed JSON record: {err}"),
307    }
308}
309
310fn malformed_array_error(message: &str) -> DataProfilerError {
311    DataProfilerError::JsonParsingError {
312        message: format!("malformed JSON array: {message}"),
313    }
314}
315
316fn drain_to_end<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
317    loop {
318        let available = reader.fill_buf().map_err(DataProfilerError::from)?.len();
319        if available == 0 {
320            return Ok(());
321        }
322        reader.consume(available);
323    }
324}
325
326fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
327    loop {
328        let mut bytes_to_consume = 0;
329        let first_non_whitespace = {
330            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
331            if buf.is_empty() {
332                return Ok(None);
333            }
334
335            let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
336            if first_non_whitespace.is_none() {
337                bytes_to_consume = buf.len();
338            }
339            first_non_whitespace
340        };
341
342        if first_non_whitespace.is_some() {
343            return Ok(first_non_whitespace);
344        }
345
346        reader.consume(bytes_to_consume);
347    }
348}
349
350fn skip_to_next_line<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
351    let mut discarded = Vec::new();
352    reader
353        .read_until(b'\n', &mut discarded)
354        .map_err(DataProfilerError::from)?;
355    Ok(())
356}
357
358/// Convert a JSON [`Value`] to a flat string for column storage.
359fn json_value_to_string(value: &Value) -> String {
360    match value {
361        Value::Null => String::new(),
362        Value::Bool(boolean) => boolean.to_string(),
363        Value::Number(number) => number.to_string(),
364        Value::String(string) => string.to_string(),
365        // decode-audit: impossible — serializing an in-memory Value back to a
366        // string cannot fail; a panic here beats folding "" (= null) into stats.
367        Value::Array(_) | Value::Object(_) => {
368            serde_json::to_string(value).expect("re-serializing a parsed JSON value cannot fail")
369        }
370    }
371}
372
373/// Feed a JSON object's fields into a [`StreamingColumnCollection`].
374fn feed_json_object(
375    obj: &JsonObject,
376    prior_rows: usize,
377    known_columns: &mut Vec<String>,
378    known_columns_set: &mut HashSet<String>,
379    column_stats: &mut StreamingColumnCollection,
380) {
381    for key in obj.keys() {
382        if known_columns_set.insert(key.clone()) {
383            known_columns.push(key.clone());
384            column_stats.init_column_with_missing(key, prior_rows);
385        }
386    }
387
388    let values: Vec<String> = known_columns
389        .iter()
390        .map(|column| {
391            // decode-audit: no-data — a key absent from this object is a
392            // missing field, and "" is the profiler's textual null.
393            obj.get(column)
394                .map(json_value_to_string)
395                .unwrap_or_default()
396        })
397        .collect();
398
399    column_stats.process_record(known_columns, values);
400}
401
402/// Analyze JSON/JSONL data from a buffered reader using streaming statistics.
403///
404/// Returns `(column_profiles, streaming_stats, rows_read, malformed_lines, detected_format)`.
405pub fn analyze_json_from_reader<R: BufRead>(
406    reader: R,
407    config: &JsonParserConfig,
408) -> Result<
409    (
410        Vec<ColumnProfile>,
411        StreamingColumnCollection,
412        usize,
413        usize,
414        FileFormat,
415    ),
416    DataProfilerError,
417> {
418    analyze_json_from_reader_with_hints(reader, config, &SemanticHints::default())
419}
420
421pub fn analyze_json_from_reader_with_hints<R: BufRead>(
422    reader: R,
423    config: &JsonParserConfig,
424    semantic_hints: &SemanticHints,
425) -> Result<
426    (
427        Vec<ColumnProfile>,
428        StreamingColumnCollection,
429        usize,
430        usize,
431        FileFormat,
432    ),
433    DataProfilerError,
434> {
435    let (profiles, stats, rows_read, malformed_lines, format, _truncated) =
436        analyze_json_from_reader_full(reader, config, semantic_hints)?;
437    Ok((profiles, stats, rows_read, malformed_lines, format))
438}
439
440/// As [`analyze_json_from_reader_with_hints`], but also reports whether the scan
441/// stopped early. Kept private so the public tuple stays stable.
442#[allow(clippy::type_complexity)]
443fn analyze_json_from_reader_full<R: BufRead>(
444    reader: R,
445    config: &JsonParserConfig,
446    semantic_hints: &SemanticHints,
447) -> Result<
448    (
449        Vec<ColumnProfile>,
450        StreamingColumnCollection,
451        usize,
452        usize,
453        FileFormat,
454        bool,
455    ),
456    DataProfilerError,
457> {
458    let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
459    let mut known_columns = Vec::new();
460    let mut known_columns_set = HashSet::new();
461    let mut rows_seen = 0;
462
463    let summary = scan_json_from_reader(reader, config, |obj| {
464        feed_json_object(
465            obj,
466            rows_seen,
467            &mut known_columns,
468            &mut known_columns_set,
469            &mut column_stats,
470        );
471        rows_seen += 1;
472    })?;
473
474    let profiles = profile_builder::profiles_from_streaming_with_hints(
475        &column_stats,
476        false,
477        false,
478        None,
479        semantic_hints,
480    );
481
482    Ok((
483        profiles,
484        column_stats,
485        summary.rows_read,
486        summary.malformed_lines,
487        summary.format,
488        summary.truncated,
489    ))
490}
491
492/// Analyze a JSON or JSONL file, returning a full [`ProfileReport`].
493pub fn analyze_json_file(
494    file_path: &Path,
495    config: &JsonParserConfig,
496) -> Result<ProfileReport, DataProfilerError> {
497    analyze_json_file_with_dimensions(file_path, config, None)
498}
499
500/// Like [`analyze_json_file`] but only computes the requested quality dimensions.
501pub fn analyze_json_file_with_dimensions(
502    file_path: &Path,
503    config: &JsonParserConfig,
504    quality_dimensions: Option<&[QualityDimension]>,
505) -> Result<ProfileReport, DataProfilerError> {
506    analyze_json_file_with_dimensions_and_hints(
507        file_path,
508        config,
509        quality_dimensions,
510        &SemanticHints::default(),
511    )
512}
513
514pub fn analyze_json_file_with_dimensions_and_hints(
515    file_path: &Path,
516    config: &JsonParserConfig,
517    quality_dimensions: Option<&[QualityDimension]>,
518    semantic_hints: &SemanticHints,
519) -> Result<ProfileReport, DataProfilerError> {
520    let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
521    let start = std::time::Instant::now();
522
523    let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
524    let buf_reader = std::io::BufReader::new(file);
525
526    let (column_profiles, column_stats, rows_read, malformed_lines, format, truncated) =
527        analyze_json_from_reader_full(buf_reader, config, semantic_hints)?;
528
529    let file_source = DataSource::File {
530        path: file_path.display().to_string(),
531        format,
532        size_bytes: metadata.len(),
533        modified_at: None,
534        parquet_metadata: None,
535    };
536
537    if rows_read == 0 {
538        if malformed_lines > 0 {
539            return Err(DataProfilerError::JsonParsingError {
540                message: "No valid JSON records found in file (malformed JSON encountered)"
541                    .to_string(),
542            });
543        }
544        let mut assembler = ReportAssembler::new(
545            file_source,
546            ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("json"),
547        )
548        .columns(column_profiles)
549        .with_quality_data(HashMap::new());
550        if let Some(dimensions) = quality_dimensions {
551            assembler = assembler.with_requested_dimensions(dimensions.to_vec());
552        }
553        return Ok(assembler.build());
554    }
555
556    let sample_columns = profile_builder::quality_check_samples(&column_stats);
557    let scan_time_ms = start.elapsed().as_millis();
558    let num_columns = column_profiles.len();
559
560    let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms)
561        .with_engine("json")
562        // Tolerant scans surface skipped malformed records here so callers can
563        // tell a partial profile from a clean one.
564        .with_error_count(malformed_lines);
565    // `truncated` is set only when a record still remained at the cap, so a source
566    // holding exactly `max_rows` records is reported as fully read, not cut short.
567    if truncated && let Some(max) = config.max_rows {
568        execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
569    }
570
571    let mut assembler = ReportAssembler::new(file_source, execution)
572        .columns(column_profiles)
573        .with_quality_data(sample_columns)
574        .with_row_duplicates(column_stats.row_duplicate_summary())
575        .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
576        .with_semantic_hints(semantic_hints.clone());
577    if let Some(dimensions) = quality_dimensions {
578        assembler = assembler.with_requested_dimensions(dimensions.to_vec());
579    }
580    Ok(assembler.build())
581}
582
583fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
584    if error.kind() == std::io::ErrorKind::NotFound {
585        DataProfilerError::FileNotFound {
586            path: file_path.display().to_string(),
587        }
588    } else {
589        DataProfilerError::from(error)
590    }
591}
592
593#[cfg(test)]
594mod tests {
595    use super::*;
596    use std::io::{Cursor, Write};
597    use tempfile::NamedTempFile;
598
599    fn write_file(content: &str) -> NamedTempFile {
600        let mut file = NamedTempFile::new().unwrap();
601        write!(file, "{}", content).unwrap();
602        file.flush().unwrap();
603        file
604    }
605
606    #[test]
607    fn test_scan_jsonl_detects_format_and_rows() {
608        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
609        let mut rows = 0;
610
611        let summary = scan_json_from_reader(
612            Cursor::new(data.as_ref()),
613            &JsonParserConfig::default(),
614            |_| {
615                rows += 1;
616            },
617        )
618        .unwrap();
619
620        assert_eq!(summary.format, FileFormat::Jsonl);
621        assert_eq!(summary.rows_read, 3);
622        assert_eq!(summary.malformed_lines, 0);
623        assert_eq!(rows, 3);
624    }
625
626    #[test]
627    fn test_scan_json_array_detects_format_and_rows() {
628        let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
629        let mut keys = Vec::new();
630
631        let summary = scan_json_from_reader(
632            Cursor::new(data.as_ref()),
633            &JsonParserConfig::default(),
634            |obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
635        )
636        .unwrap();
637
638        assert_eq!(summary.format, FileFormat::Json);
639        assert_eq!(summary.rows_read, 2);
640        assert_eq!(keys.len(), 2);
641        assert_eq!(keys[0], vec!["name".to_string()]);
642    }
643
644    #[test]
645    fn test_json_array_container_grammar_obeys_error_policy() {
646        let cases = [
647            (br#"[{"x":1}"#.as_ref(), 1, "missing closing bracket"),
648            (br#"[{"x":1},"#.as_ref(), 1, "comma followed by EOF"),
649            (br#"[{"x":1} {"x":2}]"#.as_ref(), 1, "missing comma"),
650            (br#"[{"x":1},,{"x":2}]"#.as_ref(), 1, "doubled comma"),
651            (br#"[,{"x":1}]"#.as_ref(), 0, "leading comma"),
652            (br#"[{"x":1},]"#.as_ref(), 1, "trailing comma"),
653            (br#"[{"x":1}] trailing"#.as_ref(), 1, "trailing content"),
654            (
655                br#"[{"x":1}][{"x":2}]"#.as_ref(),
656                1,
657                "second top-level value",
658            ),
659        ];
660
661        for (data, expected_rows, case) in cases {
662            let skip = JsonParserConfig::json_array();
663            let summary =
664                scan_json_from_reader(Cursor::new(data), &skip, |_| {}).expect("tolerant scan");
665            assert_eq!(summary.rows_read, expected_rows, "{case}");
666            assert_eq!(summary.malformed_lines, 1, "{case}");
667
668            let strict = JsonParserConfig::json_array().with_error_policy(JsonErrorPolicy::Strict);
669            let err = scan_json_from_reader(Cursor::new(data), &strict, |_| {})
670                .expect_err("strict mode must reject invalid array grammar");
671            assert!(
672                err.to_string()
673                    .to_lowercase()
674                    .contains("malformed json array"),
675                "{case}: {err}"
676            );
677        }
678    }
679
680    #[test]
681    fn test_json_array_max_rows_does_not_validate_unread_values() {
682        let data = br#"[{"x":1},not-valid-json"#;
683        let config = JsonParserConfig::json_array().with_max_rows(1);
684        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
685
686        assert_eq!(summary.rows_read, 1);
687        assert_eq!(summary.malformed_lines, 0);
688        assert!(summary.truncated);
689    }
690
691    #[test]
692    fn test_scan_respects_max_rows() {
693        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
694        let config = JsonParserConfig::default().with_max_rows(2);
695        let mut rows = 0;
696
697        let summary =
698            scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
699
700        assert_eq!(summary.rows_read, 2);
701        assert_eq!(rows, 2);
702    }
703
704    #[test]
705    fn test_scan_skips_malformed_jsonl_lines() {
706        let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
707        let mut rows = 0;
708
709        let summary = scan_json_from_reader(
710            Cursor::new(data.as_ref()),
711            &JsonParserConfig::jsonl(),
712            |_| {
713                rows += 1;
714            },
715        )
716        .unwrap();
717
718        assert_eq!(summary.format, FileFormat::Jsonl);
719        assert_eq!(summary.rows_read, 2);
720        assert_eq!(summary.malformed_lines, 1);
721        assert_eq!(rows, 2);
722    }
723
724    #[test]
725    fn test_scan_pretty_printed_root_object_is_one_jsonl_row() {
726        let data = br#"{
727    "type": "FeatureCollection",
728    "features": [
729        {"id": 1},
730        {"id": 2}
731  ]
732}"#;
733        let mut object_field_counts = Vec::new();
734
735        let summary = scan_json_from_reader(
736            Cursor::new(data.as_ref()),
737            &JsonParserConfig::default(),
738            |obj| object_field_counts.push(obj.len()),
739        )
740        .unwrap();
741
742        assert_eq!(summary.format, FileFormat::Jsonl);
743        assert_eq!(summary.rows_read, 1);
744        assert_eq!(summary.malformed_lines, 0);
745        assert_eq!(object_field_counts, vec![2]);
746    }
747
748    #[test]
749    fn test_forced_json_array_requires_opening_bracket() {
750        let data = br#"{"x":1}"#;
751        let err = scan_json_from_reader(
752            Cursor::new(data.as_ref()),
753            &JsonParserConfig::json_array(),
754            |_| {},
755        )
756        .expect_err("forced json array should reject non-array input");
757
758        assert!(err.to_string().contains("Expected JSON array"));
759    }
760
761    #[test]
762    fn test_analyze_json_from_reader_jsonl_streaming() {
763        let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
764        let cursor = Cursor::new(data.as_ref());
765        let config = JsonParserConfig::default();
766
767        let (profiles, _stats, rows, _malformed, format) =
768            analyze_json_from_reader(cursor, &config).unwrap();
769        assert_eq!(format, FileFormat::Jsonl);
770        assert_eq!(rows, 3);
771        assert_eq!(profiles.len(), 2);
772    }
773
774    #[test]
775    fn test_analyze_json_from_reader_json_array() {
776        let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
777        let cursor = Cursor::new(data.as_ref());
778        let config = JsonParserConfig::default();
779
780        let (profiles, _stats, rows, _malformed, format) =
781            analyze_json_from_reader(cursor, &config).unwrap();
782        assert_eq!(format, FileFormat::Json);
783        assert_eq!(rows, 2);
784        assert_eq!(profiles.len(), 2);
785    }
786
787    #[test]
788    fn test_analyze_json_from_reader_max_rows() {
789        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
790        let cursor = Cursor::new(data.as_ref());
791        let config = JsonParserConfig::default().with_max_rows(3);
792
793        let (_profiles, _stats, rows, _malformed, _format) =
794            analyze_json_from_reader(cursor, &config).unwrap();
795        assert_eq!(rows, 3);
796    }
797
798    #[test]
799    fn test_analyze_json_from_reader_missing_fields() {
800        let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
801        let cursor = Cursor::new(data.as_ref());
802        let config = JsonParserConfig::jsonl();
803
804        let (profiles, _stats, rows, _malformed, _format) =
805            analyze_json_from_reader(cursor, &config).unwrap();
806        assert_eq!(rows, 2);
807
808        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
809        assert_eq!(col_b.total_count, 2);
810    }
811
812    #[test]
813    fn test_analyze_json_file_quality_report() {
814        let file = write_file(r#"[{"x":1},{"x":2}]"#);
815        let config = JsonParserConfig::default();
816        let report = analyze_json_file(file.path(), &config).unwrap();
817
818        assert_eq!(report.execution.rows_processed, 2);
819        assert_eq!(report.column_profiles.len(), 1);
820        assert!(report.quality_score().unwrap() >= 0.0);
821    }
822
823    #[test]
824    fn test_jsonl_skips_malformed_lines() {
825        let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
826        let cursor = Cursor::new(data.as_ref());
827        let config = JsonParserConfig::jsonl();
828
829        let (profiles, _stats, rows, _malformed, format) =
830            analyze_json_from_reader(cursor, &config).unwrap();
831        assert_eq!(format, FileFormat::Jsonl);
832        assert_eq!(rows, 2);
833        assert_eq!(profiles[0].total_count, 2);
834    }
835
836    #[test]
837    fn test_strict_policy_errors_on_malformed_jsonl() {
838        // Malformed in first / middle / last position all abort under Strict.
839        for data in [
840            b"not-json\n{\"x\":1}\n{\"x\":2}\n".as_ref(),
841            b"{\"x\":1}\nnot-json\n{\"x\":2}\n".as_ref(),
842            b"{\"x\":1}\n{\"x\":2}\nnot-json\n".as_ref(),
843        ] {
844            let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
845            let err = scan_json_from_reader(Cursor::new(data), &config, |_| {})
846                .expect_err("strict mode must reject malformed records");
847            let message = err.to_string().to_lowercase();
848            assert!(message.contains("malformed json record"), "got: {message}");
849            // The offending record text must never be echoed back.
850            assert!(
851                !err.to_string().contains("not-json"),
852                "leaked record: {err}"
853            );
854        }
855    }
856
857    #[test]
858    fn test_tolerant_policy_counts_skipped_records() {
859        let data = b"{\"x\":1}\nnot-json\n{\"x\":2}\n";
860        let config = JsonParserConfig::jsonl(); // default Skip
861        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
862        assert_eq!(summary.rows_read, 2);
863        assert_eq!(summary.malformed_lines, 1);
864    }
865
866    #[test]
867    fn test_blank_lines_are_not_counted_as_malformed() {
868        let data = b"{\"x\":1}\n\n   \n{\"x\":2}\n";
869        for policy in [JsonErrorPolicy::Skip, JsonErrorPolicy::Strict] {
870            let config = JsonParserConfig::jsonl().with_error_policy(policy);
871            let summary =
872                scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
873            assert_eq!(summary.rows_read, 2, "policy {policy:?}");
874            assert_eq!(summary.malformed_lines, 0, "policy {policy:?}");
875        }
876    }
877
878    #[test]
879    fn test_truncated_final_record_obeys_error_policy() {
880        let data = b"{\"x\":1}\n{\"x\":2";
881
882        let skip = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Skip);
883        let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &skip, |_| {}).unwrap();
884        assert_eq!(summary.rows_read, 1);
885        assert_eq!(summary.malformed_lines, 1);
886
887        let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
888        let err = scan_json_from_reader(Cursor::new(data.as_ref()), &strict, |_| {})
889            .expect_err("strict mode must reject an incomplete trailing record");
890        assert!(
891            err.to_string()
892                .to_lowercase()
893                .contains("malformed json record")
894        );
895    }
896
897    #[test]
898    fn test_all_malformed_file_fails() {
899        let file = write_file("not-json\nalso-bad\n");
900        let config = JsonParserConfig::jsonl();
901        let err = analyze_json_file(file.path(), &config)
902            .expect_err("a file with no valid records must fail");
903        let message = err.to_string().to_lowercase();
904        assert!(message.contains("no valid json records"), "got: {message}");
905    }
906
907    #[test]
908    fn test_tolerant_file_reports_error_count() {
909        let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
910        let config = JsonParserConfig::jsonl();
911        let report = analyze_json_file(file.path(), &config).unwrap();
912        assert_eq!(report.execution.rows_processed, 2);
913        assert_eq!(report.execution.error_count, 1);
914    }
915
916    #[test]
917    fn test_strict_file_errors_on_malformed() {
918        let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
919        let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
920        let err = analyze_json_file(file.path(), &config)
921            .expect_err("strict mode must reject the malformed record");
922        assert!(
923            err.to_string()
924                .to_lowercase()
925                .contains("malformed json record")
926        );
927    }
928
929    #[test]
930    fn test_analyze_json_with_large_leading_whitespace() {
931        let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
932        let cursor = Cursor::new(data.into_bytes());
933        let config = JsonParserConfig::default();
934
935        let (_profiles, _stats, rows, malformed, format) =
936            analyze_json_from_reader(cursor, &config).unwrap();
937        assert_eq!(format, FileFormat::Json);
938        assert_eq!(rows, 1);
939        assert_eq!(malformed, 0);
940    }
941
942    #[test]
943    fn test_analyze_json_array() {
944        let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
945        let config = JsonParserConfig::default();
946        let report = analyze_json_file(json.path(), &config).unwrap();
947        let profiles = &report.column_profiles;
948
949        assert_eq!(profiles.len(), 2);
950        let names: Vec<&str> = profiles
951            .iter()
952            .map(|profile| profile.name.as_str())
953            .collect();
954        assert!(names.contains(&"name"));
955        assert!(names.contains(&"age"));
956
957        let age = profiles
958            .iter()
959            .find(|profile| profile.name == "age")
960            .unwrap();
961        assert_eq!(age.total_count, 2);
962        assert_eq!(age.null_count, 0);
963    }
964
965    #[test]
966    fn test_analyze_jsonl() {
967        let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
968        let config = JsonParserConfig::default();
969        let report = analyze_json_file(jsonl.path(), &config).unwrap();
970        let profiles = &report.column_profiles;
971
972        assert_eq!(profiles.len(), 1);
973        assert_eq!(profiles[0].name, "x");
974        assert_eq!(profiles[0].total_count, 3);
975    }
976
977    #[test]
978    fn test_analyze_json_with_nulls() {
979        let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
980        let config = JsonParserConfig::default();
981        let report = analyze_json_file(json.path(), &config).unwrap();
982        let profiles = &report.column_profiles;
983
984        let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
985        assert_eq!(col_a.null_count, 1);
986
987        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
988        assert_eq!(col_b.null_count, 1);
989    }
990
991    #[test]
992    fn test_analyze_json_with_missing_fields() {
993        let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
994        let config = JsonParserConfig::default();
995        let report = analyze_json_file(json.path(), &config).unwrap();
996        let profiles = &report.column_profiles;
997
998        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
999        assert_eq!(col_b.total_count, 2);
1000    }
1001
1002    #[test]
1003    fn test_analyze_json_backfills_columns_discovered_after_first_row() {
1004        let json = write_file(r#"[{"a":1},{"a":2,"b":3}]"#);
1005        let config = JsonParserConfig::default();
1006        let report = analyze_json_file(json.path(), &config).unwrap();
1007        let col_b = report
1008            .column_profiles
1009            .iter()
1010            .find(|profile| profile.name == "b")
1011            .unwrap();
1012
1013        assert_eq!(col_b.total_count, 2);
1014        assert_eq!(col_b.null_count, 1);
1015        assert_eq!(col_b.unique_count, Some(1));
1016    }
1017
1018    #[test]
1019    fn test_analyze_json_empty_array() {
1020        let json = write_file("[]");
1021        let config = JsonParserConfig::default();
1022        let report = analyze_json_file(json.path(), &config).unwrap();
1023        assert!(report.column_profiles.is_empty());
1024        // An empty dataset has nothing to assess: no score is fabricated.
1025        assert_eq!(report.quality_score(), None);
1026    }
1027
1028    #[test]
1029    fn test_analyze_json_malformed_returns_error() {
1030        let json = write_file("this is entirely invalid json");
1031        let config = JsonParserConfig::default();
1032        let err = analyze_json_file(json.path(), &config)
1033            .expect_err("malformed JSON should return an error");
1034
1035        let message = err.to_string().to_lowercase();
1036        assert!(message.contains("malformed") && message.contains("json"));
1037    }
1038
1039    #[test]
1040    fn test_analyze_json_file_detects_format() {
1041        let json_array = write_file(r#"[{"x":1}]"#);
1042        let config = JsonParserConfig::default();
1043        let report = analyze_json_file(json_array.path(), &config).unwrap();
1044        assert!(matches!(
1045            report.data_source,
1046            DataSource::File {
1047                format: FileFormat::Json,
1048                ..
1049            }
1050        ));
1051
1052        let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
1053        let report = analyze_json_file(jsonl.path(), &config).unwrap();
1054        assert!(matches!(
1055            report.data_source,
1056            DataSource::File {
1057                format: FileFormat::Jsonl,
1058                ..
1059            }
1060        ));
1061    }
1062
1063    #[test]
1064    fn test_analyze_json_file_empty() {
1065        let json = write_file("");
1066        let config = JsonParserConfig::default();
1067        let report = analyze_json_file(json.path(), &config).unwrap();
1068        assert_eq!(report.execution.rows_processed, 0);
1069        assert!(report.column_profiles.is_empty());
1070    }
1071
1072    #[test]
1073    fn test_analyze_json_boolean_and_nested() {
1074        let json =
1075            write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
1076        let config = JsonParserConfig::default();
1077        let report = analyze_json_file(json.path(), &config).unwrap();
1078        let profiles = &report.column_profiles;
1079
1080        let flag = profiles
1081            .iter()
1082            .find(|profile| profile.name == "flag")
1083            .unwrap();
1084        assert_eq!(flag.total_count, 2);
1085
1086        let nested = profiles
1087            .iter()
1088            .find(|profile| profile.name == "nested")
1089            .unwrap();
1090        assert_eq!(nested.total_count, 2);
1091    }
1092
1093    #[test]
1094    fn test_single_root_object_compact_yields_one_row() {
1095        let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
1096        let cursor = Cursor::new(data.as_ref());
1097        let config = JsonParserConfig::default();
1098
1099        let (profiles, _stats, rows, malformed, _format) =
1100            analyze_json_from_reader(cursor, &config).unwrap();
1101        assert_eq!(rows, 1);
1102        assert_eq!(malformed, 0);
1103        assert_eq!(profiles.len(), 2);
1104
1105        let names: Vec<&str> = profiles
1106            .iter()
1107            .map(|profile| profile.name.as_str())
1108            .collect();
1109        assert!(names.contains(&"type"));
1110        assert!(names.contains(&"features"));
1111    }
1112
1113    #[test]
1114    fn test_jsonl_multi_object_still_works() {
1115        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1116        let cursor = Cursor::new(data.as_ref());
1117        let config = JsonParserConfig::default();
1118
1119        let (_profiles, _stats, rows, _malformed, format) =
1120            analyze_json_from_reader(cursor, &config).unwrap();
1121        assert_eq!(format, FileFormat::Jsonl);
1122        assert_eq!(rows, 3);
1123    }
1124
1125    #[test]
1126    fn test_single_root_object_via_analyze_json_file() {
1127        let file = write_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
1128        let config = JsonParserConfig::default();
1129        let report = analyze_json_file(file.path(), &config).unwrap();
1130
1131        assert_eq!(report.execution.rows_processed, 1);
1132        assert_eq!(report.column_profiles.len(), 2);
1133    }
1134
1135    #[test]
1136    fn test_single_root_object_pretty_printed_yields_one_row() {
1137        let data = br#"{
1138  "type": "FeatureCollection",
1139  "features": [
1140    {"id": 1},
1141    {"id": 2}
1142  ]
1143}"#;
1144        let cursor = Cursor::new(data.as_ref());
1145        let config = JsonParserConfig::default();
1146
1147        let (profiles, _stats, rows, malformed, format) =
1148            analyze_json_from_reader(cursor, &config).unwrap();
1149        assert_eq!(format, FileFormat::Jsonl);
1150        assert_eq!(rows, 1);
1151        assert_eq!(malformed, 0);
1152        assert_eq!(profiles.len(), 2);
1153    }
1154
1155    #[test]
1156    fn test_single_root_object_pretty_printed_via_analyze_json_file() {
1157        let file = write_file(
1158            "{\n  \"type\": \"FeatureCollection\",\n  \"features\": [\n    {\"id\": 1},\n    {\"id\": 2}\n  ]\n}\n",
1159        );
1160        let config = JsonParserConfig::default();
1161        let report = analyze_json_file(file.path(), &config).unwrap();
1162
1163        assert_eq!(report.execution.rows_processed, 1);
1164        assert_eq!(report.column_profiles.len(), 2);
1165    }
1166}