Skip to main content

dataprof_json/
lib.rs

1use std::collections::HashSet;
2use std::io::BufRead;
3use std::path::Path;
4
5use dataprof_core::{
6    ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat, QualityDimension,
7    SemanticHints, TruncationReason,
8};
9use dataprof_runtime::{
10    ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12use serde::Deserialize;
13use serde_json::Value;
14
15/// JSON/JSONL format hint.
16#[derive(Debug, Clone, Copy, PartialEq, Eq)]
17pub enum JsonFormat {
18    /// Standard JSON array of objects (`[{...}, {...}]`).
19    JsonArray,
20    /// JSON Lines — one JSON object per line.
21    Jsonl,
22}
23
24/// Configuration for JSON/JSONL parsing and scanning.
25#[derive(Debug, Clone, Default)]
26pub struct JsonParserConfig {
27    /// Force a specific format (None = auto-detect from content).
28    pub format: Option<JsonFormat>,
29    /// Maximum rows to process (None = all rows).
30    pub max_rows: Option<usize>,
31}
32
33impl JsonParserConfig {
34    /// Set the maximum number of rows to process.
35    pub fn with_max_rows(mut self, max_rows: usize) -> Self {
36        self.max_rows = Some(max_rows);
37        self
38    }
39
40    /// Force JSONL format.
41    pub fn jsonl() -> Self {
42        Self {
43            format: Some(JsonFormat::Jsonl),
44            ..Default::default()
45        }
46    }
47
48    /// Force JSON array format.
49    pub fn json_array() -> Self {
50        Self {
51            format: Some(JsonFormat::JsonArray),
52            ..Default::default()
53        }
54    }
55}
56
57/// Borrowed JSON object callback payload.
58pub type JsonObject = serde_json::Map<String, Value>;
59
60/// Summary of a JSON/JSONL scan.
61#[derive(Debug, Clone, PartialEq, Eq)]
62#[non_exhaustive]
63pub struct JsonScanSummary {
64    pub rows_read: usize,
65    pub malformed_lines: usize,
66    pub format: FileFormat,
67    /// The scan stopped at `max_rows` while records still remained. A source
68    /// holding exactly `max_rows` records was read in full, so this stays false.
69    pub truncated: bool,
70}
71
72/// Scan JSON or JSONL input and invoke `on_object` for each object record.
73///
74/// - **JSONL**: scans one top-level JSON value at a time from the reader.
75/// - **JSON array**: streams array elements without buffering the entire input.
76pub fn scan_json_from_reader<R, F>(
77    mut reader: R,
78    config: &JsonParserConfig,
79    mut on_object: F,
80) -> Result<JsonScanSummary, DataProfilerError>
81where
82    R: BufRead,
83    F: FnMut(&JsonObject),
84{
85    let format = match config.format {
86        Some(JsonFormat::JsonArray) => FileFormat::Json,
87        Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
88        None => match consume_leading_whitespace(&mut reader)? {
89            Some(b'[') => FileFormat::Json,
90            _ => FileFormat::Jsonl,
91        },
92    };
93
94    let mut rows_read = 0;
95    let mut malformed_lines = 0;
96    let mut truncated = false;
97
98    match format {
99        FileFormat::Jsonl => loop {
100            if let Some(max) = config.max_rows
101                && rows_read >= max
102            {
103                // Reaching the cap is not truncation unless a record actually
104                // remains: a file with exactly `max_rows` rows was read in full.
105                truncated = consume_leading_whitespace(&mut reader)?.is_some();
106                break;
107            }
108
109            let mut deserializer = serde_json::Deserializer::from_reader(&mut reader);
110            let value: Value = match Value::deserialize(&mut deserializer) {
111                Ok(v) => v,
112                Err(err) if err.classify() == serde_json::error::Category::Eof => break,
113                Err(_) => {
114                    malformed_lines += 1;
115                    skip_to_next_line(&mut reader)?;
116                    continue;
117                }
118            };
119
120            if let Value::Object(ref obj) = value {
121                on_object(obj);
122                rows_read += 1;
123            }
124        },
125        FileFormat::Json => {
126            let mut found_array = false;
127            loop {
128                let mut consume = 0;
129                {
130                    let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
131                    if buf.is_empty() {
132                        break;
133                    }
134                    for &byte in buf {
135                        consume += 1;
136                        if byte == b'[' {
137                            found_array = true;
138                            break;
139                        } else if !byte.is_ascii_whitespace() {
140                            break;
141                        }
142                    }
143                }
144                reader.consume(consume);
145                if found_array || consume == 0 {
146                    break;
147                }
148            }
149
150            if !found_array {
151                if config.format.is_some() {
152                    return Err(DataProfilerError::JsonParsingError {
153                        message: "Expected JSON array (starts with '[') but input does not match"
154                            .to_string(),
155                    });
156                }
157            } else {
158                loop {
159                    let mut consume = 0;
160                    let mut found_value = false;
161                    let mut end_of_array = false;
162
163                    {
164                        let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
165                        if buf.is_empty() {
166                            break;
167                        }
168                        for &byte in buf {
169                            if byte.is_ascii_whitespace() || byte == b',' {
170                                consume += 1;
171                            } else if byte == b']' {
172                                end_of_array = true;
173                                consume += 1;
174                                break;
175                            } else {
176                                found_value = true;
177                                break;
178                            }
179                        }
180                    }
181
182                    reader.consume(consume);
183                    if end_of_array {
184                        break;
185                    }
186
187                    if found_value {
188                        if let Some(max) = config.max_rows
189                            && rows_read >= max
190                        {
191                            // `found_value` means a real element follows, not the
192                            // closing bracket, so the array was genuinely cut short.
193                            truncated = true;
194                            break;
195                        }
196
197                        let mut deserializer = serde_json::Deserializer::from_reader(&mut reader);
198                        match Value::deserialize(&mut deserializer) {
199                            Ok(Value::Object(obj)) => {
200                                on_object(&obj);
201                                rows_read += 1;
202                            }
203                            Ok(_) => {}
204                            Err(_) => {
205                                malformed_lines += 1;
206                                break;
207                            }
208                        }
209                    }
210                }
211            }
212        }
213        _ => unreachable!("json scanner only returns json or jsonl formats"),
214    }
215
216    Ok(JsonScanSummary {
217        rows_read,
218        malformed_lines,
219        format,
220        truncated,
221    })
222}
223
224fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
225    loop {
226        let mut bytes_to_consume = 0;
227        let first_non_whitespace = {
228            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
229            if buf.is_empty() {
230                return Ok(None);
231            }
232
233            let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
234            if first_non_whitespace.is_none() {
235                bytes_to_consume = buf.len();
236            }
237            first_non_whitespace
238        };
239
240        if first_non_whitespace.is_some() {
241            return Ok(first_non_whitespace);
242        }
243
244        reader.consume(bytes_to_consume);
245    }
246}
247
248fn skip_to_next_line<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
249    let mut discarded = Vec::new();
250    reader
251        .read_until(b'\n', &mut discarded)
252        .map_err(DataProfilerError::from)?;
253    Ok(())
254}
255
256/// Convert a JSON [`Value`] to a flat string for column storage.
257fn json_value_to_string(value: &Value) -> String {
258    match value {
259        Value::Null => String::new(),
260        Value::Bool(boolean) => boolean.to_string(),
261        Value::Number(number) => number.to_string(),
262        Value::String(string) => string.to_string(),
263        Value::Array(_) | Value::Object(_) => serde_json::to_string(value).unwrap_or_default(),
264    }
265}
266
267/// Feed a JSON object's fields into a [`StreamingColumnCollection`].
268fn feed_json_object(
269    obj: &JsonObject,
270    known_columns: &mut Vec<String>,
271    known_columns_set: &mut HashSet<String>,
272    column_stats: &mut StreamingColumnCollection,
273) {
274    for key in obj.keys() {
275        if known_columns_set.insert(key.clone()) {
276            known_columns.push(key.clone());
277        }
278    }
279
280    let values: Vec<String> = known_columns
281        .iter()
282        .map(|column| {
283            obj.get(column)
284                .map(json_value_to_string)
285                .unwrap_or_default()
286        })
287        .collect();
288
289    column_stats.process_record(known_columns, values);
290}
291
292/// Analyze JSON/JSONL data from a buffered reader using streaming statistics.
293///
294/// Returns `(column_profiles, streaming_stats, rows_read, malformed_lines, detected_format)`.
295pub fn analyze_json_from_reader<R: BufRead>(
296    reader: R,
297    config: &JsonParserConfig,
298) -> Result<
299    (
300        Vec<ColumnProfile>,
301        StreamingColumnCollection,
302        usize,
303        usize,
304        FileFormat,
305    ),
306    DataProfilerError,
307> {
308    analyze_json_from_reader_with_hints(reader, config, &SemanticHints::default())
309}
310
311pub fn analyze_json_from_reader_with_hints<R: BufRead>(
312    reader: R,
313    config: &JsonParserConfig,
314    semantic_hints: &SemanticHints,
315) -> Result<
316    (
317        Vec<ColumnProfile>,
318        StreamingColumnCollection,
319        usize,
320        usize,
321        FileFormat,
322    ),
323    DataProfilerError,
324> {
325    let (profiles, stats, rows_read, malformed_lines, format, _truncated) =
326        analyze_json_from_reader_full(reader, config, semantic_hints)?;
327    Ok((profiles, stats, rows_read, malformed_lines, format))
328}
329
330/// As [`analyze_json_from_reader_with_hints`], but also reports whether the scan
331/// stopped early. Kept private so the public tuple stays stable.
332#[allow(clippy::type_complexity)]
333fn analyze_json_from_reader_full<R: BufRead>(
334    reader: R,
335    config: &JsonParserConfig,
336    semantic_hints: &SemanticHints,
337) -> Result<
338    (
339        Vec<ColumnProfile>,
340        StreamingColumnCollection,
341        usize,
342        usize,
343        FileFormat,
344        bool,
345    ),
346    DataProfilerError,
347> {
348    let mut column_stats = StreamingColumnCollection::new();
349    let mut known_columns = Vec::new();
350    let mut known_columns_set = HashSet::new();
351
352    let summary = scan_json_from_reader(reader, config, |obj| {
353        feed_json_object(
354            obj,
355            &mut known_columns,
356            &mut known_columns_set,
357            &mut column_stats,
358        );
359    })?;
360
361    let profiles = profile_builder::profiles_from_streaming_with_hints(
362        &column_stats,
363        false,
364        false,
365        None,
366        semantic_hints,
367    );
368
369    Ok((
370        profiles,
371        column_stats,
372        summary.rows_read,
373        summary.malformed_lines,
374        summary.format,
375        summary.truncated,
376    ))
377}
378
379/// Analyze a JSON or JSONL file, returning a full [`ProfileReport`].
380pub fn analyze_json_file(
381    file_path: &Path,
382    config: &JsonParserConfig,
383) -> Result<ProfileReport, DataProfilerError> {
384    analyze_json_file_with_dimensions(file_path, config, None)
385}
386
387/// Like [`analyze_json_file`] but only computes the requested quality dimensions.
388pub fn analyze_json_file_with_dimensions(
389    file_path: &Path,
390    config: &JsonParserConfig,
391    quality_dimensions: Option<&[QualityDimension]>,
392) -> Result<ProfileReport, DataProfilerError> {
393    analyze_json_file_with_dimensions_and_hints(
394        file_path,
395        config,
396        quality_dimensions,
397        &SemanticHints::default(),
398    )
399}
400
401pub fn analyze_json_file_with_dimensions_and_hints(
402    file_path: &Path,
403    config: &JsonParserConfig,
404    quality_dimensions: Option<&[QualityDimension]>,
405    semantic_hints: &SemanticHints,
406) -> Result<ProfileReport, DataProfilerError> {
407    let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
408    let start = std::time::Instant::now();
409
410    let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
411    let buf_reader = std::io::BufReader::new(file);
412
413    let (column_profiles, column_stats, rows_read, malformed_lines, format, truncated) =
414        analyze_json_from_reader_full(buf_reader, config, semantic_hints)?;
415
416    let file_source = DataSource::File {
417        path: file_path.display().to_string(),
418        format,
419        size_bytes: metadata.len(),
420        modified_at: None,
421        parquet_metadata: None,
422    };
423
424    if rows_read == 0 {
425        if malformed_lines > 0 {
426            return Err(DataProfilerError::JsonParsingError {
427                message: "No valid JSON records found in file (malformed JSON encountered)"
428                    .to_string(),
429            });
430        }
431        return Ok(ReportAssembler::new(
432            file_source,
433            ExecutionMetadata::new(0, 0, start.elapsed().as_millis()),
434        )
435        .skip_quality()
436        .build());
437    }
438
439    let sample_columns = profile_builder::quality_check_samples(&column_stats);
440    let scan_time_ms = start.elapsed().as_millis();
441    let num_columns = column_profiles.len();
442
443    let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms);
444    // `truncated` is set only when a record still remained at the cap, so a source
445    // holding exactly `max_rows` records is reported as fully read, not cut short.
446    if truncated && let Some(max) = config.max_rows {
447        execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
448    }
449
450    let mut assembler = ReportAssembler::new(file_source, execution)
451        .columns(column_profiles)
452        .with_quality_data(sample_columns)
453        .with_semantic_hints(semantic_hints.clone());
454    if let Some(dimensions) = quality_dimensions {
455        assembler = assembler.with_requested_dimensions(dimensions.to_vec());
456    }
457    Ok(assembler.build())
458}
459
460fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
461    if error.kind() == std::io::ErrorKind::NotFound {
462        DataProfilerError::FileNotFound {
463            path: file_path.display().to_string(),
464        }
465    } else {
466        DataProfilerError::from(error)
467    }
468}
469
470#[cfg(test)]
471mod tests {
472    use super::*;
473    use std::io::{Cursor, Write};
474    use tempfile::NamedTempFile;
475
476    fn write_file(content: &str) -> NamedTempFile {
477        let mut file = NamedTempFile::new().unwrap();
478        write!(file, "{}", content).unwrap();
479        file.flush().unwrap();
480        file
481    }
482
483    #[test]
484    fn test_scan_jsonl_detects_format_and_rows() {
485        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
486        let mut rows = 0;
487
488        let summary = scan_json_from_reader(
489            Cursor::new(data.as_ref()),
490            &JsonParserConfig::default(),
491            |_| {
492                rows += 1;
493            },
494        )
495        .unwrap();
496
497        assert_eq!(summary.format, FileFormat::Jsonl);
498        assert_eq!(summary.rows_read, 3);
499        assert_eq!(summary.malformed_lines, 0);
500        assert_eq!(rows, 3);
501    }
502
503    #[test]
504    fn test_scan_json_array_detects_format_and_rows() {
505        let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
506        let mut keys = Vec::new();
507
508        let summary = scan_json_from_reader(
509            Cursor::new(data.as_ref()),
510            &JsonParserConfig::default(),
511            |obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
512        )
513        .unwrap();
514
515        assert_eq!(summary.format, FileFormat::Json);
516        assert_eq!(summary.rows_read, 2);
517        assert_eq!(keys.len(), 2);
518        assert_eq!(keys[0], vec!["name".to_string()]);
519    }
520
521    #[test]
522    fn test_scan_respects_max_rows() {
523        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
524        let config = JsonParserConfig::default().with_max_rows(2);
525        let mut rows = 0;
526
527        let summary =
528            scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
529
530        assert_eq!(summary.rows_read, 2);
531        assert_eq!(rows, 2);
532    }
533
534    #[test]
535    fn test_scan_skips_malformed_jsonl_lines() {
536        let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
537        let mut rows = 0;
538
539        let summary = scan_json_from_reader(
540            Cursor::new(data.as_ref()),
541            &JsonParserConfig::jsonl(),
542            |_| {
543                rows += 1;
544            },
545        )
546        .unwrap();
547
548        assert_eq!(summary.format, FileFormat::Jsonl);
549        assert_eq!(summary.rows_read, 2);
550        assert_eq!(summary.malformed_lines, 1);
551        assert_eq!(rows, 2);
552    }
553
554    #[test]
555    fn test_scan_pretty_printed_root_object_is_one_jsonl_row() {
556        let data = br#"{
557    "type": "FeatureCollection",
558    "features": [
559        {"id": 1},
560        {"id": 2}
561  ]
562}"#;
563        let mut object_field_counts = Vec::new();
564
565        let summary = scan_json_from_reader(
566            Cursor::new(data.as_ref()),
567            &JsonParserConfig::default(),
568            |obj| object_field_counts.push(obj.len()),
569        )
570        .unwrap();
571
572        assert_eq!(summary.format, FileFormat::Jsonl);
573        assert_eq!(summary.rows_read, 1);
574        assert_eq!(summary.malformed_lines, 0);
575        assert_eq!(object_field_counts, vec![2]);
576    }
577
578    #[test]
579    fn test_forced_json_array_requires_opening_bracket() {
580        let data = br#"{"x":1}"#;
581        let err = scan_json_from_reader(
582            Cursor::new(data.as_ref()),
583            &JsonParserConfig::json_array(),
584            |_| {},
585        )
586        .expect_err("forced json array should reject non-array input");
587
588        assert!(err.to_string().contains("Expected JSON array"));
589    }
590
591    #[test]
592    fn test_analyze_json_from_reader_jsonl_streaming() {
593        let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
594        let cursor = Cursor::new(data.as_ref());
595        let config = JsonParserConfig::default();
596
597        let (profiles, _stats, rows, _malformed, format) =
598            analyze_json_from_reader(cursor, &config).unwrap();
599        assert_eq!(format, FileFormat::Jsonl);
600        assert_eq!(rows, 3);
601        assert_eq!(profiles.len(), 2);
602    }
603
604    #[test]
605    fn test_analyze_json_from_reader_json_array() {
606        let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
607        let cursor = Cursor::new(data.as_ref());
608        let config = JsonParserConfig::default();
609
610        let (profiles, _stats, rows, _malformed, format) =
611            analyze_json_from_reader(cursor, &config).unwrap();
612        assert_eq!(format, FileFormat::Json);
613        assert_eq!(rows, 2);
614        assert_eq!(profiles.len(), 2);
615    }
616
617    #[test]
618    fn test_analyze_json_from_reader_max_rows() {
619        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
620        let cursor = Cursor::new(data.as_ref());
621        let config = JsonParserConfig::default().with_max_rows(3);
622
623        let (_profiles, _stats, rows, _malformed, _format) =
624            analyze_json_from_reader(cursor, &config).unwrap();
625        assert_eq!(rows, 3);
626    }
627
628    #[test]
629    fn test_analyze_json_from_reader_missing_fields() {
630        let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
631        let cursor = Cursor::new(data.as_ref());
632        let config = JsonParserConfig::jsonl();
633
634        let (profiles, _stats, rows, _malformed, _format) =
635            analyze_json_from_reader(cursor, &config).unwrap();
636        assert_eq!(rows, 2);
637
638        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
639        assert_eq!(col_b.total_count, 2);
640    }
641
642    #[test]
643    fn test_analyze_json_file_quality_report() {
644        let file = write_file(r#"[{"x":1},{"x":2}]"#);
645        let config = JsonParserConfig::default();
646        let report = analyze_json_file(file.path(), &config).unwrap();
647
648        assert_eq!(report.execution.rows_processed, 2);
649        assert_eq!(report.column_profiles.len(), 1);
650        assert!(report.quality_score().unwrap() >= 0.0);
651    }
652
653    #[test]
654    fn test_jsonl_skips_malformed_lines() {
655        let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
656        let cursor = Cursor::new(data.as_ref());
657        let config = JsonParserConfig::jsonl();
658
659        let (profiles, _stats, rows, _malformed, format) =
660            analyze_json_from_reader(cursor, &config).unwrap();
661        assert_eq!(format, FileFormat::Jsonl);
662        assert_eq!(rows, 2);
663        assert_eq!(profiles[0].total_count, 2);
664    }
665
666    #[test]
667    fn test_analyze_json_with_large_leading_whitespace() {
668        let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
669        let cursor = Cursor::new(data.into_bytes());
670        let config = JsonParserConfig::default();
671
672        let (_profiles, _stats, rows, malformed, format) =
673            analyze_json_from_reader(cursor, &config).unwrap();
674        assert_eq!(format, FileFormat::Json);
675        assert_eq!(rows, 1);
676        assert_eq!(malformed, 0);
677    }
678
679    #[test]
680    fn test_analyze_json_array() {
681        let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
682        let config = JsonParserConfig::default();
683        let report = analyze_json_file(json.path(), &config).unwrap();
684        let profiles = &report.column_profiles;
685
686        assert_eq!(profiles.len(), 2);
687        let names: Vec<&str> = profiles
688            .iter()
689            .map(|profile| profile.name.as_str())
690            .collect();
691        assert!(names.contains(&"name"));
692        assert!(names.contains(&"age"));
693
694        let age = profiles
695            .iter()
696            .find(|profile| profile.name == "age")
697            .unwrap();
698        assert_eq!(age.total_count, 2);
699        assert_eq!(age.null_count, 0);
700    }
701
702    #[test]
703    fn test_analyze_jsonl() {
704        let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
705        let config = JsonParserConfig::default();
706        let report = analyze_json_file(jsonl.path(), &config).unwrap();
707        let profiles = &report.column_profiles;
708
709        assert_eq!(profiles.len(), 1);
710        assert_eq!(profiles[0].name, "x");
711        assert_eq!(profiles[0].total_count, 3);
712    }
713
714    #[test]
715    fn test_analyze_json_with_nulls() {
716        let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
717        let config = JsonParserConfig::default();
718        let report = analyze_json_file(json.path(), &config).unwrap();
719        let profiles = &report.column_profiles;
720
721        let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
722        assert_eq!(col_a.null_count, 1);
723
724        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
725        assert_eq!(col_b.null_count, 1);
726    }
727
728    #[test]
729    fn test_analyze_json_with_missing_fields() {
730        let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
731        let config = JsonParserConfig::default();
732        let report = analyze_json_file(json.path(), &config).unwrap();
733        let profiles = &report.column_profiles;
734
735        let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
736        assert_eq!(col_b.total_count, 2);
737    }
738
739    #[test]
740    fn test_analyze_json_empty_array() {
741        let json = write_file("[]");
742        let config = JsonParserConfig::default();
743        let report = analyze_json_file(json.path(), &config).unwrap();
744        assert!(report.column_profiles.is_empty());
745    }
746
747    #[test]
748    fn test_analyze_json_malformed_returns_error() {
749        let json = write_file("this is entirely invalid json");
750        let config = JsonParserConfig::default();
751        let err = analyze_json_file(json.path(), &config)
752            .expect_err("malformed JSON should return an error");
753
754        let message = err.to_string().to_lowercase();
755        assert!(message.contains("malformed") && message.contains("json"));
756    }
757
758    #[test]
759    fn test_analyze_json_file_detects_format() {
760        let json_array = write_file(r#"[{"x":1}]"#);
761        let config = JsonParserConfig::default();
762        let report = analyze_json_file(json_array.path(), &config).unwrap();
763        assert!(matches!(
764            report.data_source,
765            DataSource::File {
766                format: FileFormat::Json,
767                ..
768            }
769        ));
770
771        let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
772        let report = analyze_json_file(jsonl.path(), &config).unwrap();
773        assert!(matches!(
774            report.data_source,
775            DataSource::File {
776                format: FileFormat::Jsonl,
777                ..
778            }
779        ));
780    }
781
782    #[test]
783    fn test_analyze_json_file_empty() {
784        let json = write_file("");
785        let config = JsonParserConfig::default();
786        let report = analyze_json_file(json.path(), &config).unwrap();
787        assert_eq!(report.execution.rows_processed, 0);
788        assert!(report.column_profiles.is_empty());
789    }
790
791    #[test]
792    fn test_analyze_json_boolean_and_nested() {
793        let json =
794            write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
795        let config = JsonParserConfig::default();
796        let report = analyze_json_file(json.path(), &config).unwrap();
797        let profiles = &report.column_profiles;
798
799        let flag = profiles
800            .iter()
801            .find(|profile| profile.name == "flag")
802            .unwrap();
803        assert_eq!(flag.total_count, 2);
804
805        let nested = profiles
806            .iter()
807            .find(|profile| profile.name == "nested")
808            .unwrap();
809        assert_eq!(nested.total_count, 2);
810    }
811
812    #[test]
813    fn test_single_root_object_compact_yields_one_row() {
814        let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
815        let cursor = Cursor::new(data.as_ref());
816        let config = JsonParserConfig::default();
817
818        let (profiles, _stats, rows, malformed, _format) =
819            analyze_json_from_reader(cursor, &config).unwrap();
820        assert_eq!(rows, 1);
821        assert_eq!(malformed, 0);
822        assert_eq!(profiles.len(), 2);
823
824        let names: Vec<&str> = profiles
825            .iter()
826            .map(|profile| profile.name.as_str())
827            .collect();
828        assert!(names.contains(&"type"));
829        assert!(names.contains(&"features"));
830    }
831
832    #[test]
833    fn test_jsonl_multi_object_still_works() {
834        let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
835        let cursor = Cursor::new(data.as_ref());
836        let config = JsonParserConfig::default();
837
838        let (_profiles, _stats, rows, _malformed, format) =
839            analyze_json_from_reader(cursor, &config).unwrap();
840        assert_eq!(format, FileFormat::Jsonl);
841        assert_eq!(rows, 3);
842    }
843
844    #[test]
845    fn test_single_root_object_via_analyze_json_file() {
846        let file = write_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
847        let config = JsonParserConfig::default();
848        let report = analyze_json_file(file.path(), &config).unwrap();
849
850        assert_eq!(report.execution.rows_processed, 1);
851        assert_eq!(report.column_profiles.len(), 2);
852    }
853
854    #[test]
855    fn test_single_root_object_pretty_printed_yields_one_row() {
856        let data = br#"{
857  "type": "FeatureCollection",
858  "features": [
859    {"id": 1},
860    {"id": 2}
861  ]
862}"#;
863        let cursor = Cursor::new(data.as_ref());
864        let config = JsonParserConfig::default();
865
866        let (profiles, _stats, rows, malformed, format) =
867            analyze_json_from_reader(cursor, &config).unwrap();
868        assert_eq!(format, FileFormat::Jsonl);
869        assert_eq!(rows, 1);
870        assert_eq!(malformed, 0);
871        assert_eq!(profiles.len(), 2);
872    }
873
874    #[test]
875    fn test_single_root_object_pretty_printed_via_analyze_json_file() {
876        let file = write_file(
877            "{\n  \"type\": \"FeatureCollection\",\n  \"features\": [\n    {\"id\": 1},\n    {\"id\": 2}\n  ]\n}\n",
878        );
879        let config = JsonParserConfig::default();
880        let report = analyze_json_file(file.path(), &config).unwrap();
881
882        assert_eq!(report.execution.rows_processed, 1);
883        assert_eq!(report.column_profiles.len(), 2);
884    }
885}