1use std::collections::{HashMap, HashSet};
30use std::io::{BufRead, Read};
31use std::path::Path;
32
33pub use dataprof_core::JsonErrorPolicy;
34
35#[doc(hidden)]
36pub mod record;
37use dataprof_core::{
38 AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat,
39 TruncationReason, Utf8BomReader,
40};
41use dataprof_runtime::{
42 ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
43};
44use record::{
45 JsonRecord, drain_to_end, json_document_error, json_value_kind, json_value_text,
46 malformed_array_error, non_object_record_error, object_record, read_json_record,
47 read_jsonl_line, rest_is_white_space, skip_ascii_whitespace, skip_leading_white_space,
48 wide_integer_record_error,
49};
50use serde_json::Value;
51
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub enum JsonFormat {
60 Json,
65 Jsonl,
68}
69
70#[derive(Debug, Clone, Default)]
104pub struct JsonParserConfig {
105 pub format: Option<JsonFormat>,
112 pub max_rows: Option<usize>,
114 pub error_policy: JsonErrorPolicy,
116}
117
118impl JsonParserConfig {
119 pub fn with_max_rows(mut self, max_rows: usize) -> Self {
156 self.max_rows = Some(max_rows);
157 self
158 }
159
160 pub fn with_error_policy(mut self, policy: JsonErrorPolicy) -> Self {
188 self.error_policy = policy;
189 self
190 }
191
192 pub fn jsonl() -> Self {
194 Self {
195 format: Some(JsonFormat::Jsonl),
196 ..Default::default()
197 }
198 }
199
200 pub fn json_document() -> Self {
203 Self {
204 format: Some(JsonFormat::Json),
205 ..Default::default()
206 }
207 }
208
209 pub fn with_format(mut self, format: JsonFormat) -> Self {
211 self.format = Some(format);
212 self
213 }
214}
215
216pub type JsonObject = serde_json::Map<String, Value>;
223
224#[derive(Debug, Clone, PartialEq, Eq)]
226#[non_exhaustive]
227pub struct JsonScanSummary {
228 pub rows_read: usize,
229 pub malformed_lines: usize,
233 pub format: FileFormat,
234 pub truncated: bool,
237 pub unread_after_fault: bool,
242}
243
244pub fn scan_json_from_reader<R, F>(
329 reader: R,
330 config: &JsonParserConfig,
331 mut on_object: F,
332) -> Result<JsonScanSummary, DataProfilerError>
333where
334 R: BufRead,
335 F: FnMut(&JsonObject),
336{
337 let mut reader = Utf8BomReader::new(reader).map_err(DataProfilerError::from)?;
338 let leading = match config.format {
343 Some(JsonFormat::Jsonl) => None,
344 _ => Some(skip_leading_white_space(&mut reader)?),
345 };
346 let first = leading.as_ref().and_then(|leading| leading.first);
347 let pending = leading.map(|leading| leading.pending).unwrap_or_default();
350 let mut reader = std::io::Cursor::new(pending).chain(reader);
351 let format = match config.format {
352 Some(JsonFormat::Json) => FileFormat::Json,
353 Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
354 None if first == Some(b'[') => FileFormat::Json,
358 None => FileFormat::Jsonl,
359 };
360
361 let mut rows_read = 0;
362 let mut malformed_lines = 0;
363 let mut unread_after_fault = false;
364 let mut truncated = false;
365
366 match format {
367 FileFormat::Jsonl => {
373 let mut line = String::new();
374 let mut line_number = 0;
375 loop {
376 if let Some(max) = config.max_rows
377 && rows_read >= max
378 {
379 truncated = next_record_line(&mut reader, &mut line, &mut line_number)?;
383 break;
384 }
385
386 if !next_record_line(&mut reader, &mut line, &mut line_number)? {
387 break;
388 }
389
390 match read_jsonl_line(&line) {
391 JsonRecord::Object(obj) => {
392 on_object(&obj);
393 rows_read += 1;
394 }
395 JsonRecord::NonObject(kind) => {
396 if config.error_policy == JsonErrorPolicy::Strict {
397 return Err(non_object_record_error(
398 kind,
399 rows_read + malformed_lines + 1,
400 ));
401 }
402 malformed_lines += 1;
403 }
404 JsonRecord::Malformed(err) => {
405 if config.error_policy == JsonErrorPolicy::Strict {
406 return Err(record::malformed_jsonl_record_error(
407 line_number,
408 &line,
409 err,
410 ));
411 }
412 malformed_lines += 1;
413 }
414 JsonRecord::WideInteger => {
415 if config.error_policy == JsonErrorPolicy::Strict {
416 return Err(wide_integer_record_error(&format!(
417 "on line {line_number}"
418 )));
419 }
420 malformed_lines += 1;
421 }
422 }
423 }
424 }
425 FileFormat::Json => {
426 let found_array = first == Some(b'[');
430
431 if !found_array {
432 let (object_rows, object_malformed, object_truncated) =
436 scan_single_json_object(&mut reader, config, &mut on_object)?;
437 rows_read += object_rows;
438 malformed_lines += object_malformed;
439 truncated |= object_truncated;
440 } else {
441 consume_peeked(&mut reader)?;
443
444 let mut expect_value = true;
445 let mut allow_end = true;
446 let mut array_closed = false;
447 let mut drain_remainder = false;
448
449 loop {
450 let Some(next) = consume_leading_whitespace(&mut reader)? else {
451 if config.error_policy == JsonErrorPolicy::Strict {
452 return Err(malformed_array_error(
453 "unexpected end of input before closing ']'",
454 ));
455 }
456 malformed_lines += 1;
457 drain_remainder = true;
458 break;
459 };
460
461 if expect_value {
462 if next == b']' {
463 if !allow_end {
464 if config.error_policy == JsonErrorPolicy::Strict {
465 return Err(malformed_array_error(
466 "trailing comma before closing ']'",
467 ));
468 }
469 malformed_lines += 1;
470 drain_remainder = true;
471 break;
472 }
473 consume_peeked(&mut reader)?;
474 array_closed = true;
475 break;
476 }
477
478 if let Some(max) = config.max_rows
479 && rows_read >= max
480 {
481 truncated = true;
484 break;
485 }
486
487 if next == b',' {
488 if config.error_policy == JsonErrorPolicy::Strict {
489 return Err(malformed_array_error(
490 "unexpected comma where an array value was required",
491 ));
492 }
493 malformed_lines += 1;
494 drain_remainder = true;
495 break;
496 }
497
498 match read_json_record(&mut reader, next)? {
499 JsonRecord::Object(obj) => {
500 on_object(&obj);
501 rows_read += 1;
502 }
503 JsonRecord::NonObject(kind) => {
504 if config.error_policy == JsonErrorPolicy::Strict {
505 return Err(non_object_record_error(
506 kind,
507 rows_read + malformed_lines + 1,
508 ));
509 }
510 malformed_lines += 1;
514 }
515 JsonRecord::Malformed(err) => {
516 if config.error_policy == JsonErrorPolicy::Strict {
517 return Err(malformed_record_error(err));
518 }
519 malformed_lines += 1;
520 drain_remainder = true;
521 break;
522 }
523 JsonRecord::WideInteger => {
524 if config.error_policy == JsonErrorPolicy::Strict {
525 return Err(wide_integer_record_error(&format!(
526 "at position {}",
527 rows_read + malformed_lines + 1
528 )));
529 }
530 malformed_lines += 1;
532 }
533 }
534 expect_value = false;
535 } else {
536 match next {
537 b',' => {
538 consume_peeked(&mut reader)?;
539 expect_value = true;
540 allow_end = false;
541 }
542 b']' => {
543 consume_peeked(&mut reader)?;
544 array_closed = true;
545 break;
546 }
547 _ => {
548 if config.error_policy == JsonErrorPolicy::Strict {
549 return Err(malformed_array_error(
550 "expected ',' or ']' after an array value",
551 ));
552 }
553 malformed_lines += 1;
554 drain_remainder = true;
555 break;
556 }
557 }
558 }
559 }
560
561 if drain_remainder {
562 unread_after_fault = !rest_is_white_space(&mut reader)?.0;
565 drain_to_end(&mut reader)?;
566 } else if array_closed && !rest_is_white_space(&mut reader)?.0 {
567 if config.error_policy == JsonErrorPolicy::Strict {
568 return Err(malformed_array_error(
569 "non-whitespace content follows the closing ']'",
570 ));
571 }
572 malformed_lines += 1;
573 drain_to_end(&mut reader)?;
574 }
575 }
576 }
577 _ => unreachable!("json scanner only returns json or jsonl formats"),
578 }
579
580 Ok(JsonScanSummary {
581 rows_read,
582 malformed_lines,
583 format,
584 truncated,
585 unread_after_fault,
586 })
587}
588
589fn next_record_line<R: BufRead>(
596 reader: &mut R,
597 line: &mut String,
598 line_number: &mut usize,
599) -> Result<bool, DataProfilerError> {
600 loop {
601 line.clear();
602 let read = reader.read_line(line).map_err(DataProfilerError::from)?;
603 if read == 0 {
604 return Ok(false);
605 }
606 *line_number += 1;
607 if !line.trim().is_empty() {
608 return Ok(true);
609 }
610 }
611}
612
613fn scan_single_json_object<R, F>(
620 reader: &mut R,
621 config: &JsonParserConfig,
622 on_object: &mut F,
623) -> Result<(usize, usize, bool), DataProfilerError>
624where
625 R: BufRead,
626 F: FnMut(&JsonObject),
627{
628 let mut text = String::new();
629 reader
630 .read_to_string(&mut text)
631 .map_err(DataProfilerError::from)?;
632
633 if text.trim().is_empty() {
634 return Ok((0, 0, false));
635 }
636
637 if config.max_rows == Some(0) {
642 return Ok((0, 0, true));
643 }
644
645 match serde_json::from_str::<Value>(text.trim()) {
646 Ok(Value::Object(obj)) => match object_record(obj, text.as_bytes()) {
647 JsonRecord::Object(obj) => {
648 on_object(&obj);
649 Ok((1, 0, false))
650 }
651 _ => {
652 if config.error_policy == JsonErrorPolicy::Strict {
653 return Err(wide_integer_record_error("at position 1"));
654 }
655 Ok((0, 1, false))
656 }
657 },
658 Ok(value) => {
659 if config.error_policy == JsonErrorPolicy::Strict {
660 return Err(non_object_record_error(json_value_kind(&value), 1));
661 }
662 Ok((0, 1, false))
663 }
664 Err(err) => {
665 if config.error_policy == JsonErrorPolicy::Strict {
666 return Err(json_document_error(err));
667 }
668 Ok((0, 1, false))
669 }
670 }
671}
672
673fn consume_peeked<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
682 skip_ascii_whitespace(reader)?;
683 reader.consume(1);
684 Ok(())
685}
686
687fn malformed_record_error(err: serde_json::Error) -> DataProfilerError {
691 let message = format!("malformed JSON record: {err}");
692 DataProfilerError::json_parsing_with_source(message, err)
693}
694
695fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
696 loop {
697 let mut bytes_to_consume = 0;
698 let first_non_whitespace = {
699 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
700 if buf.is_empty() {
701 return Ok(None);
702 }
703
704 let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
705 if first_non_whitespace.is_none() {
706 bytes_to_consume = buf.len();
707 }
708 first_non_whitespace
709 };
710
711 if first_non_whitespace.is_some() {
712 return Ok(first_non_whitespace);
713 }
714
715 reader.consume(bytes_to_consume);
716 }
717}
718
719fn feed_json_object(
729 obj: &JsonObject,
730 prior_rows: usize,
731 known_columns: &mut Vec<String>,
732 known_columns_set: &mut HashSet<String>,
733 column_stats: &mut StreamingColumnCollection,
734 container_counts: &mut HashMap<String, usize>,
735) {
736 for (key, value) in obj {
737 if known_columns_set.insert(key.clone()) {
738 known_columns.push(key.clone());
739 column_stats.init_column_with_missing(key, prior_rows);
740 }
741 if matches!(value, Value::Array(_) | Value::Object(_)) {
742 *container_counts.entry(key.clone()).or_default() += 1;
743 }
744 }
745
746 let values: Vec<String> = known_columns
747 .iter()
748 .map(|column| {
749 obj.get(column).map(json_value_text).unwrap_or_default()
752 })
753 .collect();
754 let containers: Vec<usize> = known_columns
755 .iter()
756 .enumerate()
757 .filter(|(_, column)| matches!(obj.get(*column), Some(Value::Array(_) | Value::Object(_))))
758 .map(|(position, _)| position)
759 .collect();
760
761 column_stats.process_record_with_containers(known_columns, values, &containers);
762}
763
764pub fn analyze_json_from_reader<R: BufRead>(
806 reader: R,
807 config: &JsonParserConfig,
808) -> Result<
809 (
810 Vec<ColumnProfile>,
811 StreamingColumnCollection,
812 usize,
813 usize,
814 FileFormat,
815 ),
816 DataProfilerError,
817> {
818 let options = AnalysisOptions::default();
819 let (profiles, stats, rows_read, malformed_lines, format, _truncated, _unread) =
820 analyze_json_from_reader_full(reader, config, &options)?;
821 Ok((profiles, stats, rows_read, malformed_lines, format))
822}
823
824#[allow(clippy::type_complexity)]
829fn analyze_json_from_reader_full<R: BufRead>(
830 reader: R,
831 config: &JsonParserConfig,
832 options: &AnalysisOptions,
833) -> Result<
834 (
835 Vec<ColumnProfile>,
836 StreamingColumnCollection,
837 usize,
838 usize,
839 FileFormat,
840 bool,
841 bool,
842 ),
843 DataProfilerError,
844> {
845 let semantic_hints = options.semantic_hints();
846 let mut column_stats = StreamingColumnCollection::new()
847 .with_semantic_hints(semantic_hints)
848 .with_duplicate_rows_exact_limit(options.duplicate_rows_exact_limit());
849 let mut known_columns = Vec::new();
850 let mut known_columns_set = HashSet::new();
851 let mut container_counts = HashMap::new();
852 let mut rows_seen = 0;
853
854 let summary = scan_json_from_reader(reader, config, |obj| {
855 feed_json_object(
856 obj,
857 rows_seen,
858 &mut known_columns,
859 &mut known_columns_set,
860 &mut column_stats,
861 &mut container_counts,
862 );
863 rows_seen += 1;
864 })?;
865
866 if let Some(indices) = options.column_indices(&column_stats.column_names())? {
867 let available = column_stats.column_names();
868 let selected = indices
869 .into_iter()
870 .map(|index| available[index].clone())
871 .collect::<Vec<_>>();
872 column_stats.retain_columns(&selected);
873 }
874
875 let profiles = profile_builder::mark_container_columns(
876 profile_builder::profiles_from_streaming_with_hints(
877 &column_stats,
878 !options.include_statistics(),
879 !options.include_patterns(),
880 options.locale(),
881 semantic_hints,
882 ),
883 &container_counts,
884 );
885
886 Ok((
887 profiles,
888 column_stats,
889 summary.rows_read,
890 summary.malformed_lines,
891 summary.format,
892 summary.truncated,
893 summary.unread_after_fault,
894 ))
895}
896
897pub fn analyze_json_file(
925 file_path: &Path,
926 config: &JsonParserConfig,
927) -> Result<ProfileReport, DataProfilerError> {
928 analyze_json_file_with_options(file_path, config, &AnalysisOptions::default())
929}
930
931pub fn analyze_json_file_with_options(
937 file_path: &Path,
938 config: &JsonParserConfig,
939 options: &AnalysisOptions,
940) -> Result<ProfileReport, DataProfilerError> {
941 let mut config = config.clone();
944 if config.format.is_none() {
945 config.format = grammar_for_extension(file_path);
946 }
947 let config = &config;
948 let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
949 dataprof_core::binary_input::reject_binary_input(file_path)?;
950 let start = std::time::Instant::now();
951
952 let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
953 let buf_reader = std::io::BufReader::new(file);
954
955 let (
956 column_profiles,
957 column_stats,
958 rows_read,
959 malformed_lines,
960 format,
961 truncated,
962 unread_after_fault,
963 ) = analyze_json_from_reader_full(buf_reader, config, options)?;
964 let read_as_json_document = format == FileFormat::Json;
965
966 let file_source = DataSource::File {
967 path: file_path.display().to_string(),
968 format,
969 size_bytes: metadata.len(),
970 modified_at: None,
971 parquet_metadata: None,
972 csv_metadata: None,
973 };
974
975 if rows_read == 0 {
976 if malformed_lines > 0 {
977 let hint = if read_as_json_document {
981 ". A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
982 } else {
983 ""
984 };
985 return Err(DataProfilerError::json_parsing_error(&format!(
988 "No valid JSON records found in file \
989 (every record was malformed, not a JSON object, or held an integer outside the 64-bit range){hint}"
990 )));
991 }
992 let mut execution =
997 ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("json");
998 if truncated && let Some(max) = config.max_rows {
999 execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1000 }
1001 return Ok(ReportAssembler::new(file_source, execution)
1002 .columns(column_profiles)
1003 .with_quality_data(HashMap::new())
1004 .with_analysis_options(options)
1005 .build());
1006 }
1007
1008 let sample_columns = profile_builder::quality_check_samples(&column_stats);
1009 let scan_time_ms = start.elapsed().as_millis();
1010 let num_columns = column_profiles.len();
1011
1012 let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms)
1013 .with_engine("json")
1014 .with_error_count(malformed_lines);
1017 if truncated && let Some(max) = config.max_rows {
1020 execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1021 }
1022 if unread_after_fault {
1023 execution = execution.with_source_exhausted(false);
1024 }
1025
1026 Ok(ReportAssembler::new(file_source, execution)
1027 .columns(column_profiles)
1028 .with_quality_data(sample_columns)
1029 .with_row_duplicates(column_stats.row_duplicate_summary())
1030 .with_row_completeness(column_stats.row_completeness_summary())
1031 .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
1032 .with_analysis_options(options)
1033 .build())
1034}
1035
1036fn grammar_for_extension(file_path: &Path) -> Option<JsonFormat> {
1044 let extension = file_path.extension().and_then(|ext| ext.to_str())?;
1045 if extension.eq_ignore_ascii_case("json") {
1046 Some(JsonFormat::Json)
1047 } else if extension.eq_ignore_ascii_case("jsonl") || extension.eq_ignore_ascii_case("ndjson") {
1048 Some(JsonFormat::Jsonl)
1049 } else {
1050 None
1051 }
1052}
1053
1054fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
1055 if error.kind() == std::io::ErrorKind::NotFound {
1056 DataProfilerError::FileNotFound {
1057 path: file_path.display().to_string(),
1058 }
1059 } else {
1060 DataProfilerError::from(error)
1061 }
1062}
1063
1064#[cfg(test)]
1065mod tests {
1066 use super::*;
1067 use std::io::{Cursor, Write};
1068 use tempfile::NamedTempFile;
1069
1070 fn write_file(content: &str) -> NamedTempFile {
1073 let mut file = NamedTempFile::new().unwrap();
1074 write!(file, "{}", content).unwrap();
1075 file.flush().unwrap();
1076 file
1077 }
1078
1079 fn write_json_file(content: &str) -> NamedTempFile {
1082 let mut file = NamedTempFile::with_suffix(".json").unwrap();
1083 write!(file, "{}", content).unwrap();
1084 file.flush().unwrap();
1085 file
1086 }
1087
1088 fn write_bytes(content: &[u8]) -> NamedTempFile {
1089 let mut file = NamedTempFile::new().unwrap();
1090 file.write_all(content).unwrap();
1091 file.flush().unwrap();
1092 file
1093 }
1094
1095 #[test]
1096 fn test_scan_jsonl_detects_format_and_rows() {
1097 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1098 let mut rows = 0;
1099
1100 let summary = scan_json_from_reader(
1101 Cursor::new(data.as_ref()),
1102 &JsonParserConfig::default(),
1103 |_| {
1104 rows += 1;
1105 },
1106 )
1107 .unwrap();
1108
1109 assert_eq!(summary.format, FileFormat::Jsonl);
1110 assert_eq!(summary.rows_read, 3);
1111 assert_eq!(summary.malformed_lines, 0);
1112 assert_eq!(rows, 3);
1113 }
1114
1115 #[test]
1116 fn test_scan_json_array_detects_format_and_rows() {
1117 let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
1118 let mut keys = Vec::new();
1119
1120 let summary = scan_json_from_reader(
1121 Cursor::new(data.as_ref()),
1122 &JsonParserConfig::default(),
1123 |obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
1124 )
1125 .unwrap();
1126
1127 assert_eq!(summary.format, FileFormat::Json);
1128 assert_eq!(summary.rows_read, 2);
1129 assert_eq!(keys.len(), 2);
1130 assert_eq!(keys[0], vec!["name".to_string()]);
1131 }
1132
1133 #[test]
1134 fn test_scan_accepts_one_leading_utf8_bom_for_json_and_jsonl() {
1135 for (data, config, expected_format) in [
1136 (
1137 b"\xEF\xBB\xBF[{\"x\":1},{\"x\":2}]".as_slice(),
1138 JsonParserConfig::default(),
1139 FileFormat::Json,
1140 ),
1141 (
1142 b"\xEF\xBB\xBF{\"x\":1}\n{\"x\":2}\n".as_slice(),
1143 JsonParserConfig::default(),
1144 FileFormat::Jsonl,
1145 ),
1146 ] {
1147 let mut rows = 0;
1148 let summary = scan_json_from_reader(Cursor::new(data), &config, |_| rows += 1)
1149 .expect("leading UTF-8 BOM should be accepted");
1150
1151 assert_eq!(rows, 2);
1152 assert_eq!(summary.rows_read, 2);
1153 assert_eq!(summary.malformed_lines, 0);
1154 assert_eq!(summary.format, expected_format);
1155 }
1156 }
1157
1158 #[test]
1159 fn test_scan_does_not_strip_nonleading_or_second_utf8_bom() {
1160 let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1161
1162 for data in [
1163 b" \xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1164 b"\xEF\xBB\xBF\xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1165 b"{\"x\":1}\n\xEF\xBB\xBF{\"x\":2}\n".as_slice(),
1166 ] {
1167 scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1168 .expect_err("only the BOM at absolute byte offset zero may be stripped");
1169 }
1170 }
1171
1172 #[test]
1173 fn test_bom_only_input_matches_empty_input_policy() {
1174 for config in [JsonParserConfig::default(), JsonParserConfig::jsonl()] {
1175 let empty = scan_json_from_reader(Cursor::new(b""), &config, |_| {}).unwrap();
1176 let bom =
1177 scan_json_from_reader(Cursor::new(b"\xEF\xBB\xBF".as_slice()), &config, |_| {})
1178 .unwrap();
1179 assert_eq!(bom, empty);
1180 }
1181 }
1182
1183 #[test]
1184 fn test_json_array_container_grammar_obeys_error_policy() {
1185 let cases = [
1186 (br#"[{"x":1}"#.as_ref(), 1, "missing closing bracket"),
1187 (br#"[{"x":1},"#.as_ref(), 1, "comma followed by EOF"),
1188 (br#"[{"x":1} {"x":2}]"#.as_ref(), 1, "missing comma"),
1189 (br#"[{"x":1},,{"x":2}]"#.as_ref(), 1, "doubled comma"),
1190 (br#"[,{"x":1}]"#.as_ref(), 0, "leading comma"),
1191 (br#"[{"x":1},]"#.as_ref(), 1, "trailing comma"),
1192 (br#"[{"x":1}] trailing"#.as_ref(), 1, "trailing content"),
1193 (
1194 br#"[{"x":1}][{"x":2}]"#.as_ref(),
1195 1,
1196 "second top-level value",
1197 ),
1198 ];
1199
1200 for (data, expected_rows, case) in cases {
1201 let skip = JsonParserConfig::json_document();
1202 let summary =
1203 scan_json_from_reader(Cursor::new(data), &skip, |_| {}).expect("tolerant scan");
1204 assert_eq!(summary.rows_read, expected_rows, "{case}");
1205 assert_eq!(summary.malformed_lines, 1, "{case}");
1206
1207 let strict =
1208 JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict);
1209 let err = scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1210 .expect_err("strict mode must reject invalid array grammar");
1211 assert!(
1212 err.to_string()
1213 .to_lowercase()
1214 .contains("malformed json array"),
1215 "{case}: {err}"
1216 );
1217 }
1218 }
1219
1220 #[test]
1221 fn test_json_array_max_rows_does_not_validate_unread_values() {
1222 let data = br#"[{"x":1},not-valid-json"#;
1223 let config = JsonParserConfig::json_document().with_max_rows(1);
1224 let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1225
1226 assert_eq!(summary.rows_read, 1);
1227 assert_eq!(summary.malformed_lines, 0);
1228 assert!(summary.truncated);
1229 }
1230
1231 #[test]
1232 fn test_scan_respects_max_rows() {
1233 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
1234 let config = JsonParserConfig::default().with_max_rows(2);
1235 let mut rows = 0;
1236
1237 let summary =
1238 scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
1239
1240 assert_eq!(summary.rows_read, 2);
1241 assert_eq!(rows, 2);
1242 }
1243
1244 #[test]
1245 fn test_scan_skips_malformed_jsonl_lines() {
1246 let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
1247 let mut rows = 0;
1248
1249 let summary = scan_json_from_reader(
1250 Cursor::new(data.as_ref()),
1251 &JsonParserConfig::jsonl(),
1252 |_| {
1253 rows += 1;
1254 },
1255 )
1256 .unwrap();
1257
1258 assert_eq!(summary.format, FileFormat::Jsonl);
1259 assert_eq!(summary.rows_read, 2);
1260 assert_eq!(summary.malformed_lines, 1);
1261 assert_eq!(rows, 2);
1262 }
1263
1264 #[test]
1270 fn test_pretty_printed_root_object_is_one_row_under_the_json_grammar() {
1271 let data = br#"{
1272 "type": "FeatureCollection",
1273 "features": [
1274 {"id": 1},
1275 {"id": 2}
1276 ]
1277}"#;
1278 let mut object_field_counts = Vec::new();
1279
1280 let summary = scan_json_from_reader(
1281 Cursor::new(data.as_ref()),
1282 &JsonParserConfig::json_document(),
1283 |obj| object_field_counts.push(obj.len()),
1284 )
1285 .unwrap();
1286
1287 assert_eq!(summary.format, FileFormat::Json);
1288 assert_eq!(summary.rows_read, 1);
1289 assert_eq!(summary.malformed_lines, 0);
1290 assert_eq!(object_field_counts, vec![2]);
1291 }
1292
1293 #[test]
1294 fn test_pretty_printed_root_object_is_not_valid_jsonl() {
1295 let data = br#"{
1296 "type": "FeatureCollection"
1297}"#;
1298 let err = scan_json_from_reader(
1299 Cursor::new(data.as_ref()),
1300 &JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict),
1301 |_| {},
1302 )
1303 .expect_err("a record spanning lines is not JSONL");
1304
1305 assert!(
1306 err.to_string().contains("malformed JSON record"),
1307 "{err}, expected the malformed category"
1308 );
1309 }
1310
1311 const NON_OBJECT_VALUES: &[(&str, &str)] = &[
1315 ("null", "null"),
1316 ("boolean", "true"),
1317 ("number", "42"),
1318 ("number", "-1.5e3"),
1319 ("string", r#""text""#),
1320 ("array", "[1, 2]"),
1321 ];
1322
1323 #[test]
1324 fn test_jsonl_counts_non_object_records_and_keeps_scanning() {
1325 for (kind, value) in NON_OBJECT_VALUES {
1326 let data = format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n");
1327 let mut rows = 0;
1328
1329 let summary = scan_json_from_reader(
1330 Cursor::new(data.as_bytes()),
1331 &JsonParserConfig::jsonl(),
1332 |_| rows += 1,
1333 )
1334 .unwrap();
1335
1336 assert_eq!(summary.rows_read, 2, "{kind}: record after it was lost");
1337 assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1338 assert_eq!(rows, 2, "{kind}");
1339 }
1340 }
1341
1342 #[test]
1343 fn test_json_array_counts_non_object_elements_and_keeps_scanning() {
1344 for (kind, value) in NON_OBJECT_VALUES {
1345 let data = format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]");
1346 let mut rows = 0;
1347
1348 let summary = scan_json_from_reader(
1349 Cursor::new(data.as_bytes()),
1350 &JsonParserConfig::json_document(),
1351 |_| rows += 1,
1352 )
1353 .unwrap();
1354
1355 assert_eq!(summary.rows_read, 2, "{kind}: element after it was lost");
1356 assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1357 assert_eq!(rows, 2, "{kind}");
1358 }
1359 }
1360
1361 #[test]
1362 fn test_strict_policy_rejects_the_first_non_object_record() {
1363 for (kind, value) in NON_OBJECT_VALUES {
1364 for (config, data) in [
1365 (
1366 JsonParserConfig::jsonl(),
1367 format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n"),
1368 ),
1369 (
1370 JsonParserConfig::json_document(),
1371 format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]"),
1372 ),
1373 ] {
1374 let config = config.with_error_policy(JsonErrorPolicy::Strict);
1375 let err = scan_json_from_reader(Cursor::new(data.as_bytes()), &config, |_| {})
1376 .expect_err("{kind}: strict mode must reject a non-object record");
1377
1378 let message = err.to_string();
1379 assert!(message.contains("non-object JSON record"), "{message}");
1380 assert!(message.contains("at position 2"), "{message}");
1381 assert!(message.contains(&format!("found {kind}")), "{message}");
1382 }
1383 }
1384 }
1385
1386 #[test]
1387 fn test_number_element_does_not_swallow_the_array_delimiter() {
1388 let data = br#"[{"id":1}, 1, 2, 3, {"id":2}, 4, {"id":3}]"#;
1391 let mut rows = 0;
1392
1393 let summary = scan_json_from_reader(
1394 Cursor::new(data.as_ref()),
1395 &JsonParserConfig::json_document(),
1396 |_| rows += 1,
1397 )
1398 .unwrap();
1399
1400 assert_eq!(summary.rows_read, 3);
1401 assert_eq!(summary.malformed_lines, 4);
1402 assert_eq!(rows, 3);
1403 }
1404
1405 #[test]
1406 fn test_number_shaped_garbage_stays_a_malformed_record() {
1407 let data = b"{\"id\":1}\n1.2.3\n";
1410 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1411 let err = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {})
1412 .expect_err("number-shaped garbage should be malformed");
1413
1414 assert!(
1415 err.to_string().contains("malformed JSON record"),
1416 "{err}, expected the malformed category"
1417 );
1418 }
1419
1420 #[test]
1421 fn test_input_of_only_non_object_records_fails() {
1422 let file = write_file("\"just a string\"\n1\n[2]\n");
1423 let err = analyze_json_file(file.path(), &JsonParserConfig::jsonl())
1424 .expect_err("nothing profileable should not return a clean empty profile");
1425
1426 assert!(
1427 err.to_string().contains("No valid JSON records found"),
1428 "{err}"
1429 );
1430 }
1431
1432 #[test]
1433 fn test_json_grammar_reads_a_bare_object_as_one_row() {
1436 let data = br#"{"x":1}"#;
1437 let mut rows = 0;
1438 let summary = scan_json_from_reader(
1439 Cursor::new(data.as_ref()),
1440 &JsonParserConfig::json_document(),
1441 |_| rows += 1,
1442 )
1443 .unwrap();
1444
1445 assert_eq!(summary.rows_read, 1);
1446 assert_eq!(summary.malformed_lines, 0);
1447 assert_eq!(rows, 1);
1448 }
1449
1450 #[test]
1451 fn test_json_grammar_rejects_concatenated_objects() {
1455 let data = br#"{"x":1}{"x":2}"#;
1456 let err = scan_json_from_reader(
1457 Cursor::new(data.as_ref()),
1458 &JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict),
1459 |_| {},
1460 )
1461 .expect_err("two objects are not one JSON document");
1462
1463 let message = err.to_string();
1464 assert!(message.contains("malformed JSON document"), "{message}");
1465 assert!(message.contains("format=\"jsonl\""), "{message}");
1466 }
1467
1468 #[test]
1469 fn test_analyze_json_from_reader_jsonl_streaming() {
1470 let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
1471 let cursor = Cursor::new(data.as_ref());
1472 let config = JsonParserConfig::default();
1473
1474 let (profiles, _stats, rows, _malformed, format) =
1475 analyze_json_from_reader(cursor, &config).unwrap();
1476 assert_eq!(format, FileFormat::Jsonl);
1477 assert_eq!(rows, 3);
1478 assert_eq!(profiles.len(), 2);
1479 }
1480
1481 #[test]
1482 fn test_analyze_json_from_reader_json_array() {
1483 let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
1484 let cursor = Cursor::new(data.as_ref());
1485 let config = JsonParserConfig::default();
1486
1487 let (profiles, _stats, rows, _malformed, format) =
1488 analyze_json_from_reader(cursor, &config).unwrap();
1489 assert_eq!(format, FileFormat::Json);
1490 assert_eq!(rows, 2);
1491 assert_eq!(profiles.len(), 2);
1492 }
1493
1494 #[test]
1495 fn test_integer_negative_zero_profiles_as_integer_zero_in_every_shape() {
1496 for (data, config) in [
1497 (
1498 &b"[{\"v\":0},{\"v\":-0}]"[..],
1499 JsonParserConfig::json_document(),
1500 ),
1501 (&b"{\"v\":0}\n{\"v\":-0}\n"[..], JsonParserConfig::jsonl()),
1502 (&b"{\"v\":-0}"[..], JsonParserConfig::json_document()),
1503 ] {
1504 let (profiles, _stats, _rows, _malformed, _format) =
1505 analyze_json_from_reader(Cursor::new(data), &config).unwrap();
1506 assert_eq!(
1507 profiles[0].data_type,
1508 dataprof_core::DataType::Integer,
1509 "{data:?}"
1510 );
1511 assert_eq!(profiles[0].unique_count, Some(1), "{data:?}");
1512 }
1513 }
1514
1515 #[test]
1516 fn test_analyze_json_from_reader_max_rows() {
1517 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
1518 let cursor = Cursor::new(data.as_ref());
1519 let config = JsonParserConfig::default().with_max_rows(3);
1520
1521 let (_profiles, _stats, rows, _malformed, _format) =
1522 analyze_json_from_reader(cursor, &config).unwrap();
1523 assert_eq!(rows, 3);
1524 }
1525
1526 #[test]
1527 fn test_analyze_json_from_reader_missing_fields() {
1528 let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
1529 let cursor = Cursor::new(data.as_ref());
1530 let config = JsonParserConfig::jsonl();
1531
1532 let (profiles, _stats, rows, _malformed, _format) =
1533 analyze_json_from_reader(cursor, &config).unwrap();
1534 assert_eq!(rows, 2);
1535
1536 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1537 assert_eq!(col_b.total_count, 2);
1538 }
1539
1540 #[test]
1541 fn test_identical_records_count_as_duplicates_across_a_late_key() {
1542 let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"x\"}\n";
1546 let cursor = Cursor::new(data.as_ref());
1547 let config = JsonParserConfig::jsonl();
1548
1549 let (_profiles, stats, rows, _malformed, _format) =
1550 analyze_json_from_reader(cursor, &config).unwrap();
1551 assert_eq!(rows, 3);
1552
1553 let summary = stats
1554 .row_duplicate_summary()
1555 .expect("three rows were observed");
1556 assert_eq!(summary.rows_checked, 3);
1557 assert_eq!(summary.duplicate_rows, 1);
1558 assert!(!summary.approximate);
1559 }
1560
1561 #[test]
1562 fn test_a_late_key_does_not_invent_duplicates() {
1563 let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"z\"}\n";
1566 let cursor = Cursor::new(data.as_ref());
1567 let config = JsonParserConfig::jsonl();
1568
1569 let (_profiles, stats, _rows, _malformed, _format) =
1570 analyze_json_from_reader(cursor, &config).unwrap();
1571
1572 let summary = stats
1573 .row_duplicate_summary()
1574 .expect("three rows were observed");
1575 assert_eq!(summary.duplicate_rows, 0);
1576 }
1577
1578 #[test]
1579 fn test_analyze_json_file_quality_report() {
1580 let file = write_file(r#"[{"x":1},{"x":2}]"#);
1581 let config = JsonParserConfig::default();
1582 let report = analyze_json_file(file.path(), &config).unwrap();
1583
1584 assert_eq!(report.execution.rows_processed, 2);
1585 assert_eq!(report.column_profiles.len(), 1);
1586 assert!(report.quality_score().unwrap() >= 0.0);
1587 }
1588
1589 #[test]
1590 fn test_jsonl_skips_malformed_lines() {
1591 let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
1592 let cursor = Cursor::new(data.as_ref());
1593 let config = JsonParserConfig::jsonl();
1594
1595 let (profiles, _stats, rows, _malformed, format) =
1596 analyze_json_from_reader(cursor, &config).unwrap();
1597 assert_eq!(format, FileFormat::Jsonl);
1598 assert_eq!(rows, 2);
1599 assert_eq!(profiles[0].total_count, 2);
1600 }
1601
1602 #[test]
1603 fn test_strict_policy_errors_on_malformed_jsonl() {
1604 for data in [
1606 b"not-json\n{\"x\":1}\n{\"x\":2}\n".as_ref(),
1607 b"{\"x\":1}\nnot-json\n{\"x\":2}\n".as_ref(),
1608 b"{\"x\":1}\n{\"x\":2}\nnot-json\n".as_ref(),
1609 ] {
1610 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1611 let err = scan_json_from_reader(Cursor::new(data), &config, |_| {})
1612 .expect_err("strict mode must reject malformed records");
1613 let message = err.to_string().to_lowercase();
1614 assert!(message.contains("malformed json record"), "got: {message}");
1615 assert!(
1617 !err.to_string().contains("not-json"),
1618 "leaked record: {err}"
1619 );
1620 }
1621 }
1622
1623 #[test]
1624 fn test_tolerant_policy_counts_skipped_records() {
1625 let data = b"{\"x\":1}\nnot-json\n{\"x\":2}\n";
1626 let config = JsonParserConfig::jsonl(); let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1628 assert_eq!(summary.rows_read, 2);
1629 assert_eq!(summary.malformed_lines, 1);
1630 }
1631
1632 #[test]
1633 fn test_blank_lines_are_not_counted_as_malformed() {
1634 let data = b"{\"x\":1}\n\n \n{\"x\":2}\n";
1635 for policy in [JsonErrorPolicy::Skip, JsonErrorPolicy::Strict] {
1636 let config = JsonParserConfig::jsonl().with_error_policy(policy);
1637 let summary =
1638 scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1639 assert_eq!(summary.rows_read, 2, "policy {policy:?}");
1640 assert_eq!(summary.malformed_lines, 0, "policy {policy:?}");
1641 }
1642 }
1643
1644 #[test]
1645 fn test_truncated_final_record_obeys_error_policy() {
1646 let data = b"{\"x\":1}\n{\"x\":2";
1647
1648 let skip = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Skip);
1649 let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &skip, |_| {}).unwrap();
1650 assert_eq!(summary.rows_read, 1);
1651 assert_eq!(summary.malformed_lines, 1);
1652
1653 let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1654 let err = scan_json_from_reader(Cursor::new(data.as_ref()), &strict, |_| {})
1655 .expect_err("strict mode must reject an incomplete trailing record");
1656 assert!(
1657 err.to_string()
1658 .to_lowercase()
1659 .contains("malformed json record")
1660 );
1661 }
1662
1663 #[test]
1664 fn test_all_malformed_file_fails() {
1665 let file = write_file("not-json\nalso-bad\n");
1666 let config = JsonParserConfig::jsonl();
1667 let err = analyze_json_file(file.path(), &config)
1668 .expect_err("a file with no valid records must fail");
1669 let message = err.to_string().to_lowercase();
1670 assert!(message.contains("no valid json records"), "got: {message}");
1671 }
1672
1673 #[test]
1674 fn test_tolerant_file_reports_error_count() {
1675 let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1676 let config = JsonParserConfig::jsonl();
1677 let report = analyze_json_file(file.path(), &config).unwrap();
1678 assert_eq!(report.execution.rows_processed, 2);
1679 assert_eq!(report.execution.error_count, 1);
1680 }
1681
1682 #[test]
1683 fn test_strict_file_errors_on_malformed() {
1684 let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1685 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1686 let err = analyze_json_file(file.path(), &config)
1687 .expect_err("strict mode must reject the malformed record");
1688 assert!(
1689 err.to_string()
1690 .to_lowercase()
1691 .contains("malformed json record")
1692 );
1693 }
1694
1695 #[test]
1696 fn test_analyze_json_with_large_leading_whitespace() {
1697 let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
1698 let cursor = Cursor::new(data.into_bytes());
1699 let config = JsonParserConfig::default();
1700
1701 let (_profiles, _stats, rows, malformed, format) =
1702 analyze_json_from_reader(cursor, &config).unwrap();
1703 assert_eq!(format, FileFormat::Json);
1704 assert_eq!(rows, 1);
1705 assert_eq!(malformed, 0);
1706 }
1707
1708 #[test]
1709 fn test_analyze_json_array() {
1710 let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
1711 let config = JsonParserConfig::default();
1712 let report = analyze_json_file(json.path(), &config).unwrap();
1713 let profiles = &report.column_profiles;
1714
1715 assert_eq!(profiles.len(), 2);
1716 let names: Vec<&str> = profiles
1717 .iter()
1718 .map(|profile| profile.name.as_str())
1719 .collect();
1720 assert!(names.contains(&"name"));
1721 assert!(names.contains(&"age"));
1722
1723 let age = profiles
1724 .iter()
1725 .find(|profile| profile.name == "age")
1726 .unwrap();
1727 assert_eq!(age.total_count, 2);
1728 assert_eq!(age.null_count, 0);
1729 }
1730
1731 #[test]
1732 fn test_analyze_jsonl() {
1733 let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
1734 let config = JsonParserConfig::default();
1735 let report = analyze_json_file(jsonl.path(), &config).unwrap();
1736 let profiles = &report.column_profiles;
1737
1738 assert_eq!(profiles.len(), 1);
1739 assert_eq!(profiles[0].name, "x");
1740 assert_eq!(profiles[0].total_count, 3);
1741 }
1742
1743 #[test]
1744 fn test_analyze_json_with_nulls() {
1745 let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
1746 let config = JsonParserConfig::default();
1747 let report = analyze_json_file(json.path(), &config).unwrap();
1748 let profiles = &report.column_profiles;
1749
1750 let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
1751 assert_eq!(col_a.null_count, 1);
1752
1753 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1754 assert_eq!(col_b.null_count, 1);
1755 }
1756
1757 #[test]
1758 fn test_analyze_json_with_missing_fields() {
1759 let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
1760 let config = JsonParserConfig::default();
1761 let report = analyze_json_file(json.path(), &config).unwrap();
1762 let profiles = &report.column_profiles;
1763
1764 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1765 assert_eq!(col_b.total_count, 2);
1766 }
1767
1768 const UNSORTED_FIELDS: &str = r#"{"id":1,"amount":12.5,"active":true,"date":"2026-07-23"}"#;
1771 const UNSORTED_ORDER: [&str; 4] = ["id", "amount", "active", "date"];
1772
1773 fn profile_names(report: &ProfileReport) -> Vec<&str> {
1774 report
1775 .column_profiles
1776 .iter()
1777 .map(|profile| profile.name.as_str())
1778 .collect()
1779 }
1780
1781 #[test]
1782 fn test_json_column_order_follows_source_not_alphabet() {
1783 let array = write_file(&format!("[{UNSORTED_FIELDS},{UNSORTED_FIELDS}]"));
1784 let jsonl = write_file(&format!("{UNSORTED_FIELDS}\n{UNSORTED_FIELDS}\n"));
1785
1786 for file in [&array, &jsonl] {
1787 let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1788 assert_eq!(profile_names(&report), UNSORTED_ORDER);
1789 }
1790 }
1791
1792 #[test]
1793 fn test_json_column_order_appends_later_keys_in_first_seen_order() {
1794 let data = b"{\"zulu\":1,\"mike\":2}\n{\"zulu\":3,\"mike\":4,\"delta\":5,\"alpha\":6}\n";
1797 let file = write_bytes(data);
1798 let report = analyze_json_file(file.path(), &JsonParserConfig::jsonl()).unwrap();
1799
1800 assert_eq!(profile_names(&report), ["zulu", "mike", "delta", "alpha"]);
1801 }
1802
1803 #[test]
1804 fn test_json_column_order_survives_report_serialization() {
1805 let file = write_file(&format!("[{UNSORTED_FIELDS}]"));
1806 let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1807
1808 let json = serde_json::to_string(&report).unwrap();
1809 let restored: Value = serde_json::from_str(&json).unwrap();
1810 let names: Vec<&str> = restored["column_profiles"]
1811 .as_array()
1812 .unwrap()
1813 .iter()
1814 .map(|profile| profile["name"].as_str().unwrap())
1815 .collect();
1816
1817 assert_eq!(names, UNSORTED_ORDER);
1818 }
1819
1820 #[test]
1821 fn test_analyze_json_backfills_columns_discovered_after_first_row() {
1822 let json = write_file(r#"[{"a":1},{"a":2,"b":3}]"#);
1823 let config = JsonParserConfig::default();
1824 let report = analyze_json_file(json.path(), &config).unwrap();
1825 let col_b = report
1826 .column_profiles
1827 .iter()
1828 .find(|profile| profile.name == "b")
1829 .unwrap();
1830
1831 assert_eq!(col_b.total_count, 2);
1832 assert_eq!(col_b.null_count, 1);
1833 assert_eq!(col_b.unique_count, Some(1));
1834 }
1835
1836 #[test]
1837 fn test_analyze_json_empty_array() {
1838 let json = write_file("[]");
1839 let config = JsonParserConfig::default();
1840 let report = analyze_json_file(json.path(), &config).unwrap();
1841 assert!(report.column_profiles.is_empty());
1842 assert_eq!(report.quality_score(), None);
1844 }
1845
1846 #[test]
1847 fn test_analyze_json_malformed_returns_error() {
1848 let json = write_file("this is entirely invalid json");
1849 let config = JsonParserConfig::default();
1850 let err = analyze_json_file(json.path(), &config)
1851 .expect_err("malformed JSON should return an error");
1852
1853 let message = err.to_string().to_lowercase();
1854 assert!(message.contains("malformed") && message.contains("json"));
1855 }
1856
1857 #[test]
1858 fn test_analyze_json_file_detects_format() {
1859 let json_array = write_file(r#"[{"x":1}]"#);
1860 let config = JsonParserConfig::default();
1861 let report = analyze_json_file(json_array.path(), &config).unwrap();
1862 assert!(matches!(
1863 report.data_source,
1864 DataSource::File {
1865 format: FileFormat::Json,
1866 ..
1867 }
1868 ));
1869
1870 let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
1871 let report = analyze_json_file(jsonl.path(), &config).unwrap();
1872 assert!(matches!(
1873 report.data_source,
1874 DataSource::File {
1875 format: FileFormat::Jsonl,
1876 ..
1877 }
1878 ));
1879 }
1880
1881 #[test]
1882 fn test_bom_prefixed_file_preserves_source_size() {
1883 let data = b"\xEF\xBB\xBF[{\"x\":1}]";
1884 let json = write_bytes(data);
1885 let report = analyze_json_file(json.path(), &JsonParserConfig::json_document()).unwrap();
1886
1887 assert_eq!(report.execution.rows_processed, 1);
1888 assert!(matches!(
1889 report.data_source,
1890 DataSource::File {
1891 format: FileFormat::Json,
1892 size_bytes,
1893 ..
1894 } if size_bytes == data.len() as u64
1895 ));
1896 }
1897
1898 #[test]
1899 fn test_analyze_json_file_empty() {
1900 let json = write_file("");
1901 let config = JsonParserConfig::default();
1902 let report = analyze_json_file(json.path(), &config).unwrap();
1903 assert_eq!(report.execution.rows_processed, 0);
1904 assert!(report.column_profiles.is_empty());
1905 }
1906
1907 #[test]
1908 fn test_analyze_json_boolean_and_nested() {
1909 let json =
1910 write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
1911 let config = JsonParserConfig::default();
1912 let report = analyze_json_file(json.path(), &config).unwrap();
1913 let profiles = &report.column_profiles;
1914
1915 let flag = profiles
1916 .iter()
1917 .find(|profile| profile.name == "flag")
1918 .unwrap();
1919 assert_eq!(flag.total_count, 2);
1920
1921 let nested = profiles
1922 .iter()
1923 .find(|profile| profile.name == "nested")
1924 .unwrap();
1925 assert_eq!(nested.total_count, 2);
1926 }
1927
1928 #[test]
1929 fn test_single_root_object_compact_yields_one_row() {
1930 let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
1931 let cursor = Cursor::new(data.as_ref());
1932 let config = JsonParserConfig::default();
1933
1934 let (profiles, _stats, rows, malformed, _format) =
1935 analyze_json_from_reader(cursor, &config).unwrap();
1936 assert_eq!(rows, 1);
1937 assert_eq!(malformed, 0);
1938 assert_eq!(profiles.len(), 2);
1939
1940 let names: Vec<&str> = profiles
1941 .iter()
1942 .map(|profile| profile.name.as_str())
1943 .collect();
1944 assert!(names.contains(&"type"));
1945 assert!(names.contains(&"features"));
1946 }
1947
1948 #[test]
1949 fn test_jsonl_multi_object_still_works() {
1950 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1951 let cursor = Cursor::new(data.as_ref());
1952 let config = JsonParserConfig::default();
1953
1954 let (_profiles, _stats, rows, _malformed, format) =
1955 analyze_json_from_reader(cursor, &config).unwrap();
1956 assert_eq!(format, FileFormat::Jsonl);
1957 assert_eq!(rows, 3);
1958 }
1959
1960 #[test]
1961 fn test_single_root_object_via_analyze_json_file() {
1962 let file =
1963 write_json_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
1964 let config = JsonParserConfig::default();
1965 let report = analyze_json_file(file.path(), &config).unwrap();
1966
1967 assert_eq!(report.execution.rows_processed, 1);
1968 assert_eq!(report.column_profiles.len(), 2);
1969 }
1970
1971 #[test]
1972 fn test_single_root_object_pretty_printed_yields_one_row() {
1973 let data = br#"{
1974 "type": "FeatureCollection",
1975 "features": [
1976 {"id": 1},
1977 {"id": 2}
1978 ]
1979}"#;
1980 let cursor = Cursor::new(data.as_ref());
1981 let config = JsonParserConfig::json_document();
1984
1985 let (profiles, _stats, rows, malformed, format) =
1986 analyze_json_from_reader(cursor, &config).unwrap();
1987 assert_eq!(format, FileFormat::Json);
1988 assert_eq!(rows, 1);
1989 assert_eq!(malformed, 0);
1990 assert_eq!(profiles.len(), 2);
1991 }
1992
1993 #[test]
1994 fn test_single_root_object_pretty_printed_via_analyze_json_file() {
1995 let file = write_json_file(
1996 "{\n \"type\": \"FeatureCollection\",\n \"features\": [\n {\"id\": 1},\n {\"id\": 2}\n ]\n}\n",
1997 );
1998 let config = JsonParserConfig::default();
1999 let report = analyze_json_file(file.path(), &config).unwrap();
2000
2001 assert_eq!(report.execution.rows_processed, 1);
2002 assert_eq!(report.column_profiles.len(), 2);
2003 }
2004}