1use std::collections::{HashMap, HashSet};
30use std::io::BufRead;
31use std::path::Path;
32
33pub use dataprof_core::JsonErrorPolicy;
34
35#[doc(hidden)]
36pub mod record;
37use dataprof_core::{
38 AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat,
39 TruncationReason, Utf8BomReader,
40};
41use dataprof_runtime::{
42 ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
43};
44use record::{
45 JsonRecord, drain_to_end, json_document_error, json_value_kind, json_value_text,
46 malformed_array_error, non_object_record_error, object_record, read_json_record,
47 read_jsonl_line, skip_ascii_whitespace, wide_integer_record_error,
48};
49use serde_json::Value;
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
58pub enum JsonFormat {
59 Json,
64 Jsonl,
67}
68
69#[derive(Debug, Clone, Default)]
103pub struct JsonParserConfig {
104 pub format: Option<JsonFormat>,
111 pub max_rows: Option<usize>,
113 pub error_policy: JsonErrorPolicy,
115}
116
117impl JsonParserConfig {
118 pub fn with_max_rows(mut self, max_rows: usize) -> Self {
155 self.max_rows = Some(max_rows);
156 self
157 }
158
159 pub fn with_error_policy(mut self, policy: JsonErrorPolicy) -> Self {
187 self.error_policy = policy;
188 self
189 }
190
191 pub fn jsonl() -> Self {
193 Self {
194 format: Some(JsonFormat::Jsonl),
195 ..Default::default()
196 }
197 }
198
199 pub fn json_document() -> Self {
202 Self {
203 format: Some(JsonFormat::Json),
204 ..Default::default()
205 }
206 }
207
208 pub fn with_format(mut self, format: JsonFormat) -> Self {
210 self.format = Some(format);
211 self
212 }
213}
214
215pub type JsonObject = serde_json::Map<String, Value>;
222
223#[derive(Debug, Clone, PartialEq, Eq)]
225#[non_exhaustive]
226pub struct JsonScanSummary {
227 pub rows_read: usize,
228 pub malformed_lines: usize,
232 pub format: FileFormat,
233 pub truncated: bool,
236 pub unread_after_fault: bool,
241}
242
243pub fn scan_json_from_reader<R, F>(
328 reader: R,
329 config: &JsonParserConfig,
330 mut on_object: F,
331) -> Result<JsonScanSummary, DataProfilerError>
332where
333 R: BufRead,
334 F: FnMut(&JsonObject),
335{
336 let mut reader = Utf8BomReader::new(reader).map_err(DataProfilerError::from)?;
337 let format = match config.format {
338 Some(JsonFormat::Json) => FileFormat::Json,
339 Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
340 None => match consume_leading_whitespace(&mut reader)? {
344 Some(b'[') => FileFormat::Json,
345 _ => FileFormat::Jsonl,
346 },
347 };
348
349 let mut rows_read = 0;
350 let mut malformed_lines = 0;
351 let mut unread_after_fault = false;
352 let mut truncated = false;
353
354 match format {
355 FileFormat::Jsonl => {
361 let mut line = String::new();
362 let mut line_number = 0;
363 loop {
364 if let Some(max) = config.max_rows
365 && rows_read >= max
366 {
367 truncated = next_record_line(&mut reader, &mut line, &mut line_number)?;
371 break;
372 }
373
374 if !next_record_line(&mut reader, &mut line, &mut line_number)? {
375 break;
376 }
377
378 match read_jsonl_line(&line) {
379 JsonRecord::Object(obj) => {
380 on_object(&obj);
381 rows_read += 1;
382 }
383 JsonRecord::NonObject(kind) => {
384 if config.error_policy == JsonErrorPolicy::Strict {
385 return Err(non_object_record_error(
386 kind,
387 rows_read + malformed_lines + 1,
388 ));
389 }
390 malformed_lines += 1;
391 }
392 JsonRecord::Malformed(err) => {
393 if config.error_policy == JsonErrorPolicy::Strict {
394 return Err(malformed_jsonl_record_error(line_number, err));
395 }
396 malformed_lines += 1;
397 }
398 JsonRecord::WideInteger => {
399 if config.error_policy == JsonErrorPolicy::Strict {
400 return Err(wide_integer_record_error(&format!(
401 "on line {line_number}"
402 )));
403 }
404 malformed_lines += 1;
405 }
406 }
407 }
408 }
409 FileFormat::Json => {
410 let found_array = consume_leading_whitespace(&mut reader)? == Some(b'[');
414
415 if !found_array {
416 let (object_rows, object_malformed, object_truncated) =
420 scan_single_json_object(&mut reader, config, &mut on_object)?;
421 rows_read += object_rows;
422 malformed_lines += object_malformed;
423 truncated |= object_truncated;
424 } else {
425 consume_peeked(&mut reader)?;
427
428 let mut expect_value = true;
429 let mut allow_end = true;
430 let mut array_closed = false;
431 let mut drain_remainder = false;
432
433 loop {
434 let Some(next) = consume_leading_whitespace(&mut reader)? else {
435 if config.error_policy == JsonErrorPolicy::Strict {
436 return Err(malformed_array_error(
437 "unexpected end of input before closing ']'",
438 ));
439 }
440 malformed_lines += 1;
441 drain_remainder = true;
442 break;
443 };
444
445 if expect_value {
446 if next == b']' {
447 if !allow_end {
448 if config.error_policy == JsonErrorPolicy::Strict {
449 return Err(malformed_array_error(
450 "trailing comma before closing ']'",
451 ));
452 }
453 malformed_lines += 1;
454 drain_remainder = true;
455 break;
456 }
457 consume_peeked(&mut reader)?;
458 array_closed = true;
459 break;
460 }
461
462 if let Some(max) = config.max_rows
463 && rows_read >= max
464 {
465 truncated = true;
468 break;
469 }
470
471 if next == b',' {
472 if config.error_policy == JsonErrorPolicy::Strict {
473 return Err(malformed_array_error(
474 "unexpected comma where an array value was required",
475 ));
476 }
477 malformed_lines += 1;
478 drain_remainder = true;
479 break;
480 }
481
482 match read_json_record(&mut reader, next)? {
483 JsonRecord::Object(obj) => {
484 on_object(&obj);
485 rows_read += 1;
486 }
487 JsonRecord::NonObject(kind) => {
488 if config.error_policy == JsonErrorPolicy::Strict {
489 return Err(non_object_record_error(
490 kind,
491 rows_read + malformed_lines + 1,
492 ));
493 }
494 malformed_lines += 1;
498 }
499 JsonRecord::Malformed(err) => {
500 if config.error_policy == JsonErrorPolicy::Strict {
501 return Err(malformed_record_error(err));
502 }
503 malformed_lines += 1;
504 drain_remainder = true;
505 break;
506 }
507 JsonRecord::WideInteger => {
508 if config.error_policy == JsonErrorPolicy::Strict {
509 return Err(wide_integer_record_error(&format!(
510 "at position {}",
511 rows_read + malformed_lines + 1
512 )));
513 }
514 malformed_lines += 1;
516 }
517 }
518 expect_value = false;
519 } else {
520 match next {
521 b',' => {
522 consume_peeked(&mut reader)?;
523 expect_value = true;
524 allow_end = false;
525 }
526 b']' => {
527 consume_peeked(&mut reader)?;
528 array_closed = true;
529 break;
530 }
531 _ => {
532 if config.error_policy == JsonErrorPolicy::Strict {
533 return Err(malformed_array_error(
534 "expected ',' or ']' after an array value",
535 ));
536 }
537 malformed_lines += 1;
538 drain_remainder = true;
539 break;
540 }
541 }
542 }
543 }
544
545 if drain_remainder {
546 unread_after_fault = consume_leading_whitespace(&mut reader)?.is_some();
549 drain_to_end(&mut reader)?;
550 } else if array_closed && consume_leading_whitespace(&mut reader)?.is_some() {
551 if config.error_policy == JsonErrorPolicy::Strict {
552 return Err(malformed_array_error(
553 "non-whitespace content follows the closing ']'",
554 ));
555 }
556 malformed_lines += 1;
557 drain_to_end(&mut reader)?;
558 }
559 }
560 }
561 _ => unreachable!("json scanner only returns json or jsonl formats"),
562 }
563
564 Ok(JsonScanSummary {
565 rows_read,
566 malformed_lines,
567 format,
568 truncated,
569 unread_after_fault,
570 })
571}
572
573fn next_record_line<R: BufRead>(
580 reader: &mut R,
581 line: &mut String,
582 line_number: &mut usize,
583) -> Result<bool, DataProfilerError> {
584 loop {
585 line.clear();
586 let read = reader.read_line(line).map_err(DataProfilerError::from)?;
587 if read == 0 {
588 return Ok(false);
589 }
590 *line_number += 1;
591 if !line.trim().is_empty() {
592 return Ok(true);
593 }
594 }
595}
596
597fn scan_single_json_object<R, F>(
604 reader: &mut R,
605 config: &JsonParserConfig,
606 on_object: &mut F,
607) -> Result<(usize, usize, bool), DataProfilerError>
608where
609 R: BufRead,
610 F: FnMut(&JsonObject),
611{
612 let mut text = String::new();
613 reader
614 .read_to_string(&mut text)
615 .map_err(DataProfilerError::from)?;
616
617 if text.trim().is_empty() {
618 return Ok((0, 0, false));
619 }
620
621 if config.max_rows == Some(0) {
626 return Ok((0, 0, true));
627 }
628
629 match serde_json::from_str::<Value>(text.trim()) {
630 Ok(Value::Object(obj)) => match object_record(obj, text.as_bytes()) {
631 JsonRecord::Object(obj) => {
632 on_object(&obj);
633 Ok((1, 0, false))
634 }
635 _ => {
636 if config.error_policy == JsonErrorPolicy::Strict {
637 return Err(wide_integer_record_error("at position 1"));
638 }
639 Ok((0, 1, false))
640 }
641 },
642 Ok(value) => {
643 if config.error_policy == JsonErrorPolicy::Strict {
644 return Err(non_object_record_error(json_value_kind(&value), 1));
645 }
646 Ok((0, 1, false))
647 }
648 Err(err) => {
649 if config.error_policy == JsonErrorPolicy::Strict {
650 return Err(json_document_error(err));
651 }
652 Ok((0, 1, false))
653 }
654 }
655}
656
657fn consume_peeked<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
666 skip_ascii_whitespace(reader)?;
667 reader.consume(1);
668 Ok(())
669}
670
671fn malformed_record_error(err: serde_json::Error) -> DataProfilerError {
675 let message = format!("malformed JSON record: {err}");
676 DataProfilerError::json_parsing_with_source(message, err)
677}
678
679fn malformed_jsonl_record_error(line: usize, err: serde_json::Error) -> DataProfilerError {
687 let message = format!(
688 "malformed JSON record on line {line}, column {}: a JSONL record must be one complete JSON value on one line",
689 err.column()
690 );
691 DataProfilerError::json_parsing_with_source(message, err)
692}
693
694fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
695 loop {
696 let mut bytes_to_consume = 0;
697 let first_non_whitespace = {
698 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
699 if buf.is_empty() {
700 return Ok(None);
701 }
702
703 let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
704 if first_non_whitespace.is_none() {
705 bytes_to_consume = buf.len();
706 }
707 first_non_whitespace
708 };
709
710 if first_non_whitespace.is_some() {
711 return Ok(first_non_whitespace);
712 }
713
714 reader.consume(bytes_to_consume);
715 }
716}
717
718fn feed_json_object(
728 obj: &JsonObject,
729 prior_rows: usize,
730 known_columns: &mut Vec<String>,
731 known_columns_set: &mut HashSet<String>,
732 column_stats: &mut StreamingColumnCollection,
733 container_counts: &mut HashMap<String, usize>,
734) {
735 for (key, value) in obj {
736 if known_columns_set.insert(key.clone()) {
737 known_columns.push(key.clone());
738 column_stats.init_column_with_missing(key, prior_rows);
739 }
740 if matches!(value, Value::Array(_) | Value::Object(_)) {
741 *container_counts.entry(key.clone()).or_default() += 1;
742 }
743 }
744
745 let values: Vec<String> = known_columns
746 .iter()
747 .map(|column| {
748 obj.get(column).map(json_value_text).unwrap_or_default()
751 })
752 .collect();
753 let containers: Vec<usize> = known_columns
754 .iter()
755 .enumerate()
756 .filter(|(_, column)| matches!(obj.get(*column), Some(Value::Array(_) | Value::Object(_))))
757 .map(|(position, _)| position)
758 .collect();
759
760 column_stats.process_record_with_containers(known_columns, values, &containers);
761}
762
763pub fn analyze_json_from_reader<R: BufRead>(
805 reader: R,
806 config: &JsonParserConfig,
807) -> Result<
808 (
809 Vec<ColumnProfile>,
810 StreamingColumnCollection,
811 usize,
812 usize,
813 FileFormat,
814 ),
815 DataProfilerError,
816> {
817 let options = AnalysisOptions::default();
818 let (profiles, stats, rows_read, malformed_lines, format, _truncated, _unread) =
819 analyze_json_from_reader_full(reader, config, &options)?;
820 Ok((profiles, stats, rows_read, malformed_lines, format))
821}
822
823#[allow(clippy::type_complexity)]
828fn analyze_json_from_reader_full<R: BufRead>(
829 reader: R,
830 config: &JsonParserConfig,
831 options: &AnalysisOptions,
832) -> Result<
833 (
834 Vec<ColumnProfile>,
835 StreamingColumnCollection,
836 usize,
837 usize,
838 FileFormat,
839 bool,
840 bool,
841 ),
842 DataProfilerError,
843> {
844 let semantic_hints = options.semantic_hints();
845 let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
846 let mut known_columns = Vec::new();
847 let mut known_columns_set = HashSet::new();
848 let mut container_counts = HashMap::new();
849 let mut rows_seen = 0;
850
851 let summary = scan_json_from_reader(reader, config, |obj| {
852 feed_json_object(
853 obj,
854 rows_seen,
855 &mut known_columns,
856 &mut known_columns_set,
857 &mut column_stats,
858 &mut container_counts,
859 );
860 rows_seen += 1;
861 })?;
862
863 if let Some(indices) = options.column_indices(&column_stats.column_names())? {
864 let available = column_stats.column_names();
865 let selected = indices
866 .into_iter()
867 .map(|index| available[index].clone())
868 .collect::<Vec<_>>();
869 column_stats.retain_columns(&selected);
870 }
871
872 let profiles = profile_builder::mark_container_columns(
873 profile_builder::profiles_from_streaming_with_hints(
874 &column_stats,
875 !options.include_statistics(),
876 !options.include_patterns(),
877 options.locale(),
878 semantic_hints,
879 ),
880 &container_counts,
881 );
882
883 Ok((
884 profiles,
885 column_stats,
886 summary.rows_read,
887 summary.malformed_lines,
888 summary.format,
889 summary.truncated,
890 summary.unread_after_fault,
891 ))
892}
893
894pub fn analyze_json_file(
922 file_path: &Path,
923 config: &JsonParserConfig,
924) -> Result<ProfileReport, DataProfilerError> {
925 analyze_json_file_with_options(file_path, config, &AnalysisOptions::default())
926}
927
928pub fn analyze_json_file_with_options(
934 file_path: &Path,
935 config: &JsonParserConfig,
936 options: &AnalysisOptions,
937) -> Result<ProfileReport, DataProfilerError> {
938 let mut config = config.clone();
941 if config.format.is_none() {
942 config.format = grammar_for_extension(file_path);
943 }
944 let config = &config;
945 let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
946 dataprof_core::binary_input::reject_binary_input(file_path)?;
947 let start = std::time::Instant::now();
948
949 let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
950 let buf_reader = std::io::BufReader::new(file);
951
952 let (
953 column_profiles,
954 column_stats,
955 rows_read,
956 malformed_lines,
957 format,
958 truncated,
959 unread_after_fault,
960 ) = analyze_json_from_reader_full(buf_reader, config, options)?;
961 let read_as_json_document = format == FileFormat::Json;
962
963 let file_source = DataSource::File {
964 path: file_path.display().to_string(),
965 format,
966 size_bytes: metadata.len(),
967 modified_at: None,
968 parquet_metadata: None,
969 };
970
971 if rows_read == 0 {
972 if malformed_lines > 0 {
973 let hint = if read_as_json_document {
977 ". A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
978 } else {
979 ""
980 };
981 return Err(DataProfilerError::json_parsing_error(&format!(
984 "No valid JSON records found in file \
985 (every record was malformed, not a JSON object, or held an integer outside the 64-bit range){hint}"
986 )));
987 }
988 let mut execution =
993 ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("json");
994 if truncated && let Some(max) = config.max_rows {
995 execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
996 }
997 return Ok(ReportAssembler::new(file_source, execution)
998 .columns(column_profiles)
999 .with_quality_data(HashMap::new())
1000 .with_analysis_options(options)
1001 .build());
1002 }
1003
1004 let sample_columns = profile_builder::quality_check_samples(&column_stats);
1005 let scan_time_ms = start.elapsed().as_millis();
1006 let num_columns = column_profiles.len();
1007
1008 let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms)
1009 .with_engine("json")
1010 .with_error_count(malformed_lines);
1013 if truncated && let Some(max) = config.max_rows {
1016 execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1017 }
1018 if unread_after_fault {
1019 execution = execution.with_source_exhausted(false);
1020 }
1021
1022 Ok(ReportAssembler::new(file_source, execution)
1023 .columns(column_profiles)
1024 .with_quality_data(sample_columns)
1025 .with_row_duplicates(column_stats.row_duplicate_summary())
1026 .with_row_completeness(column_stats.row_completeness_summary())
1027 .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
1028 .with_analysis_options(options)
1029 .build())
1030}
1031
1032fn grammar_for_extension(file_path: &Path) -> Option<JsonFormat> {
1040 let extension = file_path.extension().and_then(|ext| ext.to_str())?;
1041 if extension.eq_ignore_ascii_case("json") {
1042 Some(JsonFormat::Json)
1043 } else if extension.eq_ignore_ascii_case("jsonl") || extension.eq_ignore_ascii_case("ndjson") {
1044 Some(JsonFormat::Jsonl)
1045 } else {
1046 None
1047 }
1048}
1049
1050fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
1051 if error.kind() == std::io::ErrorKind::NotFound {
1052 DataProfilerError::FileNotFound {
1053 path: file_path.display().to_string(),
1054 }
1055 } else {
1056 DataProfilerError::from(error)
1057 }
1058}
1059
1060#[cfg(test)]
1061mod tests {
1062 use super::*;
1063 use std::io::{Cursor, Write};
1064 use tempfile::NamedTempFile;
1065
1066 fn write_file(content: &str) -> NamedTempFile {
1069 let mut file = NamedTempFile::new().unwrap();
1070 write!(file, "{}", content).unwrap();
1071 file.flush().unwrap();
1072 file
1073 }
1074
1075 fn write_json_file(content: &str) -> NamedTempFile {
1078 let mut file = NamedTempFile::with_suffix(".json").unwrap();
1079 write!(file, "{}", content).unwrap();
1080 file.flush().unwrap();
1081 file
1082 }
1083
1084 fn write_bytes(content: &[u8]) -> NamedTempFile {
1085 let mut file = NamedTempFile::new().unwrap();
1086 file.write_all(content).unwrap();
1087 file.flush().unwrap();
1088 file
1089 }
1090
1091 #[test]
1092 fn test_scan_jsonl_detects_format_and_rows() {
1093 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1094 let mut rows = 0;
1095
1096 let summary = scan_json_from_reader(
1097 Cursor::new(data.as_ref()),
1098 &JsonParserConfig::default(),
1099 |_| {
1100 rows += 1;
1101 },
1102 )
1103 .unwrap();
1104
1105 assert_eq!(summary.format, FileFormat::Jsonl);
1106 assert_eq!(summary.rows_read, 3);
1107 assert_eq!(summary.malformed_lines, 0);
1108 assert_eq!(rows, 3);
1109 }
1110
1111 #[test]
1112 fn test_scan_json_array_detects_format_and_rows() {
1113 let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
1114 let mut keys = Vec::new();
1115
1116 let summary = scan_json_from_reader(
1117 Cursor::new(data.as_ref()),
1118 &JsonParserConfig::default(),
1119 |obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
1120 )
1121 .unwrap();
1122
1123 assert_eq!(summary.format, FileFormat::Json);
1124 assert_eq!(summary.rows_read, 2);
1125 assert_eq!(keys.len(), 2);
1126 assert_eq!(keys[0], vec!["name".to_string()]);
1127 }
1128
1129 #[test]
1130 fn test_scan_accepts_one_leading_utf8_bom_for_json_and_jsonl() {
1131 for (data, config, expected_format) in [
1132 (
1133 b"\xEF\xBB\xBF[{\"x\":1},{\"x\":2}]".as_slice(),
1134 JsonParserConfig::default(),
1135 FileFormat::Json,
1136 ),
1137 (
1138 b"\xEF\xBB\xBF{\"x\":1}\n{\"x\":2}\n".as_slice(),
1139 JsonParserConfig::default(),
1140 FileFormat::Jsonl,
1141 ),
1142 ] {
1143 let mut rows = 0;
1144 let summary = scan_json_from_reader(Cursor::new(data), &config, |_| rows += 1)
1145 .expect("leading UTF-8 BOM should be accepted");
1146
1147 assert_eq!(rows, 2);
1148 assert_eq!(summary.rows_read, 2);
1149 assert_eq!(summary.malformed_lines, 0);
1150 assert_eq!(summary.format, expected_format);
1151 }
1152 }
1153
1154 #[test]
1155 fn test_scan_does_not_strip_nonleading_or_second_utf8_bom() {
1156 let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1157
1158 for data in [
1159 b" \xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1160 b"\xEF\xBB\xBF\xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1161 b"{\"x\":1}\n\xEF\xBB\xBF{\"x\":2}\n".as_slice(),
1162 ] {
1163 scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1164 .expect_err("only the BOM at absolute byte offset zero may be stripped");
1165 }
1166 }
1167
1168 #[test]
1169 fn test_bom_only_input_matches_empty_input_policy() {
1170 for config in [JsonParserConfig::default(), JsonParserConfig::jsonl()] {
1171 let empty = scan_json_from_reader(Cursor::new(b""), &config, |_| {}).unwrap();
1172 let bom =
1173 scan_json_from_reader(Cursor::new(b"\xEF\xBB\xBF".as_slice()), &config, |_| {})
1174 .unwrap();
1175 assert_eq!(bom, empty);
1176 }
1177 }
1178
1179 #[test]
1180 fn test_json_array_container_grammar_obeys_error_policy() {
1181 let cases = [
1182 (br#"[{"x":1}"#.as_ref(), 1, "missing closing bracket"),
1183 (br#"[{"x":1},"#.as_ref(), 1, "comma followed by EOF"),
1184 (br#"[{"x":1} {"x":2}]"#.as_ref(), 1, "missing comma"),
1185 (br#"[{"x":1},,{"x":2}]"#.as_ref(), 1, "doubled comma"),
1186 (br#"[,{"x":1}]"#.as_ref(), 0, "leading comma"),
1187 (br#"[{"x":1},]"#.as_ref(), 1, "trailing comma"),
1188 (br#"[{"x":1}] trailing"#.as_ref(), 1, "trailing content"),
1189 (
1190 br#"[{"x":1}][{"x":2}]"#.as_ref(),
1191 1,
1192 "second top-level value",
1193 ),
1194 ];
1195
1196 for (data, expected_rows, case) in cases {
1197 let skip = JsonParserConfig::json_document();
1198 let summary =
1199 scan_json_from_reader(Cursor::new(data), &skip, |_| {}).expect("tolerant scan");
1200 assert_eq!(summary.rows_read, expected_rows, "{case}");
1201 assert_eq!(summary.malformed_lines, 1, "{case}");
1202
1203 let strict =
1204 JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict);
1205 let err = scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1206 .expect_err("strict mode must reject invalid array grammar");
1207 assert!(
1208 err.to_string()
1209 .to_lowercase()
1210 .contains("malformed json array"),
1211 "{case}: {err}"
1212 );
1213 }
1214 }
1215
1216 #[test]
1217 fn test_json_array_max_rows_does_not_validate_unread_values() {
1218 let data = br#"[{"x":1},not-valid-json"#;
1219 let config = JsonParserConfig::json_document().with_max_rows(1);
1220 let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1221
1222 assert_eq!(summary.rows_read, 1);
1223 assert_eq!(summary.malformed_lines, 0);
1224 assert!(summary.truncated);
1225 }
1226
1227 #[test]
1228 fn test_scan_respects_max_rows() {
1229 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
1230 let config = JsonParserConfig::default().with_max_rows(2);
1231 let mut rows = 0;
1232
1233 let summary =
1234 scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
1235
1236 assert_eq!(summary.rows_read, 2);
1237 assert_eq!(rows, 2);
1238 }
1239
1240 #[test]
1241 fn test_scan_skips_malformed_jsonl_lines() {
1242 let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
1243 let mut rows = 0;
1244
1245 let summary = scan_json_from_reader(
1246 Cursor::new(data.as_ref()),
1247 &JsonParserConfig::jsonl(),
1248 |_| {
1249 rows += 1;
1250 },
1251 )
1252 .unwrap();
1253
1254 assert_eq!(summary.format, FileFormat::Jsonl);
1255 assert_eq!(summary.rows_read, 2);
1256 assert_eq!(summary.malformed_lines, 1);
1257 assert_eq!(rows, 2);
1258 }
1259
1260 #[test]
1266 fn test_pretty_printed_root_object_is_one_row_under_the_json_grammar() {
1267 let data = br#"{
1268 "type": "FeatureCollection",
1269 "features": [
1270 {"id": 1},
1271 {"id": 2}
1272 ]
1273}"#;
1274 let mut object_field_counts = Vec::new();
1275
1276 let summary = scan_json_from_reader(
1277 Cursor::new(data.as_ref()),
1278 &JsonParserConfig::json_document(),
1279 |obj| object_field_counts.push(obj.len()),
1280 )
1281 .unwrap();
1282
1283 assert_eq!(summary.format, FileFormat::Json);
1284 assert_eq!(summary.rows_read, 1);
1285 assert_eq!(summary.malformed_lines, 0);
1286 assert_eq!(object_field_counts, vec![2]);
1287 }
1288
1289 #[test]
1290 fn test_pretty_printed_root_object_is_not_valid_jsonl() {
1291 let data = br#"{
1292 "type": "FeatureCollection"
1293}"#;
1294 let err = scan_json_from_reader(
1295 Cursor::new(data.as_ref()),
1296 &JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict),
1297 |_| {},
1298 )
1299 .expect_err("a record spanning lines is not JSONL");
1300
1301 assert!(
1302 err.to_string().contains("malformed JSON record"),
1303 "{err}, expected the malformed category"
1304 );
1305 }
1306
1307 const NON_OBJECT_VALUES: &[(&str, &str)] = &[
1311 ("null", "null"),
1312 ("boolean", "true"),
1313 ("number", "42"),
1314 ("number", "-1.5e3"),
1315 ("string", r#""text""#),
1316 ("array", "[1, 2]"),
1317 ];
1318
1319 #[test]
1320 fn test_jsonl_counts_non_object_records_and_keeps_scanning() {
1321 for (kind, value) in NON_OBJECT_VALUES {
1322 let data = format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n");
1323 let mut rows = 0;
1324
1325 let summary = scan_json_from_reader(
1326 Cursor::new(data.as_bytes()),
1327 &JsonParserConfig::jsonl(),
1328 |_| rows += 1,
1329 )
1330 .unwrap();
1331
1332 assert_eq!(summary.rows_read, 2, "{kind}: record after it was lost");
1333 assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1334 assert_eq!(rows, 2, "{kind}");
1335 }
1336 }
1337
1338 #[test]
1339 fn test_json_array_counts_non_object_elements_and_keeps_scanning() {
1340 for (kind, value) in NON_OBJECT_VALUES {
1341 let data = format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]");
1342 let mut rows = 0;
1343
1344 let summary = scan_json_from_reader(
1345 Cursor::new(data.as_bytes()),
1346 &JsonParserConfig::json_document(),
1347 |_| rows += 1,
1348 )
1349 .unwrap();
1350
1351 assert_eq!(summary.rows_read, 2, "{kind}: element after it was lost");
1352 assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1353 assert_eq!(rows, 2, "{kind}");
1354 }
1355 }
1356
1357 #[test]
1358 fn test_strict_policy_rejects_the_first_non_object_record() {
1359 for (kind, value) in NON_OBJECT_VALUES {
1360 for (config, data) in [
1361 (
1362 JsonParserConfig::jsonl(),
1363 format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n"),
1364 ),
1365 (
1366 JsonParserConfig::json_document(),
1367 format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]"),
1368 ),
1369 ] {
1370 let config = config.with_error_policy(JsonErrorPolicy::Strict);
1371 let err = scan_json_from_reader(Cursor::new(data.as_bytes()), &config, |_| {})
1372 .expect_err("{kind}: strict mode must reject a non-object record");
1373
1374 let message = err.to_string();
1375 assert!(message.contains("non-object JSON record"), "{message}");
1376 assert!(message.contains("at position 2"), "{message}");
1377 assert!(message.contains(&format!("found {kind}")), "{message}");
1378 }
1379 }
1380 }
1381
1382 #[test]
1383 fn test_number_element_does_not_swallow_the_array_delimiter() {
1384 let data = br#"[{"id":1}, 1, 2, 3, {"id":2}, 4, {"id":3}]"#;
1387 let mut rows = 0;
1388
1389 let summary = scan_json_from_reader(
1390 Cursor::new(data.as_ref()),
1391 &JsonParserConfig::json_document(),
1392 |_| rows += 1,
1393 )
1394 .unwrap();
1395
1396 assert_eq!(summary.rows_read, 3);
1397 assert_eq!(summary.malformed_lines, 4);
1398 assert_eq!(rows, 3);
1399 }
1400
1401 #[test]
1402 fn test_number_shaped_garbage_stays_a_malformed_record() {
1403 let data = b"{\"id\":1}\n1.2.3\n";
1406 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1407 let err = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {})
1408 .expect_err("number-shaped garbage should be malformed");
1409
1410 assert!(
1411 err.to_string().contains("malformed JSON record"),
1412 "{err}, expected the malformed category"
1413 );
1414 }
1415
1416 #[test]
1417 fn test_input_of_only_non_object_records_fails() {
1418 let file = write_file("\"just a string\"\n1\n[2]\n");
1419 let err = analyze_json_file(file.path(), &JsonParserConfig::jsonl())
1420 .expect_err("nothing profileable should not return a clean empty profile");
1421
1422 assert!(
1423 err.to_string().contains("No valid JSON records found"),
1424 "{err}"
1425 );
1426 }
1427
1428 #[test]
1429 fn test_json_grammar_reads_a_bare_object_as_one_row() {
1432 let data = br#"{"x":1}"#;
1433 let mut rows = 0;
1434 let summary = scan_json_from_reader(
1435 Cursor::new(data.as_ref()),
1436 &JsonParserConfig::json_document(),
1437 |_| rows += 1,
1438 )
1439 .unwrap();
1440
1441 assert_eq!(summary.rows_read, 1);
1442 assert_eq!(summary.malformed_lines, 0);
1443 assert_eq!(rows, 1);
1444 }
1445
1446 #[test]
1447 fn test_json_grammar_rejects_concatenated_objects() {
1451 let data = br#"{"x":1}{"x":2}"#;
1452 let err = scan_json_from_reader(
1453 Cursor::new(data.as_ref()),
1454 &JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict),
1455 |_| {},
1456 )
1457 .expect_err("two objects are not one JSON document");
1458
1459 let message = err.to_string();
1460 assert!(message.contains("malformed JSON document"), "{message}");
1461 assert!(message.contains("format=\"jsonl\""), "{message}");
1462 }
1463
1464 #[test]
1465 fn test_analyze_json_from_reader_jsonl_streaming() {
1466 let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
1467 let cursor = Cursor::new(data.as_ref());
1468 let config = JsonParserConfig::default();
1469
1470 let (profiles, _stats, rows, _malformed, format) =
1471 analyze_json_from_reader(cursor, &config).unwrap();
1472 assert_eq!(format, FileFormat::Jsonl);
1473 assert_eq!(rows, 3);
1474 assert_eq!(profiles.len(), 2);
1475 }
1476
1477 #[test]
1478 fn test_analyze_json_from_reader_json_array() {
1479 let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
1480 let cursor = Cursor::new(data.as_ref());
1481 let config = JsonParserConfig::default();
1482
1483 let (profiles, _stats, rows, _malformed, format) =
1484 analyze_json_from_reader(cursor, &config).unwrap();
1485 assert_eq!(format, FileFormat::Json);
1486 assert_eq!(rows, 2);
1487 assert_eq!(profiles.len(), 2);
1488 }
1489
1490 #[test]
1491 fn test_integer_negative_zero_profiles_as_integer_zero_in_every_shape() {
1492 for (data, config) in [
1493 (
1494 &b"[{\"v\":0},{\"v\":-0}]"[..],
1495 JsonParserConfig::json_document(),
1496 ),
1497 (&b"{\"v\":0}\n{\"v\":-0}\n"[..], JsonParserConfig::jsonl()),
1498 (&b"{\"v\":-0}"[..], JsonParserConfig::json_document()),
1499 ] {
1500 let (profiles, _stats, _rows, _malformed, _format) =
1501 analyze_json_from_reader(Cursor::new(data), &config).unwrap();
1502 assert_eq!(
1503 profiles[0].data_type,
1504 dataprof_core::DataType::Integer,
1505 "{data:?}"
1506 );
1507 assert_eq!(profiles[0].unique_count, Some(1), "{data:?}");
1508 }
1509 }
1510
1511 #[test]
1512 fn test_analyze_json_from_reader_max_rows() {
1513 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
1514 let cursor = Cursor::new(data.as_ref());
1515 let config = JsonParserConfig::default().with_max_rows(3);
1516
1517 let (_profiles, _stats, rows, _malformed, _format) =
1518 analyze_json_from_reader(cursor, &config).unwrap();
1519 assert_eq!(rows, 3);
1520 }
1521
1522 #[test]
1523 fn test_analyze_json_from_reader_missing_fields() {
1524 let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
1525 let cursor = Cursor::new(data.as_ref());
1526 let config = JsonParserConfig::jsonl();
1527
1528 let (profiles, _stats, rows, _malformed, _format) =
1529 analyze_json_from_reader(cursor, &config).unwrap();
1530 assert_eq!(rows, 2);
1531
1532 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1533 assert_eq!(col_b.total_count, 2);
1534 }
1535
1536 #[test]
1537 fn test_identical_records_count_as_duplicates_across_a_late_key() {
1538 let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"x\"}\n";
1542 let cursor = Cursor::new(data.as_ref());
1543 let config = JsonParserConfig::jsonl();
1544
1545 let (_profiles, stats, rows, _malformed, _format) =
1546 analyze_json_from_reader(cursor, &config).unwrap();
1547 assert_eq!(rows, 3);
1548
1549 let summary = stats
1550 .row_duplicate_summary()
1551 .expect("three rows were observed");
1552 assert_eq!(summary.rows_checked, 3);
1553 assert_eq!(summary.duplicate_rows, 1);
1554 assert!(!summary.approximate);
1555 }
1556
1557 #[test]
1558 fn test_a_late_key_does_not_invent_duplicates() {
1559 let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"z\"}\n";
1562 let cursor = Cursor::new(data.as_ref());
1563 let config = JsonParserConfig::jsonl();
1564
1565 let (_profiles, stats, _rows, _malformed, _format) =
1566 analyze_json_from_reader(cursor, &config).unwrap();
1567
1568 let summary = stats
1569 .row_duplicate_summary()
1570 .expect("three rows were observed");
1571 assert_eq!(summary.duplicate_rows, 0);
1572 }
1573
1574 #[test]
1575 fn test_analyze_json_file_quality_report() {
1576 let file = write_file(r#"[{"x":1},{"x":2}]"#);
1577 let config = JsonParserConfig::default();
1578 let report = analyze_json_file(file.path(), &config).unwrap();
1579
1580 assert_eq!(report.execution.rows_processed, 2);
1581 assert_eq!(report.column_profiles.len(), 1);
1582 assert!(report.quality_score().unwrap() >= 0.0);
1583 }
1584
1585 #[test]
1586 fn test_jsonl_skips_malformed_lines() {
1587 let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
1588 let cursor = Cursor::new(data.as_ref());
1589 let config = JsonParserConfig::jsonl();
1590
1591 let (profiles, _stats, rows, _malformed, format) =
1592 analyze_json_from_reader(cursor, &config).unwrap();
1593 assert_eq!(format, FileFormat::Jsonl);
1594 assert_eq!(rows, 2);
1595 assert_eq!(profiles[0].total_count, 2);
1596 }
1597
1598 #[test]
1599 fn test_strict_policy_errors_on_malformed_jsonl() {
1600 for data in [
1602 b"not-json\n{\"x\":1}\n{\"x\":2}\n".as_ref(),
1603 b"{\"x\":1}\nnot-json\n{\"x\":2}\n".as_ref(),
1604 b"{\"x\":1}\n{\"x\":2}\nnot-json\n".as_ref(),
1605 ] {
1606 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1607 let err = scan_json_from_reader(Cursor::new(data), &config, |_| {})
1608 .expect_err("strict mode must reject malformed records");
1609 let message = err.to_string().to_lowercase();
1610 assert!(message.contains("malformed json record"), "got: {message}");
1611 assert!(
1613 !err.to_string().contains("not-json"),
1614 "leaked record: {err}"
1615 );
1616 }
1617 }
1618
1619 #[test]
1620 fn test_tolerant_policy_counts_skipped_records() {
1621 let data = b"{\"x\":1}\nnot-json\n{\"x\":2}\n";
1622 let config = JsonParserConfig::jsonl(); let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1624 assert_eq!(summary.rows_read, 2);
1625 assert_eq!(summary.malformed_lines, 1);
1626 }
1627
1628 #[test]
1629 fn test_blank_lines_are_not_counted_as_malformed() {
1630 let data = b"{\"x\":1}\n\n \n{\"x\":2}\n";
1631 for policy in [JsonErrorPolicy::Skip, JsonErrorPolicy::Strict] {
1632 let config = JsonParserConfig::jsonl().with_error_policy(policy);
1633 let summary =
1634 scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1635 assert_eq!(summary.rows_read, 2, "policy {policy:?}");
1636 assert_eq!(summary.malformed_lines, 0, "policy {policy:?}");
1637 }
1638 }
1639
1640 #[test]
1641 fn test_truncated_final_record_obeys_error_policy() {
1642 let data = b"{\"x\":1}\n{\"x\":2";
1643
1644 let skip = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Skip);
1645 let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &skip, |_| {}).unwrap();
1646 assert_eq!(summary.rows_read, 1);
1647 assert_eq!(summary.malformed_lines, 1);
1648
1649 let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1650 let err = scan_json_from_reader(Cursor::new(data.as_ref()), &strict, |_| {})
1651 .expect_err("strict mode must reject an incomplete trailing record");
1652 assert!(
1653 err.to_string()
1654 .to_lowercase()
1655 .contains("malformed json record")
1656 );
1657 }
1658
1659 #[test]
1660 fn test_all_malformed_file_fails() {
1661 let file = write_file("not-json\nalso-bad\n");
1662 let config = JsonParserConfig::jsonl();
1663 let err = analyze_json_file(file.path(), &config)
1664 .expect_err("a file with no valid records must fail");
1665 let message = err.to_string().to_lowercase();
1666 assert!(message.contains("no valid json records"), "got: {message}");
1667 }
1668
1669 #[test]
1670 fn test_tolerant_file_reports_error_count() {
1671 let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1672 let config = JsonParserConfig::jsonl();
1673 let report = analyze_json_file(file.path(), &config).unwrap();
1674 assert_eq!(report.execution.rows_processed, 2);
1675 assert_eq!(report.execution.error_count, 1);
1676 }
1677
1678 #[test]
1679 fn test_strict_file_errors_on_malformed() {
1680 let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1681 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1682 let err = analyze_json_file(file.path(), &config)
1683 .expect_err("strict mode must reject the malformed record");
1684 assert!(
1685 err.to_string()
1686 .to_lowercase()
1687 .contains("malformed json record")
1688 );
1689 }
1690
1691 #[test]
1692 fn test_analyze_json_with_large_leading_whitespace() {
1693 let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
1694 let cursor = Cursor::new(data.into_bytes());
1695 let config = JsonParserConfig::default();
1696
1697 let (_profiles, _stats, rows, malformed, format) =
1698 analyze_json_from_reader(cursor, &config).unwrap();
1699 assert_eq!(format, FileFormat::Json);
1700 assert_eq!(rows, 1);
1701 assert_eq!(malformed, 0);
1702 }
1703
1704 #[test]
1705 fn test_analyze_json_array() {
1706 let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
1707 let config = JsonParserConfig::default();
1708 let report = analyze_json_file(json.path(), &config).unwrap();
1709 let profiles = &report.column_profiles;
1710
1711 assert_eq!(profiles.len(), 2);
1712 let names: Vec<&str> = profiles
1713 .iter()
1714 .map(|profile| profile.name.as_str())
1715 .collect();
1716 assert!(names.contains(&"name"));
1717 assert!(names.contains(&"age"));
1718
1719 let age = profiles
1720 .iter()
1721 .find(|profile| profile.name == "age")
1722 .unwrap();
1723 assert_eq!(age.total_count, 2);
1724 assert_eq!(age.null_count, 0);
1725 }
1726
1727 #[test]
1728 fn test_analyze_jsonl() {
1729 let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
1730 let config = JsonParserConfig::default();
1731 let report = analyze_json_file(jsonl.path(), &config).unwrap();
1732 let profiles = &report.column_profiles;
1733
1734 assert_eq!(profiles.len(), 1);
1735 assert_eq!(profiles[0].name, "x");
1736 assert_eq!(profiles[0].total_count, 3);
1737 }
1738
1739 #[test]
1740 fn test_analyze_json_with_nulls() {
1741 let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
1742 let config = JsonParserConfig::default();
1743 let report = analyze_json_file(json.path(), &config).unwrap();
1744 let profiles = &report.column_profiles;
1745
1746 let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
1747 assert_eq!(col_a.null_count, 1);
1748
1749 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1750 assert_eq!(col_b.null_count, 1);
1751 }
1752
1753 #[test]
1754 fn test_analyze_json_with_missing_fields() {
1755 let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
1756 let config = JsonParserConfig::default();
1757 let report = analyze_json_file(json.path(), &config).unwrap();
1758 let profiles = &report.column_profiles;
1759
1760 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1761 assert_eq!(col_b.total_count, 2);
1762 }
1763
1764 const UNSORTED_FIELDS: &str = r#"{"id":1,"amount":12.5,"active":true,"date":"2026-07-23"}"#;
1767 const UNSORTED_ORDER: [&str; 4] = ["id", "amount", "active", "date"];
1768
1769 fn profile_names(report: &ProfileReport) -> Vec<&str> {
1770 report
1771 .column_profiles
1772 .iter()
1773 .map(|profile| profile.name.as_str())
1774 .collect()
1775 }
1776
1777 #[test]
1778 fn test_json_column_order_follows_source_not_alphabet() {
1779 let array = write_file(&format!("[{UNSORTED_FIELDS},{UNSORTED_FIELDS}]"));
1780 let jsonl = write_file(&format!("{UNSORTED_FIELDS}\n{UNSORTED_FIELDS}\n"));
1781
1782 for file in [&array, &jsonl] {
1783 let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1784 assert_eq!(profile_names(&report), UNSORTED_ORDER);
1785 }
1786 }
1787
1788 #[test]
1789 fn test_json_column_order_appends_later_keys_in_first_seen_order() {
1790 let data = b"{\"zulu\":1,\"mike\":2}\n{\"zulu\":3,\"mike\":4,\"delta\":5,\"alpha\":6}\n";
1793 let file = write_bytes(data);
1794 let report = analyze_json_file(file.path(), &JsonParserConfig::jsonl()).unwrap();
1795
1796 assert_eq!(profile_names(&report), ["zulu", "mike", "delta", "alpha"]);
1797 }
1798
1799 #[test]
1800 fn test_json_column_order_survives_report_serialization() {
1801 let file = write_file(&format!("[{UNSORTED_FIELDS}]"));
1802 let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1803
1804 let json = serde_json::to_string(&report).unwrap();
1805 let restored: Value = serde_json::from_str(&json).unwrap();
1806 let names: Vec<&str> = restored["column_profiles"]
1807 .as_array()
1808 .unwrap()
1809 .iter()
1810 .map(|profile| profile["name"].as_str().unwrap())
1811 .collect();
1812
1813 assert_eq!(names, UNSORTED_ORDER);
1814 }
1815
1816 #[test]
1817 fn test_analyze_json_backfills_columns_discovered_after_first_row() {
1818 let json = write_file(r#"[{"a":1},{"a":2,"b":3}]"#);
1819 let config = JsonParserConfig::default();
1820 let report = analyze_json_file(json.path(), &config).unwrap();
1821 let col_b = report
1822 .column_profiles
1823 .iter()
1824 .find(|profile| profile.name == "b")
1825 .unwrap();
1826
1827 assert_eq!(col_b.total_count, 2);
1828 assert_eq!(col_b.null_count, 1);
1829 assert_eq!(col_b.unique_count, Some(1));
1830 }
1831
1832 #[test]
1833 fn test_analyze_json_empty_array() {
1834 let json = write_file("[]");
1835 let config = JsonParserConfig::default();
1836 let report = analyze_json_file(json.path(), &config).unwrap();
1837 assert!(report.column_profiles.is_empty());
1838 assert_eq!(report.quality_score(), None);
1840 }
1841
1842 #[test]
1843 fn test_analyze_json_malformed_returns_error() {
1844 let json = write_file("this is entirely invalid json");
1845 let config = JsonParserConfig::default();
1846 let err = analyze_json_file(json.path(), &config)
1847 .expect_err("malformed JSON should return an error");
1848
1849 let message = err.to_string().to_lowercase();
1850 assert!(message.contains("malformed") && message.contains("json"));
1851 }
1852
1853 #[test]
1854 fn test_analyze_json_file_detects_format() {
1855 let json_array = write_file(r#"[{"x":1}]"#);
1856 let config = JsonParserConfig::default();
1857 let report = analyze_json_file(json_array.path(), &config).unwrap();
1858 assert!(matches!(
1859 report.data_source,
1860 DataSource::File {
1861 format: FileFormat::Json,
1862 ..
1863 }
1864 ));
1865
1866 let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
1867 let report = analyze_json_file(jsonl.path(), &config).unwrap();
1868 assert!(matches!(
1869 report.data_source,
1870 DataSource::File {
1871 format: FileFormat::Jsonl,
1872 ..
1873 }
1874 ));
1875 }
1876
1877 #[test]
1878 fn test_bom_prefixed_file_preserves_source_size() {
1879 let data = b"\xEF\xBB\xBF[{\"x\":1}]";
1880 let json = write_bytes(data);
1881 let report = analyze_json_file(json.path(), &JsonParserConfig::json_document()).unwrap();
1882
1883 assert_eq!(report.execution.rows_processed, 1);
1884 assert!(matches!(
1885 report.data_source,
1886 DataSource::File {
1887 format: FileFormat::Json,
1888 size_bytes,
1889 ..
1890 } if size_bytes == data.len() as u64
1891 ));
1892 }
1893
1894 #[test]
1895 fn test_analyze_json_file_empty() {
1896 let json = write_file("");
1897 let config = JsonParserConfig::default();
1898 let report = analyze_json_file(json.path(), &config).unwrap();
1899 assert_eq!(report.execution.rows_processed, 0);
1900 assert!(report.column_profiles.is_empty());
1901 }
1902
1903 #[test]
1904 fn test_analyze_json_boolean_and_nested() {
1905 let json =
1906 write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
1907 let config = JsonParserConfig::default();
1908 let report = analyze_json_file(json.path(), &config).unwrap();
1909 let profiles = &report.column_profiles;
1910
1911 let flag = profiles
1912 .iter()
1913 .find(|profile| profile.name == "flag")
1914 .unwrap();
1915 assert_eq!(flag.total_count, 2);
1916
1917 let nested = profiles
1918 .iter()
1919 .find(|profile| profile.name == "nested")
1920 .unwrap();
1921 assert_eq!(nested.total_count, 2);
1922 }
1923
1924 #[test]
1925 fn test_single_root_object_compact_yields_one_row() {
1926 let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
1927 let cursor = Cursor::new(data.as_ref());
1928 let config = JsonParserConfig::default();
1929
1930 let (profiles, _stats, rows, malformed, _format) =
1931 analyze_json_from_reader(cursor, &config).unwrap();
1932 assert_eq!(rows, 1);
1933 assert_eq!(malformed, 0);
1934 assert_eq!(profiles.len(), 2);
1935
1936 let names: Vec<&str> = profiles
1937 .iter()
1938 .map(|profile| profile.name.as_str())
1939 .collect();
1940 assert!(names.contains(&"type"));
1941 assert!(names.contains(&"features"));
1942 }
1943
1944 #[test]
1945 fn test_jsonl_multi_object_still_works() {
1946 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1947 let cursor = Cursor::new(data.as_ref());
1948 let config = JsonParserConfig::default();
1949
1950 let (_profiles, _stats, rows, _malformed, format) =
1951 analyze_json_from_reader(cursor, &config).unwrap();
1952 assert_eq!(format, FileFormat::Jsonl);
1953 assert_eq!(rows, 3);
1954 }
1955
1956 #[test]
1957 fn test_single_root_object_via_analyze_json_file() {
1958 let file =
1959 write_json_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
1960 let config = JsonParserConfig::default();
1961 let report = analyze_json_file(file.path(), &config).unwrap();
1962
1963 assert_eq!(report.execution.rows_processed, 1);
1964 assert_eq!(report.column_profiles.len(), 2);
1965 }
1966
1967 #[test]
1968 fn test_single_root_object_pretty_printed_yields_one_row() {
1969 let data = br#"{
1970 "type": "FeatureCollection",
1971 "features": [
1972 {"id": 1},
1973 {"id": 2}
1974 ]
1975}"#;
1976 let cursor = Cursor::new(data.as_ref());
1977 let config = JsonParserConfig::json_document();
1980
1981 let (profiles, _stats, rows, malformed, format) =
1982 analyze_json_from_reader(cursor, &config).unwrap();
1983 assert_eq!(format, FileFormat::Json);
1984 assert_eq!(rows, 1);
1985 assert_eq!(malformed, 0);
1986 assert_eq!(profiles.len(), 2);
1987 }
1988
1989 #[test]
1990 fn test_single_root_object_pretty_printed_via_analyze_json_file() {
1991 let file = write_json_file(
1992 "{\n \"type\": \"FeatureCollection\",\n \"features\": [\n {\"id\": 1},\n {\"id\": 2}\n ]\n}\n",
1993 );
1994 let config = JsonParserConfig::default();
1995 let report = analyze_json_file(file.path(), &config).unwrap();
1996
1997 assert_eq!(report.execution.rows_processed, 1);
1998 assert_eq!(report.column_profiles.len(), 2);
1999 }
2000}