1use std::collections::{HashMap, HashSet};
30use std::io::BufRead;
31use std::path::Path;
32
33pub use dataprof_core::JsonErrorPolicy;
34use dataprof_core::{
35 AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat,
36 QualityDimension, SemanticHints, TruncationReason, Utf8BomReader,
37};
38use dataprof_runtime::{
39 ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
40};
41use serde::Deserialize;
42use serde_json::Value;
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
51pub enum JsonFormat {
52 Json,
57 Jsonl,
60}
61
62#[derive(Debug, Clone, Default)]
96pub struct JsonParserConfig {
97 pub format: Option<JsonFormat>,
104 pub max_rows: Option<usize>,
106 pub error_policy: JsonErrorPolicy,
108}
109
110impl JsonParserConfig {
111 pub fn with_max_rows(mut self, max_rows: usize) -> Self {
148 self.max_rows = Some(max_rows);
149 self
150 }
151
152 pub fn with_error_policy(mut self, policy: JsonErrorPolicy) -> Self {
180 self.error_policy = policy;
181 self
182 }
183
184 pub fn jsonl() -> Self {
186 Self {
187 format: Some(JsonFormat::Jsonl),
188 ..Default::default()
189 }
190 }
191
192 pub fn json_document() -> Self {
195 Self {
196 format: Some(JsonFormat::Json),
197 ..Default::default()
198 }
199 }
200
201 pub fn with_format(mut self, format: JsonFormat) -> Self {
203 self.format = Some(format);
204 self
205 }
206}
207
208pub type JsonObject = serde_json::Map<String, Value>;
215
216#[derive(Debug, Clone, PartialEq, Eq)]
218#[non_exhaustive]
219pub struct JsonScanSummary {
220 pub rows_read: usize,
221 pub malformed_lines: usize,
225 pub format: FileFormat,
226 pub truncated: bool,
229}
230
231pub fn scan_json_from_reader<R, F>(
316 reader: R,
317 config: &JsonParserConfig,
318 mut on_object: F,
319) -> Result<JsonScanSummary, DataProfilerError>
320where
321 R: BufRead,
322 F: FnMut(&JsonObject),
323{
324 let mut reader = Utf8BomReader::new(reader).map_err(DataProfilerError::from)?;
325 let format = match config.format {
326 Some(JsonFormat::Json) => FileFormat::Json,
327 Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
328 None => match consume_leading_whitespace(&mut reader)? {
332 Some(b'[') => FileFormat::Json,
333 _ => FileFormat::Jsonl,
334 },
335 };
336
337 let mut rows_read = 0;
338 let mut malformed_lines = 0;
339 let mut truncated = false;
340
341 match format {
342 FileFormat::Jsonl => {
348 let mut line = String::new();
349 let mut line_number = 0;
350 loop {
351 if let Some(max) = config.max_rows
352 && rows_read >= max
353 {
354 truncated = next_record_line(&mut reader, &mut line, &mut line_number)?;
358 break;
359 }
360
361 if !next_record_line(&mut reader, &mut line, &mut line_number)? {
362 break;
363 }
364
365 match read_jsonl_line(&line) {
366 JsonRecord::Object(obj) => {
367 on_object(&obj);
368 rows_read += 1;
369 }
370 JsonRecord::NonObject(kind) => {
371 if config.error_policy == JsonErrorPolicy::Strict {
372 return Err(non_object_record_error(
373 kind,
374 rows_read + malformed_lines + 1,
375 ));
376 }
377 malformed_lines += 1;
378 }
379 JsonRecord::Malformed(err) => {
380 if config.error_policy == JsonErrorPolicy::Strict {
381 return Err(malformed_jsonl_record_error(line_number, err));
382 }
383 malformed_lines += 1;
384 }
385 }
386 }
387 }
388 FileFormat::Json => {
389 let found_array = consume_leading_whitespace(&mut reader)? == Some(b'[');
393
394 if !found_array {
395 let (object_rows, object_malformed, object_truncated) =
399 scan_single_json_object(&mut reader, config, &mut on_object)?;
400 rows_read += object_rows;
401 malformed_lines += object_malformed;
402 truncated |= object_truncated;
403 } else {
404 consume_peeked(&mut reader)?;
406
407 let mut expect_value = true;
408 let mut allow_end = true;
409 let mut array_closed = false;
410 let mut drain_remainder = false;
411
412 loop {
413 let Some(next) = consume_leading_whitespace(&mut reader)? else {
414 if config.error_policy == JsonErrorPolicy::Strict {
415 return Err(malformed_array_error(
416 "unexpected end of input before closing ']'",
417 ));
418 }
419 malformed_lines += 1;
420 drain_remainder = true;
421 break;
422 };
423
424 if expect_value {
425 if next == b']' {
426 if !allow_end {
427 if config.error_policy == JsonErrorPolicy::Strict {
428 return Err(malformed_array_error(
429 "trailing comma before closing ']'",
430 ));
431 }
432 malformed_lines += 1;
433 drain_remainder = true;
434 break;
435 }
436 consume_peeked(&mut reader)?;
437 array_closed = true;
438 break;
439 }
440
441 if let Some(max) = config.max_rows
442 && rows_read >= max
443 {
444 truncated = true;
447 break;
448 }
449
450 if next == b',' {
451 if config.error_policy == JsonErrorPolicy::Strict {
452 return Err(malformed_array_error(
453 "unexpected comma where an array value was required",
454 ));
455 }
456 malformed_lines += 1;
457 drain_remainder = true;
458 break;
459 }
460
461 match read_json_record(&mut reader, next)? {
462 JsonRecord::Object(obj) => {
463 on_object(&obj);
464 rows_read += 1;
465 }
466 JsonRecord::NonObject(kind) => {
467 if config.error_policy == JsonErrorPolicy::Strict {
468 return Err(non_object_record_error(
469 kind,
470 rows_read + malformed_lines + 1,
471 ));
472 }
473 malformed_lines += 1;
477 }
478 JsonRecord::Malformed(err) => {
479 if config.error_policy == JsonErrorPolicy::Strict {
480 return Err(malformed_record_error(err));
481 }
482 malformed_lines += 1;
483 drain_remainder = true;
484 break;
485 }
486 }
487 expect_value = false;
488 } else {
489 match next {
490 b',' => {
491 consume_peeked(&mut reader)?;
492 expect_value = true;
493 allow_end = false;
494 }
495 b']' => {
496 consume_peeked(&mut reader)?;
497 array_closed = true;
498 break;
499 }
500 _ => {
501 if config.error_policy == JsonErrorPolicy::Strict {
502 return Err(malformed_array_error(
503 "expected ',' or ']' after an array value",
504 ));
505 }
506 malformed_lines += 1;
507 drain_remainder = true;
508 break;
509 }
510 }
511 }
512 }
513
514 if drain_remainder {
515 drain_to_end(&mut reader)?;
516 } else if array_closed && consume_leading_whitespace(&mut reader)?.is_some() {
517 if config.error_policy == JsonErrorPolicy::Strict {
518 return Err(malformed_array_error(
519 "non-whitespace content follows the closing ']'",
520 ));
521 }
522 malformed_lines += 1;
523 drain_to_end(&mut reader)?;
524 }
525 }
526 }
527 _ => unreachable!("json scanner only returns json or jsonl formats"),
528 }
529
530 Ok(JsonScanSummary {
531 rows_read,
532 malformed_lines,
533 format,
534 truncated,
535 })
536}
537
538fn next_record_line<R: BufRead>(
545 reader: &mut R,
546 line: &mut String,
547 line_number: &mut usize,
548) -> Result<bool, DataProfilerError> {
549 loop {
550 line.clear();
551 let read = reader.read_line(line).map_err(DataProfilerError::from)?;
552 if read == 0 {
553 return Ok(false);
554 }
555 *line_number += 1;
556 if !line.trim().is_empty() {
557 return Ok(true);
558 }
559 }
560}
561
562fn read_jsonl_line(line: &str) -> JsonRecord {
569 match serde_json::from_str::<Value>(line.trim()) {
570 Ok(Value::Object(obj)) => JsonRecord::Object(obj),
571 Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
572 Err(err) => JsonRecord::Malformed(err),
573 }
574}
575
576fn scan_single_json_object<R, F>(
583 reader: &mut R,
584 config: &JsonParserConfig,
585 on_object: &mut F,
586) -> Result<(usize, usize, bool), DataProfilerError>
587where
588 R: BufRead,
589 F: FnMut(&JsonObject),
590{
591 let mut text = String::new();
592 reader
593 .read_to_string(&mut text)
594 .map_err(DataProfilerError::from)?;
595
596 if text.trim().is_empty() {
597 return Ok((0, 0, false));
598 }
599
600 if config.max_rows == Some(0) {
605 return Ok((0, 0, true));
606 }
607
608 match serde_json::from_str::<Value>(text.trim()) {
609 Ok(Value::Object(obj)) => {
610 on_object(&obj);
611 Ok((1, 0, false))
612 }
613 Ok(value) => {
614 if config.error_policy == JsonErrorPolicy::Strict {
615 return Err(non_object_record_error(json_value_kind(&value), 1));
616 }
617 Ok((0, 1, false))
618 }
619 Err(err) => {
620 if config.error_policy == JsonErrorPolicy::Strict {
621 return Err(json_document_error(err));
622 }
623 Ok((0, 1, false))
624 }
625 }
626}
627
628enum JsonRecord {
630 Object(JsonObject),
632 NonObject(&'static str),
635 Malformed(serde_json::Error),
637}
638
639fn read_json_record<R: BufRead>(
644 reader: &mut R,
645 first: u8,
646) -> Result<JsonRecord, DataProfilerError> {
647 if first == b'-' || first.is_ascii_digit() {
652 let token = read_number_token(reader)?;
653 return Ok(match serde_json::from_str::<Value>(&token) {
654 Ok(_) => JsonRecord::NonObject("number"),
655 Err(err) => JsonRecord::Malformed(err),
656 });
657 }
658
659 let mut deserializer = serde_json::Deserializer::from_reader(reader);
660 Ok(match Value::deserialize(&mut deserializer) {
661 Ok(Value::Object(obj)) => JsonRecord::Object(obj),
662 Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
663 Err(err) => JsonRecord::Malformed(err),
664 })
665}
666
667fn read_number_token<R: BufRead>(reader: &mut R) -> Result<String, DataProfilerError> {
675 skip_ascii_whitespace(reader)?;
676 let mut token = String::new();
677 loop {
678 let (consume, token_ended) = {
679 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
680 if buf.is_empty() {
681 break;
682 }
683 let consume = buf
684 .iter()
685 .take_while(|byte| is_json_number_byte(**byte))
686 .count();
687 token.push_str(
689 std::str::from_utf8(&buf[..consume]).expect("JSON number bytes are ASCII"),
690 );
691 (consume, consume < buf.len())
692 };
693 reader.consume(consume);
694 if token_ended {
695 break;
696 }
697 }
698 Ok(token)
699}
700
701fn is_json_number_byte(byte: u8) -> bool {
702 byte.is_ascii_digit() || matches!(byte, b'-' | b'+' | b'.' | b'e' | b'E')
703}
704
705fn consume_peeked<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
714 skip_ascii_whitespace(reader)?;
715 reader.consume(1);
716 Ok(())
717}
718
719fn skip_ascii_whitespace<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
721 loop {
722 let (consume, done) = {
723 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
724 if buf.is_empty() {
725 return Ok(());
726 }
727 let consume = buf
728 .iter()
729 .take_while(|byte| byte.is_ascii_whitespace())
730 .count();
731 (consume, consume < buf.len())
732 };
733 reader.consume(consume);
734 if done {
735 return Ok(());
736 }
737 }
738}
739
740fn json_value_kind(value: &Value) -> &'static str {
741 match value {
742 Value::Null => "null",
743 Value::Bool(_) => "boolean",
744 Value::Number(_) => "number",
745 Value::String(_) => "string",
746 Value::Array(_) => "array",
747 Value::Object(_) => "object",
748 }
749}
750
751fn non_object_record_error(kind: &str, position: usize) -> DataProfilerError {
755 DataProfilerError::json_parsing_error(&format!(
758 "non-object JSON record at position {position}: \
759 expected an object with fields to profile, found {kind}"
760 ))
761}
762
763fn malformed_record_error(err: serde_json::Error) -> DataProfilerError {
767 let message = format!("malformed JSON record: {err}");
768 DataProfilerError::json_parsing_with_source(message, err)
769}
770
771fn malformed_jsonl_record_error(line: usize, err: serde_json::Error) -> DataProfilerError {
779 let message = format!(
780 "malformed JSON record on line {line}, column {}: a JSONL record must be one complete JSON value on one line",
781 err.column()
782 );
783 DataProfilerError::json_parsing_with_source(message, err)
784}
785
786fn malformed_array_error(message: &str) -> DataProfilerError {
787 DataProfilerError::json_parsing_error(&format!("malformed JSON array: {message}"))
790}
791
792fn json_document_error(err: serde_json::Error) -> DataProfilerError {
799 let message = format!(
800 "malformed JSON document: {err}. A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
804 );
805 DataProfilerError::json_parsing_with_source(message, err)
806}
807
808fn drain_to_end<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
809 loop {
810 let available = reader.fill_buf().map_err(DataProfilerError::from)?.len();
811 if available == 0 {
812 return Ok(());
813 }
814 reader.consume(available);
815 }
816}
817
818fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
819 loop {
820 let mut bytes_to_consume = 0;
821 let first_non_whitespace = {
822 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
823 if buf.is_empty() {
824 return Ok(None);
825 }
826
827 let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
828 if first_non_whitespace.is_none() {
829 bytes_to_consume = buf.len();
830 }
831 first_non_whitespace
832 };
833
834 if first_non_whitespace.is_some() {
835 return Ok(first_non_whitespace);
836 }
837
838 reader.consume(bytes_to_consume);
839 }
840}
841
842fn json_value_to_string(value: &Value) -> String {
844 match value {
845 Value::Null => String::new(),
846 Value::Bool(boolean) => boolean.to_string(),
847 Value::Number(number) => number.to_string(),
848 Value::String(string) => string.to_string(),
849 Value::Array(_) | Value::Object(_) => {
852 serde_json::to_string(value).expect("re-serializing a parsed JSON value cannot fail")
853 }
854 }
855}
856
857fn feed_json_object(
867 obj: &JsonObject,
868 prior_rows: usize,
869 known_columns: &mut Vec<String>,
870 known_columns_set: &mut HashSet<String>,
871 column_stats: &mut StreamingColumnCollection,
872 container_counts: &mut HashMap<String, usize>,
873) {
874 for (key, value) in obj {
875 if known_columns_set.insert(key.clone()) {
876 known_columns.push(key.clone());
877 column_stats.init_column_with_missing(key, prior_rows);
878 }
879 if matches!(value, Value::Array(_) | Value::Object(_)) {
880 *container_counts.entry(key.clone()).or_default() += 1;
881 }
882 }
883
884 let values: Vec<String> = known_columns
885 .iter()
886 .map(|column| {
887 obj.get(column)
890 .map(json_value_to_string)
891 .unwrap_or_default()
892 })
893 .collect();
894
895 column_stats.process_record(known_columns, values);
896}
897
898pub fn analyze_json_from_reader<R: BufRead>(
940 reader: R,
941 config: &JsonParserConfig,
942) -> Result<
943 (
944 Vec<ColumnProfile>,
945 StreamingColumnCollection,
946 usize,
947 usize,
948 FileFormat,
949 ),
950 DataProfilerError,
951> {
952 analyze_json_from_reader_with_hints(reader, config, &SemanticHints::default())
953}
954
955pub fn analyze_json_from_reader_with_hints<R: BufRead>(
956 reader: R,
957 config: &JsonParserConfig,
958 semantic_hints: &SemanticHints,
959) -> Result<
960 (
961 Vec<ColumnProfile>,
962 StreamingColumnCollection,
963 usize,
964 usize,
965 FileFormat,
966 ),
967 DataProfilerError,
968> {
969 let options = AnalysisOptions::default().with_semantic_hints(semantic_hints.clone());
970 let (profiles, stats, rows_read, malformed_lines, format, _truncated) =
971 analyze_json_from_reader_full(reader, config, &options)?;
972 Ok((profiles, stats, rows_read, malformed_lines, format))
973}
974
975#[allow(clippy::type_complexity)]
978fn analyze_json_from_reader_full<R: BufRead>(
979 reader: R,
980 config: &JsonParserConfig,
981 options: &AnalysisOptions,
982) -> Result<
983 (
984 Vec<ColumnProfile>,
985 StreamingColumnCollection,
986 usize,
987 usize,
988 FileFormat,
989 bool,
990 ),
991 DataProfilerError,
992> {
993 let semantic_hints = options.semantic_hints();
994 let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
995 let mut known_columns = Vec::new();
996 let mut known_columns_set = HashSet::new();
997 let mut container_counts = HashMap::new();
998 let mut rows_seen = 0;
999
1000 let summary = scan_json_from_reader(reader, config, |obj| {
1001 feed_json_object(
1002 obj,
1003 rows_seen,
1004 &mut known_columns,
1005 &mut known_columns_set,
1006 &mut column_stats,
1007 &mut container_counts,
1008 );
1009 rows_seen += 1;
1010 })?;
1011
1012 if let Some(indices) = options.column_indices(&column_stats.column_names())? {
1013 let available = column_stats.column_names();
1014 let selected = indices
1015 .into_iter()
1016 .map(|index| available[index].clone())
1017 .collect::<Vec<_>>();
1018 column_stats.retain_columns(&selected);
1019 }
1020
1021 let profiles = profile_builder::mark_container_columns(
1022 profile_builder::profiles_from_streaming_with_hints(
1023 &column_stats,
1024 !options.include_statistics(),
1025 !options.include_patterns(),
1026 options.locale(),
1027 semantic_hints,
1028 ),
1029 &container_counts,
1030 );
1031
1032 Ok((
1033 profiles,
1034 column_stats,
1035 summary.rows_read,
1036 summary.malformed_lines,
1037 summary.format,
1038 summary.truncated,
1039 ))
1040}
1041
1042pub fn analyze_json_file(
1070 file_path: &Path,
1071 config: &JsonParserConfig,
1072) -> Result<ProfileReport, DataProfilerError> {
1073 analyze_json_file_with_dimensions(file_path, config, None)
1074}
1075
1076pub fn analyze_json_file_with_dimensions(
1078 file_path: &Path,
1079 config: &JsonParserConfig,
1080 quality_dimensions: Option<&[QualityDimension]>,
1081) -> Result<ProfileReport, DataProfilerError> {
1082 analyze_json_file_with_dimensions_and_hints(
1083 file_path,
1084 config,
1085 quality_dimensions,
1086 &SemanticHints::default(),
1087 )
1088}
1089
1090pub fn analyze_json_file_with_dimensions_and_hints(
1091 file_path: &Path,
1092 config: &JsonParserConfig,
1093 quality_dimensions: Option<&[QualityDimension]>,
1094 semantic_hints: &SemanticHints,
1095) -> Result<ProfileReport, DataProfilerError> {
1096 let options = AnalysisOptions::default()
1097 .with_quality_dimensions(quality_dimensions.map(<[_]>::to_vec))
1098 .with_semantic_hints(semantic_hints.clone());
1099 analyze_json_file_with_options(file_path, config, &options)
1100}
1101
1102pub fn analyze_json_file_with_options(
1108 file_path: &Path,
1109 config: &JsonParserConfig,
1110 options: &AnalysisOptions,
1111) -> Result<ProfileReport, DataProfilerError> {
1112 let mut config = config.clone();
1115 if config.format.is_none() {
1116 config.format = grammar_for_extension(file_path);
1117 }
1118 let config = &config;
1119 let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
1120 let start = std::time::Instant::now();
1121
1122 let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
1123 let buf_reader = std::io::BufReader::new(file);
1124
1125 let (column_profiles, column_stats, rows_read, malformed_lines, format, truncated) =
1126 analyze_json_from_reader_full(buf_reader, config, options)?;
1127 let read_as_json_document = format == FileFormat::Json;
1128
1129 let file_source = DataSource::File {
1130 path: file_path.display().to_string(),
1131 format,
1132 size_bytes: metadata.len(),
1133 modified_at: None,
1134 parquet_metadata: None,
1135 };
1136
1137 if rows_read == 0 {
1138 if malformed_lines > 0 {
1139 let hint = if read_as_json_document {
1143 ". A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
1144 } else {
1145 ""
1146 };
1147 return Err(DataProfilerError::json_parsing_error(&format!(
1150 "No valid JSON records found in file \
1151 (every record was malformed or not a JSON object){hint}"
1152 )));
1153 }
1154 let mut execution =
1159 ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("json");
1160 if truncated && let Some(max) = config.max_rows {
1161 execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1162 }
1163 return Ok(ReportAssembler::new(file_source, execution)
1164 .columns(column_profiles)
1165 .with_quality_data(HashMap::new())
1166 .with_analysis_options(options)
1167 .build());
1168 }
1169
1170 let sample_columns = profile_builder::quality_check_samples(&column_stats);
1171 let scan_time_ms = start.elapsed().as_millis();
1172 let num_columns = column_profiles.len();
1173
1174 let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms)
1175 .with_engine("json")
1176 .with_error_count(malformed_lines);
1179 if truncated && let Some(max) = config.max_rows {
1182 execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1183 }
1184
1185 Ok(ReportAssembler::new(file_source, execution)
1186 .columns(column_profiles)
1187 .with_quality_data(sample_columns)
1188 .with_row_duplicates(column_stats.row_duplicate_summary())
1189 .with_row_completeness(column_stats.row_completeness_summary())
1190 .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
1191 .with_analysis_options(options)
1192 .build())
1193}
1194
1195fn grammar_for_extension(file_path: &Path) -> Option<JsonFormat> {
1203 let extension = file_path.extension().and_then(|ext| ext.to_str())?;
1204 if extension.eq_ignore_ascii_case("json") {
1205 Some(JsonFormat::Json)
1206 } else if extension.eq_ignore_ascii_case("jsonl") || extension.eq_ignore_ascii_case("ndjson") {
1207 Some(JsonFormat::Jsonl)
1208 } else {
1209 None
1210 }
1211}
1212
1213fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
1214 if error.kind() == std::io::ErrorKind::NotFound {
1215 DataProfilerError::FileNotFound {
1216 path: file_path.display().to_string(),
1217 }
1218 } else {
1219 DataProfilerError::from(error)
1220 }
1221}
1222
1223#[cfg(test)]
1224mod tests {
1225 use super::*;
1226 use std::io::{Cursor, Write};
1227 use tempfile::NamedTempFile;
1228
1229 fn write_file(content: &str) -> NamedTempFile {
1232 let mut file = NamedTempFile::new().unwrap();
1233 write!(file, "{}", content).unwrap();
1234 file.flush().unwrap();
1235 file
1236 }
1237
1238 fn write_json_file(content: &str) -> NamedTempFile {
1241 let mut file = NamedTempFile::with_suffix(".json").unwrap();
1242 write!(file, "{}", content).unwrap();
1243 file.flush().unwrap();
1244 file
1245 }
1246
1247 fn write_bytes(content: &[u8]) -> NamedTempFile {
1248 let mut file = NamedTempFile::new().unwrap();
1249 file.write_all(content).unwrap();
1250 file.flush().unwrap();
1251 file
1252 }
1253
1254 #[test]
1255 fn test_scan_jsonl_detects_format_and_rows() {
1256 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1257 let mut rows = 0;
1258
1259 let summary = scan_json_from_reader(
1260 Cursor::new(data.as_ref()),
1261 &JsonParserConfig::default(),
1262 |_| {
1263 rows += 1;
1264 },
1265 )
1266 .unwrap();
1267
1268 assert_eq!(summary.format, FileFormat::Jsonl);
1269 assert_eq!(summary.rows_read, 3);
1270 assert_eq!(summary.malformed_lines, 0);
1271 assert_eq!(rows, 3);
1272 }
1273
1274 #[test]
1275 fn test_scan_json_array_detects_format_and_rows() {
1276 let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
1277 let mut keys = Vec::new();
1278
1279 let summary = scan_json_from_reader(
1280 Cursor::new(data.as_ref()),
1281 &JsonParserConfig::default(),
1282 |obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
1283 )
1284 .unwrap();
1285
1286 assert_eq!(summary.format, FileFormat::Json);
1287 assert_eq!(summary.rows_read, 2);
1288 assert_eq!(keys.len(), 2);
1289 assert_eq!(keys[0], vec!["name".to_string()]);
1290 }
1291
1292 #[test]
1293 fn test_scan_accepts_one_leading_utf8_bom_for_json_and_jsonl() {
1294 for (data, config, expected_format) in [
1295 (
1296 b"\xEF\xBB\xBF[{\"x\":1},{\"x\":2}]".as_slice(),
1297 JsonParserConfig::default(),
1298 FileFormat::Json,
1299 ),
1300 (
1301 b"\xEF\xBB\xBF{\"x\":1}\n{\"x\":2}\n".as_slice(),
1302 JsonParserConfig::default(),
1303 FileFormat::Jsonl,
1304 ),
1305 ] {
1306 let mut rows = 0;
1307 let summary = scan_json_from_reader(Cursor::new(data), &config, |_| rows += 1)
1308 .expect("leading UTF-8 BOM should be accepted");
1309
1310 assert_eq!(rows, 2);
1311 assert_eq!(summary.rows_read, 2);
1312 assert_eq!(summary.malformed_lines, 0);
1313 assert_eq!(summary.format, expected_format);
1314 }
1315 }
1316
1317 #[test]
1318 fn test_scan_does_not_strip_nonleading_or_second_utf8_bom() {
1319 let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1320
1321 for data in [
1322 b" \xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1323 b"\xEF\xBB\xBF\xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1324 b"{\"x\":1}\n\xEF\xBB\xBF{\"x\":2}\n".as_slice(),
1325 ] {
1326 scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1327 .expect_err("only the BOM at absolute byte offset zero may be stripped");
1328 }
1329 }
1330
1331 #[test]
1332 fn test_bom_only_input_matches_empty_input_policy() {
1333 for config in [JsonParserConfig::default(), JsonParserConfig::jsonl()] {
1334 let empty = scan_json_from_reader(Cursor::new(b""), &config, |_| {}).unwrap();
1335 let bom =
1336 scan_json_from_reader(Cursor::new(b"\xEF\xBB\xBF".as_slice()), &config, |_| {})
1337 .unwrap();
1338 assert_eq!(bom, empty);
1339 }
1340 }
1341
1342 #[test]
1343 fn test_json_array_container_grammar_obeys_error_policy() {
1344 let cases = [
1345 (br#"[{"x":1}"#.as_ref(), 1, "missing closing bracket"),
1346 (br#"[{"x":1},"#.as_ref(), 1, "comma followed by EOF"),
1347 (br#"[{"x":1} {"x":2}]"#.as_ref(), 1, "missing comma"),
1348 (br#"[{"x":1},,{"x":2}]"#.as_ref(), 1, "doubled comma"),
1349 (br#"[,{"x":1}]"#.as_ref(), 0, "leading comma"),
1350 (br#"[{"x":1},]"#.as_ref(), 1, "trailing comma"),
1351 (br#"[{"x":1}] trailing"#.as_ref(), 1, "trailing content"),
1352 (
1353 br#"[{"x":1}][{"x":2}]"#.as_ref(),
1354 1,
1355 "second top-level value",
1356 ),
1357 ];
1358
1359 for (data, expected_rows, case) in cases {
1360 let skip = JsonParserConfig::json_document();
1361 let summary =
1362 scan_json_from_reader(Cursor::new(data), &skip, |_| {}).expect("tolerant scan");
1363 assert_eq!(summary.rows_read, expected_rows, "{case}");
1364 assert_eq!(summary.malformed_lines, 1, "{case}");
1365
1366 let strict =
1367 JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict);
1368 let err = scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1369 .expect_err("strict mode must reject invalid array grammar");
1370 assert!(
1371 err.to_string()
1372 .to_lowercase()
1373 .contains("malformed json array"),
1374 "{case}: {err}"
1375 );
1376 }
1377 }
1378
1379 #[test]
1380 fn test_json_array_max_rows_does_not_validate_unread_values() {
1381 let data = br#"[{"x":1},not-valid-json"#;
1382 let config = JsonParserConfig::json_document().with_max_rows(1);
1383 let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1384
1385 assert_eq!(summary.rows_read, 1);
1386 assert_eq!(summary.malformed_lines, 0);
1387 assert!(summary.truncated);
1388 }
1389
1390 #[test]
1391 fn test_scan_respects_max_rows() {
1392 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
1393 let config = JsonParserConfig::default().with_max_rows(2);
1394 let mut rows = 0;
1395
1396 let summary =
1397 scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
1398
1399 assert_eq!(summary.rows_read, 2);
1400 assert_eq!(rows, 2);
1401 }
1402
1403 #[test]
1404 fn test_scan_skips_malformed_jsonl_lines() {
1405 let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
1406 let mut rows = 0;
1407
1408 let summary = scan_json_from_reader(
1409 Cursor::new(data.as_ref()),
1410 &JsonParserConfig::jsonl(),
1411 |_| {
1412 rows += 1;
1413 },
1414 )
1415 .unwrap();
1416
1417 assert_eq!(summary.format, FileFormat::Jsonl);
1418 assert_eq!(summary.rows_read, 2);
1419 assert_eq!(summary.malformed_lines, 1);
1420 assert_eq!(rows, 2);
1421 }
1422
1423 #[test]
1429 fn test_pretty_printed_root_object_is_one_row_under_the_json_grammar() {
1430 let data = br#"{
1431 "type": "FeatureCollection",
1432 "features": [
1433 {"id": 1},
1434 {"id": 2}
1435 ]
1436}"#;
1437 let mut object_field_counts = Vec::new();
1438
1439 let summary = scan_json_from_reader(
1440 Cursor::new(data.as_ref()),
1441 &JsonParserConfig::json_document(),
1442 |obj| object_field_counts.push(obj.len()),
1443 )
1444 .unwrap();
1445
1446 assert_eq!(summary.format, FileFormat::Json);
1447 assert_eq!(summary.rows_read, 1);
1448 assert_eq!(summary.malformed_lines, 0);
1449 assert_eq!(object_field_counts, vec![2]);
1450 }
1451
1452 #[test]
1453 fn test_pretty_printed_root_object_is_not_valid_jsonl() {
1454 let data = br#"{
1455 "type": "FeatureCollection"
1456}"#;
1457 let err = scan_json_from_reader(
1458 Cursor::new(data.as_ref()),
1459 &JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict),
1460 |_| {},
1461 )
1462 .expect_err("a record spanning lines is not JSONL");
1463
1464 assert!(
1465 err.to_string().contains("malformed JSON record"),
1466 "{err}, expected the malformed category"
1467 );
1468 }
1469
1470 const NON_OBJECT_VALUES: &[(&str, &str)] = &[
1474 ("null", "null"),
1475 ("boolean", "true"),
1476 ("number", "42"),
1477 ("number", "-1.5e3"),
1478 ("string", r#""text""#),
1479 ("array", "[1, 2]"),
1480 ];
1481
1482 #[test]
1483 fn test_jsonl_counts_non_object_records_and_keeps_scanning() {
1484 for (kind, value) in NON_OBJECT_VALUES {
1485 let data = format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n");
1486 let mut rows = 0;
1487
1488 let summary = scan_json_from_reader(
1489 Cursor::new(data.as_bytes()),
1490 &JsonParserConfig::jsonl(),
1491 |_| rows += 1,
1492 )
1493 .unwrap();
1494
1495 assert_eq!(summary.rows_read, 2, "{kind}: record after it was lost");
1496 assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1497 assert_eq!(rows, 2, "{kind}");
1498 }
1499 }
1500
1501 #[test]
1502 fn test_json_array_counts_non_object_elements_and_keeps_scanning() {
1503 for (kind, value) in NON_OBJECT_VALUES {
1504 let data = format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]");
1505 let mut rows = 0;
1506
1507 let summary = scan_json_from_reader(
1508 Cursor::new(data.as_bytes()),
1509 &JsonParserConfig::json_document(),
1510 |_| rows += 1,
1511 )
1512 .unwrap();
1513
1514 assert_eq!(summary.rows_read, 2, "{kind}: element after it was lost");
1515 assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1516 assert_eq!(rows, 2, "{kind}");
1517 }
1518 }
1519
1520 #[test]
1521 fn test_strict_policy_rejects_the_first_non_object_record() {
1522 for (kind, value) in NON_OBJECT_VALUES {
1523 for (config, data) in [
1524 (
1525 JsonParserConfig::jsonl(),
1526 format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n"),
1527 ),
1528 (
1529 JsonParserConfig::json_document(),
1530 format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]"),
1531 ),
1532 ] {
1533 let config = config.with_error_policy(JsonErrorPolicy::Strict);
1534 let err = scan_json_from_reader(Cursor::new(data.as_bytes()), &config, |_| {})
1535 .expect_err("{kind}: strict mode must reject a non-object record");
1536
1537 let message = err.to_string();
1538 assert!(message.contains("non-object JSON record"), "{message}");
1539 assert!(message.contains("at position 2"), "{message}");
1540 assert!(message.contains(&format!("found {kind}")), "{message}");
1541 }
1542 }
1543 }
1544
1545 #[test]
1546 fn test_number_element_does_not_swallow_the_array_delimiter() {
1547 let data = br#"[{"id":1}, 1, 2, 3, {"id":2}, 4, {"id":3}]"#;
1550 let mut rows = 0;
1551
1552 let summary = scan_json_from_reader(
1553 Cursor::new(data.as_ref()),
1554 &JsonParserConfig::json_document(),
1555 |_| rows += 1,
1556 )
1557 .unwrap();
1558
1559 assert_eq!(summary.rows_read, 3);
1560 assert_eq!(summary.malformed_lines, 4);
1561 assert_eq!(rows, 3);
1562 }
1563
1564 #[test]
1565 fn test_number_shaped_garbage_stays_a_malformed_record() {
1566 let data = b"{\"id\":1}\n1.2.3\n";
1569 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1570 let err = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {})
1571 .expect_err("number-shaped garbage should be malformed");
1572
1573 assert!(
1574 err.to_string().contains("malformed JSON record"),
1575 "{err}, expected the malformed category"
1576 );
1577 }
1578
1579 #[test]
1580 fn test_input_of_only_non_object_records_fails() {
1581 let file = write_file("\"just a string\"\n1\n[2]\n");
1582 let err = analyze_json_file(file.path(), &JsonParserConfig::jsonl())
1583 .expect_err("nothing profileable should not return a clean empty profile");
1584
1585 assert!(
1586 err.to_string().contains("No valid JSON records found"),
1587 "{err}"
1588 );
1589 }
1590
1591 #[test]
1592 fn test_json_grammar_reads_a_bare_object_as_one_row() {
1595 let data = br#"{"x":1}"#;
1596 let mut rows = 0;
1597 let summary = scan_json_from_reader(
1598 Cursor::new(data.as_ref()),
1599 &JsonParserConfig::json_document(),
1600 |_| rows += 1,
1601 )
1602 .unwrap();
1603
1604 assert_eq!(summary.rows_read, 1);
1605 assert_eq!(summary.malformed_lines, 0);
1606 assert_eq!(rows, 1);
1607 }
1608
1609 #[test]
1610 fn test_json_grammar_rejects_concatenated_objects() {
1614 let data = br#"{"x":1}{"x":2}"#;
1615 let err = scan_json_from_reader(
1616 Cursor::new(data.as_ref()),
1617 &JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict),
1618 |_| {},
1619 )
1620 .expect_err("two objects are not one JSON document");
1621
1622 let message = err.to_string();
1623 assert!(message.contains("malformed JSON document"), "{message}");
1624 assert!(message.contains("format=\"jsonl\""), "{message}");
1625 }
1626
1627 #[test]
1628 fn test_analyze_json_from_reader_jsonl_streaming() {
1629 let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
1630 let cursor = Cursor::new(data.as_ref());
1631 let config = JsonParserConfig::default();
1632
1633 let (profiles, _stats, rows, _malformed, format) =
1634 analyze_json_from_reader(cursor, &config).unwrap();
1635 assert_eq!(format, FileFormat::Jsonl);
1636 assert_eq!(rows, 3);
1637 assert_eq!(profiles.len(), 2);
1638 }
1639
1640 #[test]
1641 fn test_analyze_json_from_reader_json_array() {
1642 let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
1643 let cursor = Cursor::new(data.as_ref());
1644 let config = JsonParserConfig::default();
1645
1646 let (profiles, _stats, rows, _malformed, format) =
1647 analyze_json_from_reader(cursor, &config).unwrap();
1648 assert_eq!(format, FileFormat::Json);
1649 assert_eq!(rows, 2);
1650 assert_eq!(profiles.len(), 2);
1651 }
1652
1653 #[test]
1654 fn test_analyze_json_from_reader_max_rows() {
1655 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
1656 let cursor = Cursor::new(data.as_ref());
1657 let config = JsonParserConfig::default().with_max_rows(3);
1658
1659 let (_profiles, _stats, rows, _malformed, _format) =
1660 analyze_json_from_reader(cursor, &config).unwrap();
1661 assert_eq!(rows, 3);
1662 }
1663
1664 #[test]
1665 fn test_analyze_json_from_reader_missing_fields() {
1666 let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
1667 let cursor = Cursor::new(data.as_ref());
1668 let config = JsonParserConfig::jsonl();
1669
1670 let (profiles, _stats, rows, _malformed, _format) =
1671 analyze_json_from_reader(cursor, &config).unwrap();
1672 assert_eq!(rows, 2);
1673
1674 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1675 assert_eq!(col_b.total_count, 2);
1676 }
1677
1678 #[test]
1679 fn test_identical_records_count_as_duplicates_across_a_late_key() {
1680 let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"x\"}\n";
1684 let cursor = Cursor::new(data.as_ref());
1685 let config = JsonParserConfig::jsonl();
1686
1687 let (_profiles, stats, rows, _malformed, _format) =
1688 analyze_json_from_reader(cursor, &config).unwrap();
1689 assert_eq!(rows, 3);
1690
1691 let summary = stats
1692 .row_duplicate_summary()
1693 .expect("three rows were observed");
1694 assert_eq!(summary.rows_checked, 3);
1695 assert_eq!(summary.duplicate_rows, 1);
1696 assert!(!summary.approximate);
1697 }
1698
1699 #[test]
1700 fn test_a_late_key_does_not_invent_duplicates() {
1701 let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"z\"}\n";
1704 let cursor = Cursor::new(data.as_ref());
1705 let config = JsonParserConfig::jsonl();
1706
1707 let (_profiles, stats, _rows, _malformed, _format) =
1708 analyze_json_from_reader(cursor, &config).unwrap();
1709
1710 let summary = stats
1711 .row_duplicate_summary()
1712 .expect("three rows were observed");
1713 assert_eq!(summary.duplicate_rows, 0);
1714 }
1715
1716 #[test]
1717 fn test_analyze_json_file_quality_report() {
1718 let file = write_file(r#"[{"x":1},{"x":2}]"#);
1719 let config = JsonParserConfig::default();
1720 let report = analyze_json_file(file.path(), &config).unwrap();
1721
1722 assert_eq!(report.execution.rows_processed, 2);
1723 assert_eq!(report.column_profiles.len(), 1);
1724 assert!(report.quality_score().unwrap() >= 0.0);
1725 }
1726
1727 #[test]
1728 fn test_jsonl_skips_malformed_lines() {
1729 let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
1730 let cursor = Cursor::new(data.as_ref());
1731 let config = JsonParserConfig::jsonl();
1732
1733 let (profiles, _stats, rows, _malformed, format) =
1734 analyze_json_from_reader(cursor, &config).unwrap();
1735 assert_eq!(format, FileFormat::Jsonl);
1736 assert_eq!(rows, 2);
1737 assert_eq!(profiles[0].total_count, 2);
1738 }
1739
1740 #[test]
1741 fn test_strict_policy_errors_on_malformed_jsonl() {
1742 for data in [
1744 b"not-json\n{\"x\":1}\n{\"x\":2}\n".as_ref(),
1745 b"{\"x\":1}\nnot-json\n{\"x\":2}\n".as_ref(),
1746 b"{\"x\":1}\n{\"x\":2}\nnot-json\n".as_ref(),
1747 ] {
1748 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1749 let err = scan_json_from_reader(Cursor::new(data), &config, |_| {})
1750 .expect_err("strict mode must reject malformed records");
1751 let message = err.to_string().to_lowercase();
1752 assert!(message.contains("malformed json record"), "got: {message}");
1753 assert!(
1755 !err.to_string().contains("not-json"),
1756 "leaked record: {err}"
1757 );
1758 }
1759 }
1760
1761 #[test]
1762 fn test_tolerant_policy_counts_skipped_records() {
1763 let data = b"{\"x\":1}\nnot-json\n{\"x\":2}\n";
1764 let config = JsonParserConfig::jsonl(); let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1766 assert_eq!(summary.rows_read, 2);
1767 assert_eq!(summary.malformed_lines, 1);
1768 }
1769
1770 #[test]
1771 fn test_blank_lines_are_not_counted_as_malformed() {
1772 let data = b"{\"x\":1}\n\n \n{\"x\":2}\n";
1773 for policy in [JsonErrorPolicy::Skip, JsonErrorPolicy::Strict] {
1774 let config = JsonParserConfig::jsonl().with_error_policy(policy);
1775 let summary =
1776 scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1777 assert_eq!(summary.rows_read, 2, "policy {policy:?}");
1778 assert_eq!(summary.malformed_lines, 0, "policy {policy:?}");
1779 }
1780 }
1781
1782 #[test]
1783 fn test_truncated_final_record_obeys_error_policy() {
1784 let data = b"{\"x\":1}\n{\"x\":2";
1785
1786 let skip = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Skip);
1787 let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &skip, |_| {}).unwrap();
1788 assert_eq!(summary.rows_read, 1);
1789 assert_eq!(summary.malformed_lines, 1);
1790
1791 let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1792 let err = scan_json_from_reader(Cursor::new(data.as_ref()), &strict, |_| {})
1793 .expect_err("strict mode must reject an incomplete trailing record");
1794 assert!(
1795 err.to_string()
1796 .to_lowercase()
1797 .contains("malformed json record")
1798 );
1799 }
1800
1801 #[test]
1802 fn test_all_malformed_file_fails() {
1803 let file = write_file("not-json\nalso-bad\n");
1804 let config = JsonParserConfig::jsonl();
1805 let err = analyze_json_file(file.path(), &config)
1806 .expect_err("a file with no valid records must fail");
1807 let message = err.to_string().to_lowercase();
1808 assert!(message.contains("no valid json records"), "got: {message}");
1809 }
1810
1811 #[test]
1812 fn test_tolerant_file_reports_error_count() {
1813 let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1814 let config = JsonParserConfig::jsonl();
1815 let report = analyze_json_file(file.path(), &config).unwrap();
1816 assert_eq!(report.execution.rows_processed, 2);
1817 assert_eq!(report.execution.error_count, 1);
1818 }
1819
1820 #[test]
1821 fn test_strict_file_errors_on_malformed() {
1822 let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1823 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1824 let err = analyze_json_file(file.path(), &config)
1825 .expect_err("strict mode must reject the malformed record");
1826 assert!(
1827 err.to_string()
1828 .to_lowercase()
1829 .contains("malformed json record")
1830 );
1831 }
1832
1833 #[test]
1834 fn test_analyze_json_with_large_leading_whitespace() {
1835 let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
1836 let cursor = Cursor::new(data.into_bytes());
1837 let config = JsonParserConfig::default();
1838
1839 let (_profiles, _stats, rows, malformed, format) =
1840 analyze_json_from_reader(cursor, &config).unwrap();
1841 assert_eq!(format, FileFormat::Json);
1842 assert_eq!(rows, 1);
1843 assert_eq!(malformed, 0);
1844 }
1845
1846 #[test]
1847 fn test_analyze_json_array() {
1848 let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
1849 let config = JsonParserConfig::default();
1850 let report = analyze_json_file(json.path(), &config).unwrap();
1851 let profiles = &report.column_profiles;
1852
1853 assert_eq!(profiles.len(), 2);
1854 let names: Vec<&str> = profiles
1855 .iter()
1856 .map(|profile| profile.name.as_str())
1857 .collect();
1858 assert!(names.contains(&"name"));
1859 assert!(names.contains(&"age"));
1860
1861 let age = profiles
1862 .iter()
1863 .find(|profile| profile.name == "age")
1864 .unwrap();
1865 assert_eq!(age.total_count, 2);
1866 assert_eq!(age.null_count, 0);
1867 }
1868
1869 #[test]
1870 fn test_analyze_jsonl() {
1871 let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
1872 let config = JsonParserConfig::default();
1873 let report = analyze_json_file(jsonl.path(), &config).unwrap();
1874 let profiles = &report.column_profiles;
1875
1876 assert_eq!(profiles.len(), 1);
1877 assert_eq!(profiles[0].name, "x");
1878 assert_eq!(profiles[0].total_count, 3);
1879 }
1880
1881 #[test]
1882 fn test_analyze_json_with_nulls() {
1883 let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
1884 let config = JsonParserConfig::default();
1885 let report = analyze_json_file(json.path(), &config).unwrap();
1886 let profiles = &report.column_profiles;
1887
1888 let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
1889 assert_eq!(col_a.null_count, 1);
1890
1891 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1892 assert_eq!(col_b.null_count, 1);
1893 }
1894
1895 #[test]
1896 fn test_analyze_json_with_missing_fields() {
1897 let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
1898 let config = JsonParserConfig::default();
1899 let report = analyze_json_file(json.path(), &config).unwrap();
1900 let profiles = &report.column_profiles;
1901
1902 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1903 assert_eq!(col_b.total_count, 2);
1904 }
1905
1906 const UNSORTED_FIELDS: &str = r#"{"id":1,"amount":12.5,"active":true,"date":"2026-07-23"}"#;
1909 const UNSORTED_ORDER: [&str; 4] = ["id", "amount", "active", "date"];
1910
1911 fn profile_names(report: &ProfileReport) -> Vec<&str> {
1912 report
1913 .column_profiles
1914 .iter()
1915 .map(|profile| profile.name.as_str())
1916 .collect()
1917 }
1918
1919 #[test]
1920 fn test_json_column_order_follows_source_not_alphabet() {
1921 let array = write_file(&format!("[{UNSORTED_FIELDS},{UNSORTED_FIELDS}]"));
1922 let jsonl = write_file(&format!("{UNSORTED_FIELDS}\n{UNSORTED_FIELDS}\n"));
1923
1924 for file in [&array, &jsonl] {
1925 let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1926 assert_eq!(profile_names(&report), UNSORTED_ORDER);
1927 }
1928 }
1929
1930 #[test]
1931 fn test_json_column_order_appends_later_keys_in_first_seen_order() {
1932 let data = b"{\"zulu\":1,\"mike\":2}\n{\"zulu\":3,\"mike\":4,\"delta\":5,\"alpha\":6}\n";
1935 let file = write_bytes(data);
1936 let report = analyze_json_file(file.path(), &JsonParserConfig::jsonl()).unwrap();
1937
1938 assert_eq!(profile_names(&report), ["zulu", "mike", "delta", "alpha"]);
1939 }
1940
1941 #[test]
1942 fn test_json_column_order_survives_report_serialization() {
1943 let file = write_file(&format!("[{UNSORTED_FIELDS}]"));
1944 let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1945
1946 let json = serde_json::to_string(&report).unwrap();
1947 let restored: Value = serde_json::from_str(&json).unwrap();
1948 let names: Vec<&str> = restored["column_profiles"]
1949 .as_array()
1950 .unwrap()
1951 .iter()
1952 .map(|profile| profile["name"].as_str().unwrap())
1953 .collect();
1954
1955 assert_eq!(names, UNSORTED_ORDER);
1956 }
1957
1958 #[test]
1959 fn test_analyze_json_backfills_columns_discovered_after_first_row() {
1960 let json = write_file(r#"[{"a":1},{"a":2,"b":3}]"#);
1961 let config = JsonParserConfig::default();
1962 let report = analyze_json_file(json.path(), &config).unwrap();
1963 let col_b = report
1964 .column_profiles
1965 .iter()
1966 .find(|profile| profile.name == "b")
1967 .unwrap();
1968
1969 assert_eq!(col_b.total_count, 2);
1970 assert_eq!(col_b.null_count, 1);
1971 assert_eq!(col_b.unique_count, Some(1));
1972 }
1973
1974 #[test]
1975 fn test_analyze_json_empty_array() {
1976 let json = write_file("[]");
1977 let config = JsonParserConfig::default();
1978 let report = analyze_json_file(json.path(), &config).unwrap();
1979 assert!(report.column_profiles.is_empty());
1980 assert_eq!(report.quality_score(), None);
1982 }
1983
1984 #[test]
1985 fn test_analyze_json_malformed_returns_error() {
1986 let json = write_file("this is entirely invalid json");
1987 let config = JsonParserConfig::default();
1988 let err = analyze_json_file(json.path(), &config)
1989 .expect_err("malformed JSON should return an error");
1990
1991 let message = err.to_string().to_lowercase();
1992 assert!(message.contains("malformed") && message.contains("json"));
1993 }
1994
1995 #[test]
1996 fn test_analyze_json_file_detects_format() {
1997 let json_array = write_file(r#"[{"x":1}]"#);
1998 let config = JsonParserConfig::default();
1999 let report = analyze_json_file(json_array.path(), &config).unwrap();
2000 assert!(matches!(
2001 report.data_source,
2002 DataSource::File {
2003 format: FileFormat::Json,
2004 ..
2005 }
2006 ));
2007
2008 let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
2009 let report = analyze_json_file(jsonl.path(), &config).unwrap();
2010 assert!(matches!(
2011 report.data_source,
2012 DataSource::File {
2013 format: FileFormat::Jsonl,
2014 ..
2015 }
2016 ));
2017 }
2018
2019 #[test]
2020 fn test_bom_prefixed_file_preserves_source_size() {
2021 let data = b"\xEF\xBB\xBF[{\"x\":1}]";
2022 let json = write_bytes(data);
2023 let report = analyze_json_file(json.path(), &JsonParserConfig::json_document()).unwrap();
2024
2025 assert_eq!(report.execution.rows_processed, 1);
2026 assert!(matches!(
2027 report.data_source,
2028 DataSource::File {
2029 format: FileFormat::Json,
2030 size_bytes,
2031 ..
2032 } if size_bytes == data.len() as u64
2033 ));
2034 }
2035
2036 #[test]
2037 fn test_analyze_json_file_empty() {
2038 let json = write_file("");
2039 let config = JsonParserConfig::default();
2040 let report = analyze_json_file(json.path(), &config).unwrap();
2041 assert_eq!(report.execution.rows_processed, 0);
2042 assert!(report.column_profiles.is_empty());
2043 }
2044
2045 #[test]
2046 fn test_analyze_json_boolean_and_nested() {
2047 let json =
2048 write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
2049 let config = JsonParserConfig::default();
2050 let report = analyze_json_file(json.path(), &config).unwrap();
2051 let profiles = &report.column_profiles;
2052
2053 let flag = profiles
2054 .iter()
2055 .find(|profile| profile.name == "flag")
2056 .unwrap();
2057 assert_eq!(flag.total_count, 2);
2058
2059 let nested = profiles
2060 .iter()
2061 .find(|profile| profile.name == "nested")
2062 .unwrap();
2063 assert_eq!(nested.total_count, 2);
2064 }
2065
2066 #[test]
2067 fn test_single_root_object_compact_yields_one_row() {
2068 let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
2069 let cursor = Cursor::new(data.as_ref());
2070 let config = JsonParserConfig::default();
2071
2072 let (profiles, _stats, rows, malformed, _format) =
2073 analyze_json_from_reader(cursor, &config).unwrap();
2074 assert_eq!(rows, 1);
2075 assert_eq!(malformed, 0);
2076 assert_eq!(profiles.len(), 2);
2077
2078 let names: Vec<&str> = profiles
2079 .iter()
2080 .map(|profile| profile.name.as_str())
2081 .collect();
2082 assert!(names.contains(&"type"));
2083 assert!(names.contains(&"features"));
2084 }
2085
2086 #[test]
2087 fn test_jsonl_multi_object_still_works() {
2088 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
2089 let cursor = Cursor::new(data.as_ref());
2090 let config = JsonParserConfig::default();
2091
2092 let (_profiles, _stats, rows, _malformed, format) =
2093 analyze_json_from_reader(cursor, &config).unwrap();
2094 assert_eq!(format, FileFormat::Jsonl);
2095 assert_eq!(rows, 3);
2096 }
2097
2098 #[test]
2099 fn test_single_root_object_via_analyze_json_file() {
2100 let file =
2101 write_json_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
2102 let config = JsonParserConfig::default();
2103 let report = analyze_json_file(file.path(), &config).unwrap();
2104
2105 assert_eq!(report.execution.rows_processed, 1);
2106 assert_eq!(report.column_profiles.len(), 2);
2107 }
2108
2109 #[test]
2110 fn test_single_root_object_pretty_printed_yields_one_row() {
2111 let data = br#"{
2112 "type": "FeatureCollection",
2113 "features": [
2114 {"id": 1},
2115 {"id": 2}
2116 ]
2117}"#;
2118 let cursor = Cursor::new(data.as_ref());
2119 let config = JsonParserConfig::json_document();
2122
2123 let (profiles, _stats, rows, malformed, format) =
2124 analyze_json_from_reader(cursor, &config).unwrap();
2125 assert_eq!(format, FileFormat::Json);
2126 assert_eq!(rows, 1);
2127 assert_eq!(malformed, 0);
2128 assert_eq!(profiles.len(), 2);
2129 }
2130
2131 #[test]
2132 fn test_single_root_object_pretty_printed_via_analyze_json_file() {
2133 let file = write_json_file(
2134 "{\n \"type\": \"FeatureCollection\",\n \"features\": [\n {\"id\": 1},\n {\"id\": 2}\n ]\n}\n",
2135 );
2136 let config = JsonParserConfig::default();
2137 let report = analyze_json_file(file.path(), &config).unwrap();
2138
2139 assert_eq!(report.execution.rows_processed, 1);
2140 assert_eq!(report.column_profiles.len(), 2);
2141 }
2142}