1use std::collections::{HashMap, HashSet};
30use std::io::BufRead;
31use std::path::Path;
32
33pub use dataprof_core::JsonErrorPolicy;
34use dataprof_core::{
35 AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat,
36 QualityDimension, SemanticHints, TruncationReason, Utf8BomReader,
37};
38use dataprof_runtime::{
39 ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
40};
41use serde::Deserialize;
42use serde_json::Value;
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
51pub enum JsonFormat {
52 Json,
57 Jsonl,
60}
61
62#[derive(Debug, Clone, Default)]
96pub struct JsonParserConfig {
97 pub format: Option<JsonFormat>,
104 pub max_rows: Option<usize>,
106 pub error_policy: JsonErrorPolicy,
108}
109
110impl JsonParserConfig {
111 pub fn with_max_rows(mut self, max_rows: usize) -> Self {
148 self.max_rows = Some(max_rows);
149 self
150 }
151
152 pub fn with_error_policy(mut self, policy: JsonErrorPolicy) -> Self {
180 self.error_policy = policy;
181 self
182 }
183
184 pub fn jsonl() -> Self {
186 Self {
187 format: Some(JsonFormat::Jsonl),
188 ..Default::default()
189 }
190 }
191
192 pub fn json_document() -> Self {
195 Self {
196 format: Some(JsonFormat::Json),
197 ..Default::default()
198 }
199 }
200
201 pub fn with_format(mut self, format: JsonFormat) -> Self {
203 self.format = Some(format);
204 self
205 }
206}
207
208pub type JsonObject = serde_json::Map<String, Value>;
215
216#[derive(Debug, Clone, PartialEq, Eq)]
218#[non_exhaustive]
219pub struct JsonScanSummary {
220 pub rows_read: usize,
221 pub malformed_lines: usize,
225 pub format: FileFormat,
226 pub truncated: bool,
229}
230
231pub fn scan_json_from_reader<R, F>(
316 reader: R,
317 config: &JsonParserConfig,
318 mut on_object: F,
319) -> Result<JsonScanSummary, DataProfilerError>
320where
321 R: BufRead,
322 F: FnMut(&JsonObject),
323{
324 let mut reader = Utf8BomReader::new(reader).map_err(DataProfilerError::from)?;
325 let format = match config.format {
326 Some(JsonFormat::Json) => FileFormat::Json,
327 Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
328 None => match consume_leading_whitespace(&mut reader)? {
332 Some(b'[') => FileFormat::Json,
333 _ => FileFormat::Jsonl,
334 },
335 };
336
337 let mut rows_read = 0;
338 let mut malformed_lines = 0;
339 let mut truncated = false;
340
341 match format {
342 FileFormat::Jsonl => {
348 let mut line = String::new();
349 let mut line_number = 0;
350 loop {
351 if let Some(max) = config.max_rows
352 && rows_read >= max
353 {
354 truncated = next_record_line(&mut reader, &mut line, &mut line_number)?;
358 break;
359 }
360
361 if !next_record_line(&mut reader, &mut line, &mut line_number)? {
362 break;
363 }
364
365 match read_jsonl_line(&line) {
366 JsonRecord::Object(obj) => {
367 on_object(&obj);
368 rows_read += 1;
369 }
370 JsonRecord::NonObject(kind) => {
371 if config.error_policy == JsonErrorPolicy::Strict {
372 return Err(non_object_record_error(
373 kind,
374 rows_read + malformed_lines + 1,
375 ));
376 }
377 malformed_lines += 1;
378 }
379 JsonRecord::Malformed(err) => {
380 if config.error_policy == JsonErrorPolicy::Strict {
381 return Err(malformed_jsonl_record_error(line_number, err));
382 }
383 malformed_lines += 1;
384 }
385 }
386 }
387 }
388 FileFormat::Json => {
389 let found_array = consume_leading_whitespace(&mut reader)? == Some(b'[');
393
394 if !found_array {
395 let (object_rows, object_malformed, object_truncated) =
399 scan_single_json_object(&mut reader, config, &mut on_object)?;
400 rows_read += object_rows;
401 malformed_lines += object_malformed;
402 truncated |= object_truncated;
403 } else {
404 consume_peeked(&mut reader)?;
406
407 let mut expect_value = true;
408 let mut allow_end = true;
409 let mut array_closed = false;
410 let mut drain_remainder = false;
411
412 loop {
413 let Some(next) = consume_leading_whitespace(&mut reader)? else {
414 if config.error_policy == JsonErrorPolicy::Strict {
415 return Err(malformed_array_error(
416 "unexpected end of input before closing ']'",
417 ));
418 }
419 malformed_lines += 1;
420 drain_remainder = true;
421 break;
422 };
423
424 if expect_value {
425 if next == b']' {
426 if !allow_end {
427 if config.error_policy == JsonErrorPolicy::Strict {
428 return Err(malformed_array_error(
429 "trailing comma before closing ']'",
430 ));
431 }
432 malformed_lines += 1;
433 drain_remainder = true;
434 break;
435 }
436 consume_peeked(&mut reader)?;
437 array_closed = true;
438 break;
439 }
440
441 if let Some(max) = config.max_rows
442 && rows_read >= max
443 {
444 truncated = true;
447 break;
448 }
449
450 if next == b',' {
451 if config.error_policy == JsonErrorPolicy::Strict {
452 return Err(malformed_array_error(
453 "unexpected comma where an array value was required",
454 ));
455 }
456 malformed_lines += 1;
457 drain_remainder = true;
458 break;
459 }
460
461 match read_json_record(&mut reader, next)? {
462 JsonRecord::Object(obj) => {
463 on_object(&obj);
464 rows_read += 1;
465 }
466 JsonRecord::NonObject(kind) => {
467 if config.error_policy == JsonErrorPolicy::Strict {
468 return Err(non_object_record_error(
469 kind,
470 rows_read + malformed_lines + 1,
471 ));
472 }
473 malformed_lines += 1;
477 }
478 JsonRecord::Malformed(err) => {
479 if config.error_policy == JsonErrorPolicy::Strict {
480 return Err(malformed_record_error(err));
481 }
482 malformed_lines += 1;
483 drain_remainder = true;
484 break;
485 }
486 }
487 expect_value = false;
488 } else {
489 match next {
490 b',' => {
491 consume_peeked(&mut reader)?;
492 expect_value = true;
493 allow_end = false;
494 }
495 b']' => {
496 consume_peeked(&mut reader)?;
497 array_closed = true;
498 break;
499 }
500 _ => {
501 if config.error_policy == JsonErrorPolicy::Strict {
502 return Err(malformed_array_error(
503 "expected ',' or ']' after an array value",
504 ));
505 }
506 malformed_lines += 1;
507 drain_remainder = true;
508 break;
509 }
510 }
511 }
512 }
513
514 if drain_remainder {
515 drain_to_end(&mut reader)?;
516 } else if array_closed && consume_leading_whitespace(&mut reader)?.is_some() {
517 if config.error_policy == JsonErrorPolicy::Strict {
518 return Err(malformed_array_error(
519 "non-whitespace content follows the closing ']'",
520 ));
521 }
522 malformed_lines += 1;
523 drain_to_end(&mut reader)?;
524 }
525 }
526 }
527 _ => unreachable!("json scanner only returns json or jsonl formats"),
528 }
529
530 Ok(JsonScanSummary {
531 rows_read,
532 malformed_lines,
533 format,
534 truncated,
535 })
536}
537
538fn next_record_line<R: BufRead>(
545 reader: &mut R,
546 line: &mut String,
547 line_number: &mut usize,
548) -> Result<bool, DataProfilerError> {
549 loop {
550 line.clear();
551 let read = reader.read_line(line).map_err(DataProfilerError::from)?;
552 if read == 0 {
553 return Ok(false);
554 }
555 *line_number += 1;
556 if !line.trim().is_empty() {
557 return Ok(true);
558 }
559 }
560}
561
562fn read_jsonl_line(line: &str) -> JsonRecord {
569 match serde_json::from_str::<Value>(line.trim()) {
570 Ok(Value::Object(obj)) => JsonRecord::Object(obj),
571 Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
572 Err(err) => JsonRecord::Malformed(err),
573 }
574}
575
576fn scan_single_json_object<R, F>(
583 reader: &mut R,
584 config: &JsonParserConfig,
585 on_object: &mut F,
586) -> Result<(usize, usize, bool), DataProfilerError>
587where
588 R: BufRead,
589 F: FnMut(&JsonObject),
590{
591 let mut text = String::new();
592 reader
593 .read_to_string(&mut text)
594 .map_err(DataProfilerError::from)?;
595
596 if text.trim().is_empty() {
597 return Ok((0, 0, false));
598 }
599
600 if config.max_rows == Some(0) {
605 return Ok((0, 0, true));
606 }
607
608 match serde_json::from_str::<Value>(text.trim()) {
609 Ok(Value::Object(obj)) => {
610 on_object(&obj);
611 Ok((1, 0, false))
612 }
613 Ok(value) => {
614 if config.error_policy == JsonErrorPolicy::Strict {
615 return Err(non_object_record_error(json_value_kind(&value), 1));
616 }
617 Ok((0, 1, false))
618 }
619 Err(err) => {
620 if config.error_policy == JsonErrorPolicy::Strict {
621 return Err(json_document_error(err));
622 }
623 Ok((0, 1, false))
624 }
625 }
626}
627
628enum JsonRecord {
630 Object(JsonObject),
632 NonObject(&'static str),
635 Malformed(serde_json::Error),
637}
638
639fn read_json_record<R: BufRead>(
644 reader: &mut R,
645 first: u8,
646) -> Result<JsonRecord, DataProfilerError> {
647 if first == b'-' || first.is_ascii_digit() {
652 let token = read_number_token(reader)?;
653 return Ok(match serde_json::from_str::<Value>(&token) {
654 Ok(_) => JsonRecord::NonObject("number"),
655 Err(err) => JsonRecord::Malformed(err),
656 });
657 }
658
659 let mut deserializer = serde_json::Deserializer::from_reader(reader);
660 Ok(match Value::deserialize(&mut deserializer) {
661 Ok(Value::Object(obj)) => JsonRecord::Object(obj),
662 Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
663 Err(err) => JsonRecord::Malformed(err),
664 })
665}
666
667fn read_number_token<R: BufRead>(reader: &mut R) -> Result<String, DataProfilerError> {
675 skip_ascii_whitespace(reader)?;
676 let mut token = String::new();
677 loop {
678 let (consume, token_ended) = {
679 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
680 if buf.is_empty() {
681 break;
682 }
683 let consume = buf
684 .iter()
685 .take_while(|byte| is_json_number_byte(**byte))
686 .count();
687 token.push_str(
689 std::str::from_utf8(&buf[..consume]).expect("JSON number bytes are ASCII"),
690 );
691 (consume, consume < buf.len())
692 };
693 reader.consume(consume);
694 if token_ended {
695 break;
696 }
697 }
698 Ok(token)
699}
700
701fn is_json_number_byte(byte: u8) -> bool {
702 byte.is_ascii_digit() || matches!(byte, b'-' | b'+' | b'.' | b'e' | b'E')
703}
704
705fn consume_peeked<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
714 skip_ascii_whitespace(reader)?;
715 reader.consume(1);
716 Ok(())
717}
718
719fn skip_ascii_whitespace<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
721 loop {
722 let (consume, done) = {
723 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
724 if buf.is_empty() {
725 return Ok(());
726 }
727 let consume = buf
728 .iter()
729 .take_while(|byte| byte.is_ascii_whitespace())
730 .count();
731 (consume, consume < buf.len())
732 };
733 reader.consume(consume);
734 if done {
735 return Ok(());
736 }
737 }
738}
739
740fn json_value_kind(value: &Value) -> &'static str {
741 match value {
742 Value::Null => "null",
743 Value::Bool(_) => "boolean",
744 Value::Number(_) => "number",
745 Value::String(_) => "string",
746 Value::Array(_) => "array",
747 Value::Object(_) => "object",
748 }
749}
750
751fn non_object_record_error(kind: &str, position: usize) -> DataProfilerError {
755 DataProfilerError::json_parsing_error(&format!(
758 "non-object JSON record at position {position}: \
759 expected an object with fields to profile, found {kind}"
760 ))
761}
762
763fn malformed_record_error(err: serde_json::Error) -> DataProfilerError {
767 let message = format!("malformed JSON record: {err}");
768 DataProfilerError::json_parsing_with_source(message, err)
769}
770
771fn malformed_jsonl_record_error(line: usize, err: serde_json::Error) -> DataProfilerError {
779 let message = format!(
780 "malformed JSON record on line {line}, column {}: a JSONL record must be one complete JSON value on one line",
781 err.column()
782 );
783 DataProfilerError::json_parsing_with_source(message, err)
784}
785
786fn malformed_array_error(message: &str) -> DataProfilerError {
787 DataProfilerError::json_parsing_error(&format!("malformed JSON array: {message}"))
790}
791
792fn json_document_error(err: serde_json::Error) -> DataProfilerError {
799 let message = format!(
800 "malformed JSON document: {err}. A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
804 );
805 DataProfilerError::json_parsing_with_source(message, err)
806}
807
808fn drain_to_end<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
809 loop {
810 let available = reader.fill_buf().map_err(DataProfilerError::from)?.len();
811 if available == 0 {
812 return Ok(());
813 }
814 reader.consume(available);
815 }
816}
817
818fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
819 loop {
820 let mut bytes_to_consume = 0;
821 let first_non_whitespace = {
822 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
823 if buf.is_empty() {
824 return Ok(None);
825 }
826
827 let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
828 if first_non_whitespace.is_none() {
829 bytes_to_consume = buf.len();
830 }
831 first_non_whitespace
832 };
833
834 if first_non_whitespace.is_some() {
835 return Ok(first_non_whitespace);
836 }
837
838 reader.consume(bytes_to_consume);
839 }
840}
841
842fn json_value_to_string(value: &Value) -> String {
844 match value {
845 Value::Null => String::new(),
846 Value::Bool(boolean) => boolean.to_string(),
847 Value::Number(number) => number.to_string(),
848 Value::String(string) => string.to_string(),
849 Value::Array(_) | Value::Object(_) => {
852 serde_json::to_string(value).expect("re-serializing a parsed JSON value cannot fail")
853 }
854 }
855}
856
857fn feed_json_object(
863 obj: &JsonObject,
864 prior_rows: usize,
865 known_columns: &mut Vec<String>,
866 known_columns_set: &mut HashSet<String>,
867 column_stats: &mut StreamingColumnCollection,
868) {
869 for key in obj.keys() {
870 if known_columns_set.insert(key.clone()) {
871 known_columns.push(key.clone());
872 column_stats.init_column_with_missing(key, prior_rows);
873 }
874 }
875
876 let values: Vec<String> = known_columns
877 .iter()
878 .map(|column| {
879 obj.get(column)
882 .map(json_value_to_string)
883 .unwrap_or_default()
884 })
885 .collect();
886
887 column_stats.process_record(known_columns, values);
888}
889
890pub fn analyze_json_from_reader<R: BufRead>(
932 reader: R,
933 config: &JsonParserConfig,
934) -> Result<
935 (
936 Vec<ColumnProfile>,
937 StreamingColumnCollection,
938 usize,
939 usize,
940 FileFormat,
941 ),
942 DataProfilerError,
943> {
944 analyze_json_from_reader_with_hints(reader, config, &SemanticHints::default())
945}
946
947pub fn analyze_json_from_reader_with_hints<R: BufRead>(
948 reader: R,
949 config: &JsonParserConfig,
950 semantic_hints: &SemanticHints,
951) -> Result<
952 (
953 Vec<ColumnProfile>,
954 StreamingColumnCollection,
955 usize,
956 usize,
957 FileFormat,
958 ),
959 DataProfilerError,
960> {
961 let options = AnalysisOptions::default().with_semantic_hints(semantic_hints.clone());
962 let (profiles, stats, rows_read, malformed_lines, format, _truncated) =
963 analyze_json_from_reader_full(reader, config, &options)?;
964 Ok((profiles, stats, rows_read, malformed_lines, format))
965}
966
967#[allow(clippy::type_complexity)]
970fn analyze_json_from_reader_full<R: BufRead>(
971 reader: R,
972 config: &JsonParserConfig,
973 options: &AnalysisOptions,
974) -> Result<
975 (
976 Vec<ColumnProfile>,
977 StreamingColumnCollection,
978 usize,
979 usize,
980 FileFormat,
981 bool,
982 ),
983 DataProfilerError,
984> {
985 let semantic_hints = options.semantic_hints();
986 let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
987 let mut known_columns = Vec::new();
988 let mut known_columns_set = HashSet::new();
989 let mut rows_seen = 0;
990
991 let summary = scan_json_from_reader(reader, config, |obj| {
992 feed_json_object(
993 obj,
994 rows_seen,
995 &mut known_columns,
996 &mut known_columns_set,
997 &mut column_stats,
998 );
999 rows_seen += 1;
1000 })?;
1001
1002 let profiles = profile_builder::profiles_from_streaming_with_hints(
1003 &column_stats,
1004 !options.include_statistics(),
1005 !options.include_patterns(),
1006 options.locale(),
1007 semantic_hints,
1008 );
1009
1010 Ok((
1011 profiles,
1012 column_stats,
1013 summary.rows_read,
1014 summary.malformed_lines,
1015 summary.format,
1016 summary.truncated,
1017 ))
1018}
1019
1020pub fn analyze_json_file(
1048 file_path: &Path,
1049 config: &JsonParserConfig,
1050) -> Result<ProfileReport, DataProfilerError> {
1051 analyze_json_file_with_dimensions(file_path, config, None)
1052}
1053
1054pub fn analyze_json_file_with_dimensions(
1056 file_path: &Path,
1057 config: &JsonParserConfig,
1058 quality_dimensions: Option<&[QualityDimension]>,
1059) -> Result<ProfileReport, DataProfilerError> {
1060 analyze_json_file_with_dimensions_and_hints(
1061 file_path,
1062 config,
1063 quality_dimensions,
1064 &SemanticHints::default(),
1065 )
1066}
1067
1068pub fn analyze_json_file_with_dimensions_and_hints(
1069 file_path: &Path,
1070 config: &JsonParserConfig,
1071 quality_dimensions: Option<&[QualityDimension]>,
1072 semantic_hints: &SemanticHints,
1073) -> Result<ProfileReport, DataProfilerError> {
1074 let options = AnalysisOptions::default()
1075 .with_quality_dimensions(quality_dimensions.map(<[_]>::to_vec))
1076 .with_semantic_hints(semantic_hints.clone());
1077 analyze_json_file_with_options(file_path, config, &options)
1078}
1079
1080pub fn analyze_json_file_with_options(
1086 file_path: &Path,
1087 config: &JsonParserConfig,
1088 options: &AnalysisOptions,
1089) -> Result<ProfileReport, DataProfilerError> {
1090 let mut config = config.clone();
1093 if config.format.is_none() {
1094 config.format = grammar_for_extension(file_path);
1095 }
1096 let config = &config;
1097 let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
1098 let start = std::time::Instant::now();
1099
1100 let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
1101 let buf_reader = std::io::BufReader::new(file);
1102
1103 let (column_profiles, column_stats, rows_read, malformed_lines, format, truncated) =
1104 analyze_json_from_reader_full(buf_reader, config, options)?;
1105 let read_as_json_document = format == FileFormat::Json;
1106
1107 let file_source = DataSource::File {
1108 path: file_path.display().to_string(),
1109 format,
1110 size_bytes: metadata.len(),
1111 modified_at: None,
1112 parquet_metadata: None,
1113 };
1114
1115 if rows_read == 0 {
1116 if malformed_lines > 0 {
1117 let hint = if read_as_json_document {
1121 ". A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
1122 } else {
1123 ""
1124 };
1125 return Err(DataProfilerError::json_parsing_error(&format!(
1128 "No valid JSON records found in file \
1129 (every record was malformed or not a JSON object){hint}"
1130 )));
1131 }
1132 let mut execution =
1137 ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("json");
1138 if truncated && let Some(max) = config.max_rows {
1139 execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1140 }
1141 return Ok(ReportAssembler::new(file_source, execution)
1142 .columns(column_profiles)
1143 .with_quality_data(HashMap::new())
1144 .with_analysis_options(options)
1145 .build());
1146 }
1147
1148 let sample_columns = profile_builder::quality_check_samples(&column_stats);
1149 let scan_time_ms = start.elapsed().as_millis();
1150 let num_columns = column_profiles.len();
1151
1152 let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms)
1153 .with_engine("json")
1154 .with_error_count(malformed_lines);
1157 if truncated && let Some(max) = config.max_rows {
1160 execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
1161 }
1162
1163 Ok(ReportAssembler::new(file_source, execution)
1164 .columns(column_profiles)
1165 .with_quality_data(sample_columns)
1166 .with_row_duplicates(column_stats.row_duplicate_summary())
1167 .with_row_completeness(column_stats.row_completeness_summary())
1168 .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
1169 .with_analysis_options(options)
1170 .build())
1171}
1172
1173fn grammar_for_extension(file_path: &Path) -> Option<JsonFormat> {
1181 let extension = file_path.extension().and_then(|ext| ext.to_str())?;
1182 if extension.eq_ignore_ascii_case("json") {
1183 Some(JsonFormat::Json)
1184 } else if extension.eq_ignore_ascii_case("jsonl") || extension.eq_ignore_ascii_case("ndjson") {
1185 Some(JsonFormat::Jsonl)
1186 } else {
1187 None
1188 }
1189}
1190
1191fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
1192 if error.kind() == std::io::ErrorKind::NotFound {
1193 DataProfilerError::FileNotFound {
1194 path: file_path.display().to_string(),
1195 }
1196 } else {
1197 DataProfilerError::from(error)
1198 }
1199}
1200
1201#[cfg(test)]
1202mod tests {
1203 use super::*;
1204 use std::io::{Cursor, Write};
1205 use tempfile::NamedTempFile;
1206
1207 fn write_file(content: &str) -> NamedTempFile {
1210 let mut file = NamedTempFile::new().unwrap();
1211 write!(file, "{}", content).unwrap();
1212 file.flush().unwrap();
1213 file
1214 }
1215
1216 fn write_json_file(content: &str) -> NamedTempFile {
1219 let mut file = NamedTempFile::with_suffix(".json").unwrap();
1220 write!(file, "{}", content).unwrap();
1221 file.flush().unwrap();
1222 file
1223 }
1224
1225 fn write_bytes(content: &[u8]) -> NamedTempFile {
1226 let mut file = NamedTempFile::new().unwrap();
1227 file.write_all(content).unwrap();
1228 file.flush().unwrap();
1229 file
1230 }
1231
1232 #[test]
1233 fn test_scan_jsonl_detects_format_and_rows() {
1234 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1235 let mut rows = 0;
1236
1237 let summary = scan_json_from_reader(
1238 Cursor::new(data.as_ref()),
1239 &JsonParserConfig::default(),
1240 |_| {
1241 rows += 1;
1242 },
1243 )
1244 .unwrap();
1245
1246 assert_eq!(summary.format, FileFormat::Jsonl);
1247 assert_eq!(summary.rows_read, 3);
1248 assert_eq!(summary.malformed_lines, 0);
1249 assert_eq!(rows, 3);
1250 }
1251
1252 #[test]
1253 fn test_scan_json_array_detects_format_and_rows() {
1254 let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
1255 let mut keys = Vec::new();
1256
1257 let summary = scan_json_from_reader(
1258 Cursor::new(data.as_ref()),
1259 &JsonParserConfig::default(),
1260 |obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
1261 )
1262 .unwrap();
1263
1264 assert_eq!(summary.format, FileFormat::Json);
1265 assert_eq!(summary.rows_read, 2);
1266 assert_eq!(keys.len(), 2);
1267 assert_eq!(keys[0], vec!["name".to_string()]);
1268 }
1269
1270 #[test]
1271 fn test_scan_accepts_one_leading_utf8_bom_for_json_and_jsonl() {
1272 for (data, config, expected_format) in [
1273 (
1274 b"\xEF\xBB\xBF[{\"x\":1},{\"x\":2}]".as_slice(),
1275 JsonParserConfig::default(),
1276 FileFormat::Json,
1277 ),
1278 (
1279 b"\xEF\xBB\xBF{\"x\":1}\n{\"x\":2}\n".as_slice(),
1280 JsonParserConfig::default(),
1281 FileFormat::Jsonl,
1282 ),
1283 ] {
1284 let mut rows = 0;
1285 let summary = scan_json_from_reader(Cursor::new(data), &config, |_| rows += 1)
1286 .expect("leading UTF-8 BOM should be accepted");
1287
1288 assert_eq!(rows, 2);
1289 assert_eq!(summary.rows_read, 2);
1290 assert_eq!(summary.malformed_lines, 0);
1291 assert_eq!(summary.format, expected_format);
1292 }
1293 }
1294
1295 #[test]
1296 fn test_scan_does_not_strip_nonleading_or_second_utf8_bom() {
1297 let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1298
1299 for data in [
1300 b" \xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1301 b"\xEF\xBB\xBF\xEF\xBB\xBF{\"x\":1}\n".as_slice(),
1302 b"{\"x\":1}\n\xEF\xBB\xBF{\"x\":2}\n".as_slice(),
1303 ] {
1304 scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1305 .expect_err("only the BOM at absolute byte offset zero may be stripped");
1306 }
1307 }
1308
1309 #[test]
1310 fn test_bom_only_input_matches_empty_input_policy() {
1311 for config in [JsonParserConfig::default(), JsonParserConfig::jsonl()] {
1312 let empty = scan_json_from_reader(Cursor::new(b""), &config, |_| {}).unwrap();
1313 let bom =
1314 scan_json_from_reader(Cursor::new(b"\xEF\xBB\xBF".as_slice()), &config, |_| {})
1315 .unwrap();
1316 assert_eq!(bom, empty);
1317 }
1318 }
1319
1320 #[test]
1321 fn test_json_array_container_grammar_obeys_error_policy() {
1322 let cases = [
1323 (br#"[{"x":1}"#.as_ref(), 1, "missing closing bracket"),
1324 (br#"[{"x":1},"#.as_ref(), 1, "comma followed by EOF"),
1325 (br#"[{"x":1} {"x":2}]"#.as_ref(), 1, "missing comma"),
1326 (br#"[{"x":1},,{"x":2}]"#.as_ref(), 1, "doubled comma"),
1327 (br#"[,{"x":1}]"#.as_ref(), 0, "leading comma"),
1328 (br#"[{"x":1},]"#.as_ref(), 1, "trailing comma"),
1329 (br#"[{"x":1}] trailing"#.as_ref(), 1, "trailing content"),
1330 (
1331 br#"[{"x":1}][{"x":2}]"#.as_ref(),
1332 1,
1333 "second top-level value",
1334 ),
1335 ];
1336
1337 for (data, expected_rows, case) in cases {
1338 let skip = JsonParserConfig::json_document();
1339 let summary =
1340 scan_json_from_reader(Cursor::new(data), &skip, |_| {}).expect("tolerant scan");
1341 assert_eq!(summary.rows_read, expected_rows, "{case}");
1342 assert_eq!(summary.malformed_lines, 1, "{case}");
1343
1344 let strict =
1345 JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict);
1346 let err = scan_json_from_reader(Cursor::new(data), &strict, |_| {})
1347 .expect_err("strict mode must reject invalid array grammar");
1348 assert!(
1349 err.to_string()
1350 .to_lowercase()
1351 .contains("malformed json array"),
1352 "{case}: {err}"
1353 );
1354 }
1355 }
1356
1357 #[test]
1358 fn test_json_array_max_rows_does_not_validate_unread_values() {
1359 let data = br#"[{"x":1},not-valid-json"#;
1360 let config = JsonParserConfig::json_document().with_max_rows(1);
1361 let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1362
1363 assert_eq!(summary.rows_read, 1);
1364 assert_eq!(summary.malformed_lines, 0);
1365 assert!(summary.truncated);
1366 }
1367
1368 #[test]
1369 fn test_scan_respects_max_rows() {
1370 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
1371 let config = JsonParserConfig::default().with_max_rows(2);
1372 let mut rows = 0;
1373
1374 let summary =
1375 scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
1376
1377 assert_eq!(summary.rows_read, 2);
1378 assert_eq!(rows, 2);
1379 }
1380
1381 #[test]
1382 fn test_scan_skips_malformed_jsonl_lines() {
1383 let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
1384 let mut rows = 0;
1385
1386 let summary = scan_json_from_reader(
1387 Cursor::new(data.as_ref()),
1388 &JsonParserConfig::jsonl(),
1389 |_| {
1390 rows += 1;
1391 },
1392 )
1393 .unwrap();
1394
1395 assert_eq!(summary.format, FileFormat::Jsonl);
1396 assert_eq!(summary.rows_read, 2);
1397 assert_eq!(summary.malformed_lines, 1);
1398 assert_eq!(rows, 2);
1399 }
1400
1401 #[test]
1407 fn test_pretty_printed_root_object_is_one_row_under_the_json_grammar() {
1408 let data = br#"{
1409 "type": "FeatureCollection",
1410 "features": [
1411 {"id": 1},
1412 {"id": 2}
1413 ]
1414}"#;
1415 let mut object_field_counts = Vec::new();
1416
1417 let summary = scan_json_from_reader(
1418 Cursor::new(data.as_ref()),
1419 &JsonParserConfig::json_document(),
1420 |obj| object_field_counts.push(obj.len()),
1421 )
1422 .unwrap();
1423
1424 assert_eq!(summary.format, FileFormat::Json);
1425 assert_eq!(summary.rows_read, 1);
1426 assert_eq!(summary.malformed_lines, 0);
1427 assert_eq!(object_field_counts, vec![2]);
1428 }
1429
1430 #[test]
1431 fn test_pretty_printed_root_object_is_not_valid_jsonl() {
1432 let data = br#"{
1433 "type": "FeatureCollection"
1434}"#;
1435 let err = scan_json_from_reader(
1436 Cursor::new(data.as_ref()),
1437 &JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict),
1438 |_| {},
1439 )
1440 .expect_err("a record spanning lines is not JSONL");
1441
1442 assert!(
1443 err.to_string().contains("malformed JSON record"),
1444 "{err}, expected the malformed category"
1445 );
1446 }
1447
1448 const NON_OBJECT_VALUES: &[(&str, &str)] = &[
1452 ("null", "null"),
1453 ("boolean", "true"),
1454 ("number", "42"),
1455 ("number", "-1.5e3"),
1456 ("string", r#""text""#),
1457 ("array", "[1, 2]"),
1458 ];
1459
1460 #[test]
1461 fn test_jsonl_counts_non_object_records_and_keeps_scanning() {
1462 for (kind, value) in NON_OBJECT_VALUES {
1463 let data = format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n");
1464 let mut rows = 0;
1465
1466 let summary = scan_json_from_reader(
1467 Cursor::new(data.as_bytes()),
1468 &JsonParserConfig::jsonl(),
1469 |_| rows += 1,
1470 )
1471 .unwrap();
1472
1473 assert_eq!(summary.rows_read, 2, "{kind}: record after it was lost");
1474 assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1475 assert_eq!(rows, 2, "{kind}");
1476 }
1477 }
1478
1479 #[test]
1480 fn test_json_array_counts_non_object_elements_and_keeps_scanning() {
1481 for (kind, value) in NON_OBJECT_VALUES {
1482 let data = format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]");
1483 let mut rows = 0;
1484
1485 let summary = scan_json_from_reader(
1486 Cursor::new(data.as_bytes()),
1487 &JsonParserConfig::json_document(),
1488 |_| rows += 1,
1489 )
1490 .unwrap();
1491
1492 assert_eq!(summary.rows_read, 2, "{kind}: element after it was lost");
1493 assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
1494 assert_eq!(rows, 2, "{kind}");
1495 }
1496 }
1497
1498 #[test]
1499 fn test_strict_policy_rejects_the_first_non_object_record() {
1500 for (kind, value) in NON_OBJECT_VALUES {
1501 for (config, data) in [
1502 (
1503 JsonParserConfig::jsonl(),
1504 format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n"),
1505 ),
1506 (
1507 JsonParserConfig::json_document(),
1508 format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]"),
1509 ),
1510 ] {
1511 let config = config.with_error_policy(JsonErrorPolicy::Strict);
1512 let err = scan_json_from_reader(Cursor::new(data.as_bytes()), &config, |_| {})
1513 .expect_err("{kind}: strict mode must reject a non-object record");
1514
1515 let message = err.to_string();
1516 assert!(message.contains("non-object JSON record"), "{message}");
1517 assert!(message.contains("at position 2"), "{message}");
1518 assert!(message.contains(&format!("found {kind}")), "{message}");
1519 }
1520 }
1521 }
1522
1523 #[test]
1524 fn test_number_element_does_not_swallow_the_array_delimiter() {
1525 let data = br#"[{"id":1}, 1, 2, 3, {"id":2}, 4, {"id":3}]"#;
1528 let mut rows = 0;
1529
1530 let summary = scan_json_from_reader(
1531 Cursor::new(data.as_ref()),
1532 &JsonParserConfig::json_document(),
1533 |_| rows += 1,
1534 )
1535 .unwrap();
1536
1537 assert_eq!(summary.rows_read, 3);
1538 assert_eq!(summary.malformed_lines, 4);
1539 assert_eq!(rows, 3);
1540 }
1541
1542 #[test]
1543 fn test_number_shaped_garbage_stays_a_malformed_record() {
1544 let data = b"{\"id\":1}\n1.2.3\n";
1547 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1548 let err = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {})
1549 .expect_err("number-shaped garbage should be malformed");
1550
1551 assert!(
1552 err.to_string().contains("malformed JSON record"),
1553 "{err}, expected the malformed category"
1554 );
1555 }
1556
1557 #[test]
1558 fn test_input_of_only_non_object_records_fails() {
1559 let file = write_file("\"just a string\"\n1\n[2]\n");
1560 let err = analyze_json_file(file.path(), &JsonParserConfig::jsonl())
1561 .expect_err("nothing profileable should not return a clean empty profile");
1562
1563 assert!(
1564 err.to_string().contains("No valid JSON records found"),
1565 "{err}"
1566 );
1567 }
1568
1569 #[test]
1570 fn test_json_grammar_reads_a_bare_object_as_one_row() {
1573 let data = br#"{"x":1}"#;
1574 let mut rows = 0;
1575 let summary = scan_json_from_reader(
1576 Cursor::new(data.as_ref()),
1577 &JsonParserConfig::json_document(),
1578 |_| rows += 1,
1579 )
1580 .unwrap();
1581
1582 assert_eq!(summary.rows_read, 1);
1583 assert_eq!(summary.malformed_lines, 0);
1584 assert_eq!(rows, 1);
1585 }
1586
1587 #[test]
1588 fn test_json_grammar_rejects_concatenated_objects() {
1592 let data = br#"{"x":1}{"x":2}"#;
1593 let err = scan_json_from_reader(
1594 Cursor::new(data.as_ref()),
1595 &JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict),
1596 |_| {},
1597 )
1598 .expect_err("two objects are not one JSON document");
1599
1600 let message = err.to_string();
1601 assert!(message.contains("malformed JSON document"), "{message}");
1602 assert!(message.contains("format=\"jsonl\""), "{message}");
1603 }
1604
1605 #[test]
1606 fn test_analyze_json_from_reader_jsonl_streaming() {
1607 let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
1608 let cursor = Cursor::new(data.as_ref());
1609 let config = JsonParserConfig::default();
1610
1611 let (profiles, _stats, rows, _malformed, format) =
1612 analyze_json_from_reader(cursor, &config).unwrap();
1613 assert_eq!(format, FileFormat::Jsonl);
1614 assert_eq!(rows, 3);
1615 assert_eq!(profiles.len(), 2);
1616 }
1617
1618 #[test]
1619 fn test_analyze_json_from_reader_json_array() {
1620 let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
1621 let cursor = Cursor::new(data.as_ref());
1622 let config = JsonParserConfig::default();
1623
1624 let (profiles, _stats, rows, _malformed, format) =
1625 analyze_json_from_reader(cursor, &config).unwrap();
1626 assert_eq!(format, FileFormat::Json);
1627 assert_eq!(rows, 2);
1628 assert_eq!(profiles.len(), 2);
1629 }
1630
1631 #[test]
1632 fn test_analyze_json_from_reader_max_rows() {
1633 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
1634 let cursor = Cursor::new(data.as_ref());
1635 let config = JsonParserConfig::default().with_max_rows(3);
1636
1637 let (_profiles, _stats, rows, _malformed, _format) =
1638 analyze_json_from_reader(cursor, &config).unwrap();
1639 assert_eq!(rows, 3);
1640 }
1641
1642 #[test]
1643 fn test_analyze_json_from_reader_missing_fields() {
1644 let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
1645 let cursor = Cursor::new(data.as_ref());
1646 let config = JsonParserConfig::jsonl();
1647
1648 let (profiles, _stats, rows, _malformed, _format) =
1649 analyze_json_from_reader(cursor, &config).unwrap();
1650 assert_eq!(rows, 2);
1651
1652 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1653 assert_eq!(col_b.total_count, 2);
1654 }
1655
1656 #[test]
1657 fn test_analyze_json_file_quality_report() {
1658 let file = write_file(r#"[{"x":1},{"x":2}]"#);
1659 let config = JsonParserConfig::default();
1660 let report = analyze_json_file(file.path(), &config).unwrap();
1661
1662 assert_eq!(report.execution.rows_processed, 2);
1663 assert_eq!(report.column_profiles.len(), 1);
1664 assert!(report.quality_score().unwrap() >= 0.0);
1665 }
1666
1667 #[test]
1668 fn test_jsonl_skips_malformed_lines() {
1669 let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
1670 let cursor = Cursor::new(data.as_ref());
1671 let config = JsonParserConfig::jsonl();
1672
1673 let (profiles, _stats, rows, _malformed, format) =
1674 analyze_json_from_reader(cursor, &config).unwrap();
1675 assert_eq!(format, FileFormat::Jsonl);
1676 assert_eq!(rows, 2);
1677 assert_eq!(profiles[0].total_count, 2);
1678 }
1679
1680 #[test]
1681 fn test_strict_policy_errors_on_malformed_jsonl() {
1682 for data in [
1684 b"not-json\n{\"x\":1}\n{\"x\":2}\n".as_ref(),
1685 b"{\"x\":1}\nnot-json\n{\"x\":2}\n".as_ref(),
1686 b"{\"x\":1}\n{\"x\":2}\nnot-json\n".as_ref(),
1687 ] {
1688 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1689 let err = scan_json_from_reader(Cursor::new(data), &config, |_| {})
1690 .expect_err("strict mode must reject malformed records");
1691 let message = err.to_string().to_lowercase();
1692 assert!(message.contains("malformed json record"), "got: {message}");
1693 assert!(
1695 !err.to_string().contains("not-json"),
1696 "leaked record: {err}"
1697 );
1698 }
1699 }
1700
1701 #[test]
1702 fn test_tolerant_policy_counts_skipped_records() {
1703 let data = b"{\"x\":1}\nnot-json\n{\"x\":2}\n";
1704 let config = JsonParserConfig::jsonl(); let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1706 assert_eq!(summary.rows_read, 2);
1707 assert_eq!(summary.malformed_lines, 1);
1708 }
1709
1710 #[test]
1711 fn test_blank_lines_are_not_counted_as_malformed() {
1712 let data = b"{\"x\":1}\n\n \n{\"x\":2}\n";
1713 for policy in [JsonErrorPolicy::Skip, JsonErrorPolicy::Strict] {
1714 let config = JsonParserConfig::jsonl().with_error_policy(policy);
1715 let summary =
1716 scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
1717 assert_eq!(summary.rows_read, 2, "policy {policy:?}");
1718 assert_eq!(summary.malformed_lines, 0, "policy {policy:?}");
1719 }
1720 }
1721
1722 #[test]
1723 fn test_truncated_final_record_obeys_error_policy() {
1724 let data = b"{\"x\":1}\n{\"x\":2";
1725
1726 let skip = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Skip);
1727 let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &skip, |_| {}).unwrap();
1728 assert_eq!(summary.rows_read, 1);
1729 assert_eq!(summary.malformed_lines, 1);
1730
1731 let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1732 let err = scan_json_from_reader(Cursor::new(data.as_ref()), &strict, |_| {})
1733 .expect_err("strict mode must reject an incomplete trailing record");
1734 assert!(
1735 err.to_string()
1736 .to_lowercase()
1737 .contains("malformed json record")
1738 );
1739 }
1740
1741 #[test]
1742 fn test_all_malformed_file_fails() {
1743 let file = write_file("not-json\nalso-bad\n");
1744 let config = JsonParserConfig::jsonl();
1745 let err = analyze_json_file(file.path(), &config)
1746 .expect_err("a file with no valid records must fail");
1747 let message = err.to_string().to_lowercase();
1748 assert!(message.contains("no valid json records"), "got: {message}");
1749 }
1750
1751 #[test]
1752 fn test_tolerant_file_reports_error_count() {
1753 let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1754 let config = JsonParserConfig::jsonl();
1755 let report = analyze_json_file(file.path(), &config).unwrap();
1756 assert_eq!(report.execution.rows_processed, 2);
1757 assert_eq!(report.execution.error_count, 1);
1758 }
1759
1760 #[test]
1761 fn test_strict_file_errors_on_malformed() {
1762 let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
1763 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
1764 let err = analyze_json_file(file.path(), &config)
1765 .expect_err("strict mode must reject the malformed record");
1766 assert!(
1767 err.to_string()
1768 .to_lowercase()
1769 .contains("malformed json record")
1770 );
1771 }
1772
1773 #[test]
1774 fn test_analyze_json_with_large_leading_whitespace() {
1775 let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
1776 let cursor = Cursor::new(data.into_bytes());
1777 let config = JsonParserConfig::default();
1778
1779 let (_profiles, _stats, rows, malformed, format) =
1780 analyze_json_from_reader(cursor, &config).unwrap();
1781 assert_eq!(format, FileFormat::Json);
1782 assert_eq!(rows, 1);
1783 assert_eq!(malformed, 0);
1784 }
1785
1786 #[test]
1787 fn test_analyze_json_array() {
1788 let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
1789 let config = JsonParserConfig::default();
1790 let report = analyze_json_file(json.path(), &config).unwrap();
1791 let profiles = &report.column_profiles;
1792
1793 assert_eq!(profiles.len(), 2);
1794 let names: Vec<&str> = profiles
1795 .iter()
1796 .map(|profile| profile.name.as_str())
1797 .collect();
1798 assert!(names.contains(&"name"));
1799 assert!(names.contains(&"age"));
1800
1801 let age = profiles
1802 .iter()
1803 .find(|profile| profile.name == "age")
1804 .unwrap();
1805 assert_eq!(age.total_count, 2);
1806 assert_eq!(age.null_count, 0);
1807 }
1808
1809 #[test]
1810 fn test_analyze_jsonl() {
1811 let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
1812 let config = JsonParserConfig::default();
1813 let report = analyze_json_file(jsonl.path(), &config).unwrap();
1814 let profiles = &report.column_profiles;
1815
1816 assert_eq!(profiles.len(), 1);
1817 assert_eq!(profiles[0].name, "x");
1818 assert_eq!(profiles[0].total_count, 3);
1819 }
1820
1821 #[test]
1822 fn test_analyze_json_with_nulls() {
1823 let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
1824 let config = JsonParserConfig::default();
1825 let report = analyze_json_file(json.path(), &config).unwrap();
1826 let profiles = &report.column_profiles;
1827
1828 let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
1829 assert_eq!(col_a.null_count, 1);
1830
1831 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1832 assert_eq!(col_b.null_count, 1);
1833 }
1834
1835 #[test]
1836 fn test_analyze_json_with_missing_fields() {
1837 let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
1838 let config = JsonParserConfig::default();
1839 let report = analyze_json_file(json.path(), &config).unwrap();
1840 let profiles = &report.column_profiles;
1841
1842 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
1843 assert_eq!(col_b.total_count, 2);
1844 }
1845
1846 const UNSORTED_FIELDS: &str = r#"{"id":1,"amount":12.5,"active":true,"date":"2026-07-23"}"#;
1849 const UNSORTED_ORDER: [&str; 4] = ["id", "amount", "active", "date"];
1850
1851 fn profile_names(report: &ProfileReport) -> Vec<&str> {
1852 report
1853 .column_profiles
1854 .iter()
1855 .map(|profile| profile.name.as_str())
1856 .collect()
1857 }
1858
1859 #[test]
1860 fn test_json_column_order_follows_source_not_alphabet() {
1861 let array = write_file(&format!("[{UNSORTED_FIELDS},{UNSORTED_FIELDS}]"));
1862 let jsonl = write_file(&format!("{UNSORTED_FIELDS}\n{UNSORTED_FIELDS}\n"));
1863
1864 for file in [&array, &jsonl] {
1865 let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1866 assert_eq!(profile_names(&report), UNSORTED_ORDER);
1867 }
1868 }
1869
1870 #[test]
1871 fn test_json_column_order_appends_later_keys_in_first_seen_order() {
1872 let data = b"{\"zulu\":1,\"mike\":2}\n{\"zulu\":3,\"mike\":4,\"delta\":5,\"alpha\":6}\n";
1875 let file = write_bytes(data);
1876 let report = analyze_json_file(file.path(), &JsonParserConfig::jsonl()).unwrap();
1877
1878 assert_eq!(profile_names(&report), ["zulu", "mike", "delta", "alpha"]);
1879 }
1880
1881 #[test]
1882 fn test_json_column_order_survives_report_serialization() {
1883 let file = write_file(&format!("[{UNSORTED_FIELDS}]"));
1884 let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
1885
1886 let json = serde_json::to_string(&report).unwrap();
1887 let restored: Value = serde_json::from_str(&json).unwrap();
1888 let names: Vec<&str> = restored["column_profiles"]
1889 .as_array()
1890 .unwrap()
1891 .iter()
1892 .map(|profile| profile["name"].as_str().unwrap())
1893 .collect();
1894
1895 assert_eq!(names, UNSORTED_ORDER);
1896 }
1897
1898 #[test]
1899 fn test_analyze_json_backfills_columns_discovered_after_first_row() {
1900 let json = write_file(r#"[{"a":1},{"a":2,"b":3}]"#);
1901 let config = JsonParserConfig::default();
1902 let report = analyze_json_file(json.path(), &config).unwrap();
1903 let col_b = report
1904 .column_profiles
1905 .iter()
1906 .find(|profile| profile.name == "b")
1907 .unwrap();
1908
1909 assert_eq!(col_b.total_count, 2);
1910 assert_eq!(col_b.null_count, 1);
1911 assert_eq!(col_b.unique_count, Some(1));
1912 }
1913
1914 #[test]
1915 fn test_analyze_json_empty_array() {
1916 let json = write_file("[]");
1917 let config = JsonParserConfig::default();
1918 let report = analyze_json_file(json.path(), &config).unwrap();
1919 assert!(report.column_profiles.is_empty());
1920 assert_eq!(report.quality_score(), None);
1922 }
1923
1924 #[test]
1925 fn test_analyze_json_malformed_returns_error() {
1926 let json = write_file("this is entirely invalid json");
1927 let config = JsonParserConfig::default();
1928 let err = analyze_json_file(json.path(), &config)
1929 .expect_err("malformed JSON should return an error");
1930
1931 let message = err.to_string().to_lowercase();
1932 assert!(message.contains("malformed") && message.contains("json"));
1933 }
1934
1935 #[test]
1936 fn test_analyze_json_file_detects_format() {
1937 let json_array = write_file(r#"[{"x":1}]"#);
1938 let config = JsonParserConfig::default();
1939 let report = analyze_json_file(json_array.path(), &config).unwrap();
1940 assert!(matches!(
1941 report.data_source,
1942 DataSource::File {
1943 format: FileFormat::Json,
1944 ..
1945 }
1946 ));
1947
1948 let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
1949 let report = analyze_json_file(jsonl.path(), &config).unwrap();
1950 assert!(matches!(
1951 report.data_source,
1952 DataSource::File {
1953 format: FileFormat::Jsonl,
1954 ..
1955 }
1956 ));
1957 }
1958
1959 #[test]
1960 fn test_bom_prefixed_file_preserves_source_size() {
1961 let data = b"\xEF\xBB\xBF[{\"x\":1}]";
1962 let json = write_bytes(data);
1963 let report = analyze_json_file(json.path(), &JsonParserConfig::json_document()).unwrap();
1964
1965 assert_eq!(report.execution.rows_processed, 1);
1966 assert!(matches!(
1967 report.data_source,
1968 DataSource::File {
1969 format: FileFormat::Json,
1970 size_bytes,
1971 ..
1972 } if size_bytes == data.len() as u64
1973 ));
1974 }
1975
1976 #[test]
1977 fn test_analyze_json_file_empty() {
1978 let json = write_file("");
1979 let config = JsonParserConfig::default();
1980 let report = analyze_json_file(json.path(), &config).unwrap();
1981 assert_eq!(report.execution.rows_processed, 0);
1982 assert!(report.column_profiles.is_empty());
1983 }
1984
1985 #[test]
1986 fn test_analyze_json_boolean_and_nested() {
1987 let json =
1988 write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
1989 let config = JsonParserConfig::default();
1990 let report = analyze_json_file(json.path(), &config).unwrap();
1991 let profiles = &report.column_profiles;
1992
1993 let flag = profiles
1994 .iter()
1995 .find(|profile| profile.name == "flag")
1996 .unwrap();
1997 assert_eq!(flag.total_count, 2);
1998
1999 let nested = profiles
2000 .iter()
2001 .find(|profile| profile.name == "nested")
2002 .unwrap();
2003 assert_eq!(nested.total_count, 2);
2004 }
2005
2006 #[test]
2007 fn test_single_root_object_compact_yields_one_row() {
2008 let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
2009 let cursor = Cursor::new(data.as_ref());
2010 let config = JsonParserConfig::default();
2011
2012 let (profiles, _stats, rows, malformed, _format) =
2013 analyze_json_from_reader(cursor, &config).unwrap();
2014 assert_eq!(rows, 1);
2015 assert_eq!(malformed, 0);
2016 assert_eq!(profiles.len(), 2);
2017
2018 let names: Vec<&str> = profiles
2019 .iter()
2020 .map(|profile| profile.name.as_str())
2021 .collect();
2022 assert!(names.contains(&"type"));
2023 assert!(names.contains(&"features"));
2024 }
2025
2026 #[test]
2027 fn test_jsonl_multi_object_still_works() {
2028 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
2029 let cursor = Cursor::new(data.as_ref());
2030 let config = JsonParserConfig::default();
2031
2032 let (_profiles, _stats, rows, _malformed, format) =
2033 analyze_json_from_reader(cursor, &config).unwrap();
2034 assert_eq!(format, FileFormat::Jsonl);
2035 assert_eq!(rows, 3);
2036 }
2037
2038 #[test]
2039 fn test_single_root_object_via_analyze_json_file() {
2040 let file =
2041 write_json_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
2042 let config = JsonParserConfig::default();
2043 let report = analyze_json_file(file.path(), &config).unwrap();
2044
2045 assert_eq!(report.execution.rows_processed, 1);
2046 assert_eq!(report.column_profiles.len(), 2);
2047 }
2048
2049 #[test]
2050 fn test_single_root_object_pretty_printed_yields_one_row() {
2051 let data = br#"{
2052 "type": "FeatureCollection",
2053 "features": [
2054 {"id": 1},
2055 {"id": 2}
2056 ]
2057}"#;
2058 let cursor = Cursor::new(data.as_ref());
2059 let config = JsonParserConfig::json_document();
2062
2063 let (profiles, _stats, rows, malformed, format) =
2064 analyze_json_from_reader(cursor, &config).unwrap();
2065 assert_eq!(format, FileFormat::Json);
2066 assert_eq!(rows, 1);
2067 assert_eq!(malformed, 0);
2068 assert_eq!(profiles.len(), 2);
2069 }
2070
2071 #[test]
2072 fn test_single_root_object_pretty_printed_via_analyze_json_file() {
2073 let file = write_json_file(
2074 "{\n \"type\": \"FeatureCollection\",\n \"features\": [\n {\"id\": 1},\n {\"id\": 2}\n ]\n}\n",
2075 );
2076 let config = JsonParserConfig::default();
2077 let report = analyze_json_file(file.path(), &config).unwrap();
2078
2079 assert_eq!(report.execution.rows_processed, 1);
2080 assert_eq!(report.column_profiles.len(), 2);
2081 }
2082}