1use std::collections::{HashMap, HashSet};
2use std::io::BufRead;
3use std::path::Path;
4
5pub use dataprof_core::JsonErrorPolicy;
6use dataprof_core::{
7 ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat, QualityDimension,
8 SemanticHints, TruncationReason,
9};
10use dataprof_runtime::{
11 ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
12};
13use serde::Deserialize;
14use serde_json::Value;
15
16#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub enum JsonFormat {
19 JsonArray,
21 Jsonl,
23}
24
25#[derive(Debug, Clone, Default)]
27pub struct JsonParserConfig {
28 pub format: Option<JsonFormat>,
30 pub max_rows: Option<usize>,
32 pub error_policy: JsonErrorPolicy,
34}
35
36impl JsonParserConfig {
37 pub fn with_max_rows(mut self, max_rows: usize) -> Self {
39 self.max_rows = Some(max_rows);
40 self
41 }
42
43 pub fn with_error_policy(mut self, policy: JsonErrorPolicy) -> Self {
45 self.error_policy = policy;
46 self
47 }
48
49 pub fn jsonl() -> Self {
51 Self {
52 format: Some(JsonFormat::Jsonl),
53 ..Default::default()
54 }
55 }
56
57 pub fn json_array() -> Self {
59 Self {
60 format: Some(JsonFormat::JsonArray),
61 ..Default::default()
62 }
63 }
64}
65
66pub type JsonObject = serde_json::Map<String, Value>;
68
69#[derive(Debug, Clone, PartialEq, Eq)]
71#[non_exhaustive]
72pub struct JsonScanSummary {
73 pub rows_read: usize,
74 pub malformed_lines: usize,
75 pub format: FileFormat,
76 pub truncated: bool,
79}
80
81pub fn scan_json_from_reader<R, F>(
86 mut reader: R,
87 config: &JsonParserConfig,
88 mut on_object: F,
89) -> Result<JsonScanSummary, DataProfilerError>
90where
91 R: BufRead,
92 F: FnMut(&JsonObject),
93{
94 let format = match config.format {
95 Some(JsonFormat::JsonArray) => FileFormat::Json,
96 Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
97 None => match consume_leading_whitespace(&mut reader)? {
98 Some(b'[') => FileFormat::Json,
99 _ => FileFormat::Jsonl,
100 },
101 };
102
103 let mut rows_read = 0;
104 let mut malformed_lines = 0;
105 let mut truncated = false;
106
107 match format {
108 FileFormat::Jsonl => loop {
109 if let Some(max) = config.max_rows
110 && rows_read >= max
111 {
112 truncated = consume_leading_whitespace(&mut reader)?.is_some();
115 break;
116 }
117
118 if consume_leading_whitespace(&mut reader)?.is_none() {
123 break;
124 }
125
126 let mut deserializer = serde_json::Deserializer::from_reader(&mut reader);
127 let value: Value = match Value::deserialize(&mut deserializer) {
128 Ok(v) => v,
129 Err(err) => {
130 if config.error_policy == JsonErrorPolicy::Strict {
131 return Err(malformed_record_error(&err));
132 }
133 malformed_lines += 1;
134 if err.classify() == serde_json::error::Category::Eof {
135 break;
136 }
137 skip_to_next_line(&mut reader)?;
138 continue;
139 }
140 };
141
142 if let Value::Object(ref obj) = value {
143 on_object(obj);
144 rows_read += 1;
145 }
146 },
147 FileFormat::Json => {
148 let mut found_array = false;
149 loop {
150 let mut consume = 0;
151 {
152 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
153 if buf.is_empty() {
154 break;
155 }
156 for &byte in buf {
157 consume += 1;
158 if byte == b'[' {
159 found_array = true;
160 break;
161 } else if !byte.is_ascii_whitespace() {
162 break;
163 }
164 }
165 }
166 reader.consume(consume);
167 if found_array || consume == 0 {
168 break;
169 }
170 }
171
172 if !found_array {
173 if config.format.is_some() {
174 return Err(DataProfilerError::JsonParsingError {
175 message: "Expected JSON array (starts with '[') but input does not match"
176 .to_string(),
177 });
178 }
179 } else {
180 let mut expect_value = true;
181 let mut allow_end = true;
182 let mut array_closed = false;
183 let mut drain_remainder = false;
184
185 loop {
186 let Some(next) = consume_leading_whitespace(&mut reader)? else {
187 if config.error_policy == JsonErrorPolicy::Strict {
188 return Err(malformed_array_error(
189 "unexpected end of input before closing ']'",
190 ));
191 }
192 malformed_lines += 1;
193 drain_remainder = true;
194 break;
195 };
196
197 if expect_value {
198 if next == b']' {
199 if !allow_end {
200 if config.error_policy == JsonErrorPolicy::Strict {
201 return Err(malformed_array_error(
202 "trailing comma before closing ']'",
203 ));
204 }
205 malformed_lines += 1;
206 drain_remainder = true;
207 break;
208 }
209 reader.consume(1);
210 array_closed = true;
211 break;
212 }
213
214 if let Some(max) = config.max_rows
215 && rows_read >= max
216 {
217 truncated = true;
220 break;
221 }
222
223 if next == b',' {
224 if config.error_policy == JsonErrorPolicy::Strict {
225 return Err(malformed_array_error(
226 "unexpected comma where an array value was required",
227 ));
228 }
229 malformed_lines += 1;
230 drain_remainder = true;
231 break;
232 }
233
234 let mut deserializer = serde_json::Deserializer::from_reader(&mut reader);
235 match Value::deserialize(&mut deserializer) {
236 Ok(Value::Object(obj)) => {
237 on_object(&obj);
238 rows_read += 1;
239 }
240 Ok(_) => {}
241 Err(err) => {
242 if config.error_policy == JsonErrorPolicy::Strict {
243 return Err(malformed_record_error(&err));
244 }
245 malformed_lines += 1;
246 drain_remainder = true;
247 break;
248 }
249 }
250 expect_value = false;
251 } else {
252 match next {
253 b',' => {
254 reader.consume(1);
255 expect_value = true;
256 allow_end = false;
257 }
258 b']' => {
259 reader.consume(1);
260 array_closed = true;
261 break;
262 }
263 _ => {
264 if config.error_policy == JsonErrorPolicy::Strict {
265 return Err(malformed_array_error(
266 "expected ',' or ']' after an array value",
267 ));
268 }
269 malformed_lines += 1;
270 drain_remainder = true;
271 break;
272 }
273 }
274 }
275 }
276
277 if drain_remainder {
278 drain_to_end(&mut reader)?;
279 } else if array_closed && consume_leading_whitespace(&mut reader)?.is_some() {
280 if config.error_policy == JsonErrorPolicy::Strict {
281 return Err(malformed_array_error(
282 "non-whitespace content follows the closing ']'",
283 ));
284 }
285 malformed_lines += 1;
286 drain_to_end(&mut reader)?;
287 }
288 }
289 }
290 _ => unreachable!("json scanner only returns json or jsonl formats"),
291 }
292
293 Ok(JsonScanSummary {
294 rows_read,
295 malformed_lines,
296 format,
297 truncated,
298 })
299}
300
301fn malformed_record_error(err: &serde_json::Error) -> DataProfilerError {
305 DataProfilerError::JsonParsingError {
306 message: format!("malformed JSON record: {err}"),
307 }
308}
309
310fn malformed_array_error(message: &str) -> DataProfilerError {
311 DataProfilerError::JsonParsingError {
312 message: format!("malformed JSON array: {message}"),
313 }
314}
315
316fn drain_to_end<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
317 loop {
318 let available = reader.fill_buf().map_err(DataProfilerError::from)?.len();
319 if available == 0 {
320 return Ok(());
321 }
322 reader.consume(available);
323 }
324}
325
326fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
327 loop {
328 let mut bytes_to_consume = 0;
329 let first_non_whitespace = {
330 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
331 if buf.is_empty() {
332 return Ok(None);
333 }
334
335 let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
336 if first_non_whitespace.is_none() {
337 bytes_to_consume = buf.len();
338 }
339 first_non_whitespace
340 };
341
342 if first_non_whitespace.is_some() {
343 return Ok(first_non_whitespace);
344 }
345
346 reader.consume(bytes_to_consume);
347 }
348}
349
350fn skip_to_next_line<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
351 let mut discarded = Vec::new();
352 reader
353 .read_until(b'\n', &mut discarded)
354 .map_err(DataProfilerError::from)?;
355 Ok(())
356}
357
358fn json_value_to_string(value: &Value) -> String {
360 match value {
361 Value::Null => String::new(),
362 Value::Bool(boolean) => boolean.to_string(),
363 Value::Number(number) => number.to_string(),
364 Value::String(string) => string.to_string(),
365 Value::Array(_) | Value::Object(_) => {
368 serde_json::to_string(value).expect("re-serializing a parsed JSON value cannot fail")
369 }
370 }
371}
372
373fn feed_json_object(
375 obj: &JsonObject,
376 prior_rows: usize,
377 known_columns: &mut Vec<String>,
378 known_columns_set: &mut HashSet<String>,
379 column_stats: &mut StreamingColumnCollection,
380) {
381 for key in obj.keys() {
382 if known_columns_set.insert(key.clone()) {
383 known_columns.push(key.clone());
384 column_stats.init_column_with_missing(key, prior_rows);
385 }
386 }
387
388 let values: Vec<String> = known_columns
389 .iter()
390 .map(|column| {
391 obj.get(column)
394 .map(json_value_to_string)
395 .unwrap_or_default()
396 })
397 .collect();
398
399 column_stats.process_record(known_columns, values);
400}
401
402pub fn analyze_json_from_reader<R: BufRead>(
406 reader: R,
407 config: &JsonParserConfig,
408) -> Result<
409 (
410 Vec<ColumnProfile>,
411 StreamingColumnCollection,
412 usize,
413 usize,
414 FileFormat,
415 ),
416 DataProfilerError,
417> {
418 analyze_json_from_reader_with_hints(reader, config, &SemanticHints::default())
419}
420
421pub fn analyze_json_from_reader_with_hints<R: BufRead>(
422 reader: R,
423 config: &JsonParserConfig,
424 semantic_hints: &SemanticHints,
425) -> Result<
426 (
427 Vec<ColumnProfile>,
428 StreamingColumnCollection,
429 usize,
430 usize,
431 FileFormat,
432 ),
433 DataProfilerError,
434> {
435 let (profiles, stats, rows_read, malformed_lines, format, _truncated) =
436 analyze_json_from_reader_full(reader, config, semantic_hints)?;
437 Ok((profiles, stats, rows_read, malformed_lines, format))
438}
439
440#[allow(clippy::type_complexity)]
443fn analyze_json_from_reader_full<R: BufRead>(
444 reader: R,
445 config: &JsonParserConfig,
446 semantic_hints: &SemanticHints,
447) -> Result<
448 (
449 Vec<ColumnProfile>,
450 StreamingColumnCollection,
451 usize,
452 usize,
453 FileFormat,
454 bool,
455 ),
456 DataProfilerError,
457> {
458 let mut column_stats = StreamingColumnCollection::new().with_semantic_hints(semantic_hints);
459 let mut known_columns = Vec::new();
460 let mut known_columns_set = HashSet::new();
461 let mut rows_seen = 0;
462
463 let summary = scan_json_from_reader(reader, config, |obj| {
464 feed_json_object(
465 obj,
466 rows_seen,
467 &mut known_columns,
468 &mut known_columns_set,
469 &mut column_stats,
470 );
471 rows_seen += 1;
472 })?;
473
474 let profiles = profile_builder::profiles_from_streaming_with_hints(
475 &column_stats,
476 false,
477 false,
478 None,
479 semantic_hints,
480 );
481
482 Ok((
483 profiles,
484 column_stats,
485 summary.rows_read,
486 summary.malformed_lines,
487 summary.format,
488 summary.truncated,
489 ))
490}
491
492pub fn analyze_json_file(
494 file_path: &Path,
495 config: &JsonParserConfig,
496) -> Result<ProfileReport, DataProfilerError> {
497 analyze_json_file_with_dimensions(file_path, config, None)
498}
499
500pub fn analyze_json_file_with_dimensions(
502 file_path: &Path,
503 config: &JsonParserConfig,
504 quality_dimensions: Option<&[QualityDimension]>,
505) -> Result<ProfileReport, DataProfilerError> {
506 analyze_json_file_with_dimensions_and_hints(
507 file_path,
508 config,
509 quality_dimensions,
510 &SemanticHints::default(),
511 )
512}
513
514pub fn analyze_json_file_with_dimensions_and_hints(
515 file_path: &Path,
516 config: &JsonParserConfig,
517 quality_dimensions: Option<&[QualityDimension]>,
518 semantic_hints: &SemanticHints,
519) -> Result<ProfileReport, DataProfilerError> {
520 let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
521 let start = std::time::Instant::now();
522
523 let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
524 let buf_reader = std::io::BufReader::new(file);
525
526 let (column_profiles, column_stats, rows_read, malformed_lines, format, truncated) =
527 analyze_json_from_reader_full(buf_reader, config, semantic_hints)?;
528
529 let file_source = DataSource::File {
530 path: file_path.display().to_string(),
531 format,
532 size_bytes: metadata.len(),
533 modified_at: None,
534 parquet_metadata: None,
535 };
536
537 if rows_read == 0 {
538 if malformed_lines > 0 {
539 return Err(DataProfilerError::JsonParsingError {
540 message: "No valid JSON records found in file (malformed JSON encountered)"
541 .to_string(),
542 });
543 }
544 let mut assembler = ReportAssembler::new(
545 file_source,
546 ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("json"),
547 )
548 .columns(column_profiles)
549 .with_quality_data(HashMap::new());
550 if let Some(dimensions) = quality_dimensions {
551 assembler = assembler.with_requested_dimensions(dimensions.to_vec());
552 }
553 return Ok(assembler.build());
554 }
555
556 let sample_columns = profile_builder::quality_check_samples(&column_stats);
557 let scan_time_ms = start.elapsed().as_millis();
558 let num_columns = column_profiles.len();
559
560 let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms)
561 .with_engine("json")
562 .with_error_count(malformed_lines);
565 if truncated && let Some(max) = config.max_rows {
568 execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
569 }
570
571 let mut assembler = ReportAssembler::new(file_source, execution)
572 .columns(column_profiles)
573 .with_quality_data(sample_columns)
574 .with_row_duplicates(column_stats.row_duplicate_summary())
575 .with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
576 .with_semantic_hints(semantic_hints.clone());
577 if let Some(dimensions) = quality_dimensions {
578 assembler = assembler.with_requested_dimensions(dimensions.to_vec());
579 }
580 Ok(assembler.build())
581}
582
583fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
584 if error.kind() == std::io::ErrorKind::NotFound {
585 DataProfilerError::FileNotFound {
586 path: file_path.display().to_string(),
587 }
588 } else {
589 DataProfilerError::from(error)
590 }
591}
592
593#[cfg(test)]
594mod tests {
595 use super::*;
596 use std::io::{Cursor, Write};
597 use tempfile::NamedTempFile;
598
599 fn write_file(content: &str) -> NamedTempFile {
600 let mut file = NamedTempFile::new().unwrap();
601 write!(file, "{}", content).unwrap();
602 file.flush().unwrap();
603 file
604 }
605
606 #[test]
607 fn test_scan_jsonl_detects_format_and_rows() {
608 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
609 let mut rows = 0;
610
611 let summary = scan_json_from_reader(
612 Cursor::new(data.as_ref()),
613 &JsonParserConfig::default(),
614 |_| {
615 rows += 1;
616 },
617 )
618 .unwrap();
619
620 assert_eq!(summary.format, FileFormat::Jsonl);
621 assert_eq!(summary.rows_read, 3);
622 assert_eq!(summary.malformed_lines, 0);
623 assert_eq!(rows, 3);
624 }
625
626 #[test]
627 fn test_scan_json_array_detects_format_and_rows() {
628 let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
629 let mut keys = Vec::new();
630
631 let summary = scan_json_from_reader(
632 Cursor::new(data.as_ref()),
633 &JsonParserConfig::default(),
634 |obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
635 )
636 .unwrap();
637
638 assert_eq!(summary.format, FileFormat::Json);
639 assert_eq!(summary.rows_read, 2);
640 assert_eq!(keys.len(), 2);
641 assert_eq!(keys[0], vec!["name".to_string()]);
642 }
643
644 #[test]
645 fn test_json_array_container_grammar_obeys_error_policy() {
646 let cases = [
647 (br#"[{"x":1}"#.as_ref(), 1, "missing closing bracket"),
648 (br#"[{"x":1},"#.as_ref(), 1, "comma followed by EOF"),
649 (br#"[{"x":1} {"x":2}]"#.as_ref(), 1, "missing comma"),
650 (br#"[{"x":1},,{"x":2}]"#.as_ref(), 1, "doubled comma"),
651 (br#"[,{"x":1}]"#.as_ref(), 0, "leading comma"),
652 (br#"[{"x":1},]"#.as_ref(), 1, "trailing comma"),
653 (br#"[{"x":1}] trailing"#.as_ref(), 1, "trailing content"),
654 (
655 br#"[{"x":1}][{"x":2}]"#.as_ref(),
656 1,
657 "second top-level value",
658 ),
659 ];
660
661 for (data, expected_rows, case) in cases {
662 let skip = JsonParserConfig::json_array();
663 let summary =
664 scan_json_from_reader(Cursor::new(data), &skip, |_| {}).expect("tolerant scan");
665 assert_eq!(summary.rows_read, expected_rows, "{case}");
666 assert_eq!(summary.malformed_lines, 1, "{case}");
667
668 let strict = JsonParserConfig::json_array().with_error_policy(JsonErrorPolicy::Strict);
669 let err = scan_json_from_reader(Cursor::new(data), &strict, |_| {})
670 .expect_err("strict mode must reject invalid array grammar");
671 assert!(
672 err.to_string()
673 .to_lowercase()
674 .contains("malformed json array"),
675 "{case}: {err}"
676 );
677 }
678 }
679
680 #[test]
681 fn test_json_array_max_rows_does_not_validate_unread_values() {
682 let data = br#"[{"x":1},not-valid-json"#;
683 let config = JsonParserConfig::json_array().with_max_rows(1);
684 let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
685
686 assert_eq!(summary.rows_read, 1);
687 assert_eq!(summary.malformed_lines, 0);
688 assert!(summary.truncated);
689 }
690
691 #[test]
692 fn test_scan_respects_max_rows() {
693 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
694 let config = JsonParserConfig::default().with_max_rows(2);
695 let mut rows = 0;
696
697 let summary =
698 scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
699
700 assert_eq!(summary.rows_read, 2);
701 assert_eq!(rows, 2);
702 }
703
704 #[test]
705 fn test_scan_skips_malformed_jsonl_lines() {
706 let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
707 let mut rows = 0;
708
709 let summary = scan_json_from_reader(
710 Cursor::new(data.as_ref()),
711 &JsonParserConfig::jsonl(),
712 |_| {
713 rows += 1;
714 },
715 )
716 .unwrap();
717
718 assert_eq!(summary.format, FileFormat::Jsonl);
719 assert_eq!(summary.rows_read, 2);
720 assert_eq!(summary.malformed_lines, 1);
721 assert_eq!(rows, 2);
722 }
723
724 #[test]
725 fn test_scan_pretty_printed_root_object_is_one_jsonl_row() {
726 let data = br#"{
727 "type": "FeatureCollection",
728 "features": [
729 {"id": 1},
730 {"id": 2}
731 ]
732}"#;
733 let mut object_field_counts = Vec::new();
734
735 let summary = scan_json_from_reader(
736 Cursor::new(data.as_ref()),
737 &JsonParserConfig::default(),
738 |obj| object_field_counts.push(obj.len()),
739 )
740 .unwrap();
741
742 assert_eq!(summary.format, FileFormat::Jsonl);
743 assert_eq!(summary.rows_read, 1);
744 assert_eq!(summary.malformed_lines, 0);
745 assert_eq!(object_field_counts, vec![2]);
746 }
747
748 #[test]
749 fn test_forced_json_array_requires_opening_bracket() {
750 let data = br#"{"x":1}"#;
751 let err = scan_json_from_reader(
752 Cursor::new(data.as_ref()),
753 &JsonParserConfig::json_array(),
754 |_| {},
755 )
756 .expect_err("forced json array should reject non-array input");
757
758 assert!(err.to_string().contains("Expected JSON array"));
759 }
760
761 #[test]
762 fn test_analyze_json_from_reader_jsonl_streaming() {
763 let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
764 let cursor = Cursor::new(data.as_ref());
765 let config = JsonParserConfig::default();
766
767 let (profiles, _stats, rows, _malformed, format) =
768 analyze_json_from_reader(cursor, &config).unwrap();
769 assert_eq!(format, FileFormat::Jsonl);
770 assert_eq!(rows, 3);
771 assert_eq!(profiles.len(), 2);
772 }
773
774 #[test]
775 fn test_analyze_json_from_reader_json_array() {
776 let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
777 let cursor = Cursor::new(data.as_ref());
778 let config = JsonParserConfig::default();
779
780 let (profiles, _stats, rows, _malformed, format) =
781 analyze_json_from_reader(cursor, &config).unwrap();
782 assert_eq!(format, FileFormat::Json);
783 assert_eq!(rows, 2);
784 assert_eq!(profiles.len(), 2);
785 }
786
787 #[test]
788 fn test_analyze_json_from_reader_max_rows() {
789 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
790 let cursor = Cursor::new(data.as_ref());
791 let config = JsonParserConfig::default().with_max_rows(3);
792
793 let (_profiles, _stats, rows, _malformed, _format) =
794 analyze_json_from_reader(cursor, &config).unwrap();
795 assert_eq!(rows, 3);
796 }
797
798 #[test]
799 fn test_analyze_json_from_reader_missing_fields() {
800 let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
801 let cursor = Cursor::new(data.as_ref());
802 let config = JsonParserConfig::jsonl();
803
804 let (profiles, _stats, rows, _malformed, _format) =
805 analyze_json_from_reader(cursor, &config).unwrap();
806 assert_eq!(rows, 2);
807
808 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
809 assert_eq!(col_b.total_count, 2);
810 }
811
812 #[test]
813 fn test_analyze_json_file_quality_report() {
814 let file = write_file(r#"[{"x":1},{"x":2}]"#);
815 let config = JsonParserConfig::default();
816 let report = analyze_json_file(file.path(), &config).unwrap();
817
818 assert_eq!(report.execution.rows_processed, 2);
819 assert_eq!(report.column_profiles.len(), 1);
820 assert!(report.quality_score().unwrap() >= 0.0);
821 }
822
823 #[test]
824 fn test_jsonl_skips_malformed_lines() {
825 let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
826 let cursor = Cursor::new(data.as_ref());
827 let config = JsonParserConfig::jsonl();
828
829 let (profiles, _stats, rows, _malformed, format) =
830 analyze_json_from_reader(cursor, &config).unwrap();
831 assert_eq!(format, FileFormat::Jsonl);
832 assert_eq!(rows, 2);
833 assert_eq!(profiles[0].total_count, 2);
834 }
835
836 #[test]
837 fn test_strict_policy_errors_on_malformed_jsonl() {
838 for data in [
840 b"not-json\n{\"x\":1}\n{\"x\":2}\n".as_ref(),
841 b"{\"x\":1}\nnot-json\n{\"x\":2}\n".as_ref(),
842 b"{\"x\":1}\n{\"x\":2}\nnot-json\n".as_ref(),
843 ] {
844 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
845 let err = scan_json_from_reader(Cursor::new(data), &config, |_| {})
846 .expect_err("strict mode must reject malformed records");
847 let message = err.to_string().to_lowercase();
848 assert!(message.contains("malformed json record"), "got: {message}");
849 assert!(
851 !err.to_string().contains("not-json"),
852 "leaked record: {err}"
853 );
854 }
855 }
856
857 #[test]
858 fn test_tolerant_policy_counts_skipped_records() {
859 let data = b"{\"x\":1}\nnot-json\n{\"x\":2}\n";
860 let config = JsonParserConfig::jsonl(); let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
862 assert_eq!(summary.rows_read, 2);
863 assert_eq!(summary.malformed_lines, 1);
864 }
865
866 #[test]
867 fn test_blank_lines_are_not_counted_as_malformed() {
868 let data = b"{\"x\":1}\n\n \n{\"x\":2}\n";
869 for policy in [JsonErrorPolicy::Skip, JsonErrorPolicy::Strict] {
870 let config = JsonParserConfig::jsonl().with_error_policy(policy);
871 let summary =
872 scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
873 assert_eq!(summary.rows_read, 2, "policy {policy:?}");
874 assert_eq!(summary.malformed_lines, 0, "policy {policy:?}");
875 }
876 }
877
878 #[test]
879 fn test_truncated_final_record_obeys_error_policy() {
880 let data = b"{\"x\":1}\n{\"x\":2";
881
882 let skip = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Skip);
883 let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &skip, |_| {}).unwrap();
884 assert_eq!(summary.rows_read, 1);
885 assert_eq!(summary.malformed_lines, 1);
886
887 let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
888 let err = scan_json_from_reader(Cursor::new(data.as_ref()), &strict, |_| {})
889 .expect_err("strict mode must reject an incomplete trailing record");
890 assert!(
891 err.to_string()
892 .to_lowercase()
893 .contains("malformed json record")
894 );
895 }
896
897 #[test]
898 fn test_all_malformed_file_fails() {
899 let file = write_file("not-json\nalso-bad\n");
900 let config = JsonParserConfig::jsonl();
901 let err = analyze_json_file(file.path(), &config)
902 .expect_err("a file with no valid records must fail");
903 let message = err.to_string().to_lowercase();
904 assert!(message.contains("no valid json records"), "got: {message}");
905 }
906
907 #[test]
908 fn test_tolerant_file_reports_error_count() {
909 let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
910 let config = JsonParserConfig::jsonl();
911 let report = analyze_json_file(file.path(), &config).unwrap();
912 assert_eq!(report.execution.rows_processed, 2);
913 assert_eq!(report.execution.error_count, 1);
914 }
915
916 #[test]
917 fn test_strict_file_errors_on_malformed() {
918 let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
919 let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
920 let err = analyze_json_file(file.path(), &config)
921 .expect_err("strict mode must reject the malformed record");
922 assert!(
923 err.to_string()
924 .to_lowercase()
925 .contains("malformed json record")
926 );
927 }
928
929 #[test]
930 fn test_analyze_json_with_large_leading_whitespace() {
931 let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
932 let cursor = Cursor::new(data.into_bytes());
933 let config = JsonParserConfig::default();
934
935 let (_profiles, _stats, rows, malformed, format) =
936 analyze_json_from_reader(cursor, &config).unwrap();
937 assert_eq!(format, FileFormat::Json);
938 assert_eq!(rows, 1);
939 assert_eq!(malformed, 0);
940 }
941
942 #[test]
943 fn test_analyze_json_array() {
944 let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
945 let config = JsonParserConfig::default();
946 let report = analyze_json_file(json.path(), &config).unwrap();
947 let profiles = &report.column_profiles;
948
949 assert_eq!(profiles.len(), 2);
950 let names: Vec<&str> = profiles
951 .iter()
952 .map(|profile| profile.name.as_str())
953 .collect();
954 assert!(names.contains(&"name"));
955 assert!(names.contains(&"age"));
956
957 let age = profiles
958 .iter()
959 .find(|profile| profile.name == "age")
960 .unwrap();
961 assert_eq!(age.total_count, 2);
962 assert_eq!(age.null_count, 0);
963 }
964
965 #[test]
966 fn test_analyze_jsonl() {
967 let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
968 let config = JsonParserConfig::default();
969 let report = analyze_json_file(jsonl.path(), &config).unwrap();
970 let profiles = &report.column_profiles;
971
972 assert_eq!(profiles.len(), 1);
973 assert_eq!(profiles[0].name, "x");
974 assert_eq!(profiles[0].total_count, 3);
975 }
976
977 #[test]
978 fn test_analyze_json_with_nulls() {
979 let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
980 let config = JsonParserConfig::default();
981 let report = analyze_json_file(json.path(), &config).unwrap();
982 let profiles = &report.column_profiles;
983
984 let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
985 assert_eq!(col_a.null_count, 1);
986
987 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
988 assert_eq!(col_b.null_count, 1);
989 }
990
991 #[test]
992 fn test_analyze_json_with_missing_fields() {
993 let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
994 let config = JsonParserConfig::default();
995 let report = analyze_json_file(json.path(), &config).unwrap();
996 let profiles = &report.column_profiles;
997
998 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
999 assert_eq!(col_b.total_count, 2);
1000 }
1001
1002 #[test]
1003 fn test_analyze_json_backfills_columns_discovered_after_first_row() {
1004 let json = write_file(r#"[{"a":1},{"a":2,"b":3}]"#);
1005 let config = JsonParserConfig::default();
1006 let report = analyze_json_file(json.path(), &config).unwrap();
1007 let col_b = report
1008 .column_profiles
1009 .iter()
1010 .find(|profile| profile.name == "b")
1011 .unwrap();
1012
1013 assert_eq!(col_b.total_count, 2);
1014 assert_eq!(col_b.null_count, 1);
1015 assert_eq!(col_b.unique_count, Some(1));
1016 }
1017
1018 #[test]
1019 fn test_analyze_json_empty_array() {
1020 let json = write_file("[]");
1021 let config = JsonParserConfig::default();
1022 let report = analyze_json_file(json.path(), &config).unwrap();
1023 assert!(report.column_profiles.is_empty());
1024 assert_eq!(report.quality_score(), None);
1026 }
1027
1028 #[test]
1029 fn test_analyze_json_malformed_returns_error() {
1030 let json = write_file("this is entirely invalid json");
1031 let config = JsonParserConfig::default();
1032 let err = analyze_json_file(json.path(), &config)
1033 .expect_err("malformed JSON should return an error");
1034
1035 let message = err.to_string().to_lowercase();
1036 assert!(message.contains("malformed") && message.contains("json"));
1037 }
1038
1039 #[test]
1040 fn test_analyze_json_file_detects_format() {
1041 let json_array = write_file(r#"[{"x":1}]"#);
1042 let config = JsonParserConfig::default();
1043 let report = analyze_json_file(json_array.path(), &config).unwrap();
1044 assert!(matches!(
1045 report.data_source,
1046 DataSource::File {
1047 format: FileFormat::Json,
1048 ..
1049 }
1050 ));
1051
1052 let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
1053 let report = analyze_json_file(jsonl.path(), &config).unwrap();
1054 assert!(matches!(
1055 report.data_source,
1056 DataSource::File {
1057 format: FileFormat::Jsonl,
1058 ..
1059 }
1060 ));
1061 }
1062
1063 #[test]
1064 fn test_analyze_json_file_empty() {
1065 let json = write_file("");
1066 let config = JsonParserConfig::default();
1067 let report = analyze_json_file(json.path(), &config).unwrap();
1068 assert_eq!(report.execution.rows_processed, 0);
1069 assert!(report.column_profiles.is_empty());
1070 }
1071
1072 #[test]
1073 fn test_analyze_json_boolean_and_nested() {
1074 let json =
1075 write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
1076 let config = JsonParserConfig::default();
1077 let report = analyze_json_file(json.path(), &config).unwrap();
1078 let profiles = &report.column_profiles;
1079
1080 let flag = profiles
1081 .iter()
1082 .find(|profile| profile.name == "flag")
1083 .unwrap();
1084 assert_eq!(flag.total_count, 2);
1085
1086 let nested = profiles
1087 .iter()
1088 .find(|profile| profile.name == "nested")
1089 .unwrap();
1090 assert_eq!(nested.total_count, 2);
1091 }
1092
1093 #[test]
1094 fn test_single_root_object_compact_yields_one_row() {
1095 let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
1096 let cursor = Cursor::new(data.as_ref());
1097 let config = JsonParserConfig::default();
1098
1099 let (profiles, _stats, rows, malformed, _format) =
1100 analyze_json_from_reader(cursor, &config).unwrap();
1101 assert_eq!(rows, 1);
1102 assert_eq!(malformed, 0);
1103 assert_eq!(profiles.len(), 2);
1104
1105 let names: Vec<&str> = profiles
1106 .iter()
1107 .map(|profile| profile.name.as_str())
1108 .collect();
1109 assert!(names.contains(&"type"));
1110 assert!(names.contains(&"features"));
1111 }
1112
1113 #[test]
1114 fn test_jsonl_multi_object_still_works() {
1115 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
1116 let cursor = Cursor::new(data.as_ref());
1117 let config = JsonParserConfig::default();
1118
1119 let (_profiles, _stats, rows, _malformed, format) =
1120 analyze_json_from_reader(cursor, &config).unwrap();
1121 assert_eq!(format, FileFormat::Jsonl);
1122 assert_eq!(rows, 3);
1123 }
1124
1125 #[test]
1126 fn test_single_root_object_via_analyze_json_file() {
1127 let file = write_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
1128 let config = JsonParserConfig::default();
1129 let report = analyze_json_file(file.path(), &config).unwrap();
1130
1131 assert_eq!(report.execution.rows_processed, 1);
1132 assert_eq!(report.column_profiles.len(), 2);
1133 }
1134
1135 #[test]
1136 fn test_single_root_object_pretty_printed_yields_one_row() {
1137 let data = br#"{
1138 "type": "FeatureCollection",
1139 "features": [
1140 {"id": 1},
1141 {"id": 2}
1142 ]
1143}"#;
1144 let cursor = Cursor::new(data.as_ref());
1145 let config = JsonParserConfig::default();
1146
1147 let (profiles, _stats, rows, malformed, format) =
1148 analyze_json_from_reader(cursor, &config).unwrap();
1149 assert_eq!(format, FileFormat::Jsonl);
1150 assert_eq!(rows, 1);
1151 assert_eq!(malformed, 0);
1152 assert_eq!(profiles.len(), 2);
1153 }
1154
1155 #[test]
1156 fn test_single_root_object_pretty_printed_via_analyze_json_file() {
1157 let file = write_file(
1158 "{\n \"type\": \"FeatureCollection\",\n \"features\": [\n {\"id\": 1},\n {\"id\": 2}\n ]\n}\n",
1159 );
1160 let config = JsonParserConfig::default();
1161 let report = analyze_json_file(file.path(), &config).unwrap();
1162
1163 assert_eq!(report.execution.rows_processed, 1);
1164 assert_eq!(report.column_profiles.len(), 2);
1165 }
1166}