1use std::collections::HashSet;
2use std::io::BufRead;
3use std::path::Path;
4
5use dataprof_core::{
6 ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat, QualityDimension,
7 SemanticHints, TruncationReason,
8};
9use dataprof_runtime::{
10 ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
11};
12use serde::Deserialize;
13use serde_json::Value;
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq)]
17pub enum JsonFormat {
18 JsonArray,
20 Jsonl,
22}
23
24#[derive(Debug, Clone, Default)]
26pub struct JsonParserConfig {
27 pub format: Option<JsonFormat>,
29 pub max_rows: Option<usize>,
31}
32
33impl JsonParserConfig {
34 pub fn with_max_rows(mut self, max_rows: usize) -> Self {
36 self.max_rows = Some(max_rows);
37 self
38 }
39
40 pub fn jsonl() -> Self {
42 Self {
43 format: Some(JsonFormat::Jsonl),
44 ..Default::default()
45 }
46 }
47
48 pub fn json_array() -> Self {
50 Self {
51 format: Some(JsonFormat::JsonArray),
52 ..Default::default()
53 }
54 }
55}
56
57pub type JsonObject = serde_json::Map<String, Value>;
59
60#[derive(Debug, Clone, PartialEq, Eq)]
62#[non_exhaustive]
63pub struct JsonScanSummary {
64 pub rows_read: usize,
65 pub malformed_lines: usize,
66 pub format: FileFormat,
67 pub truncated: bool,
70}
71
72pub fn scan_json_from_reader<R, F>(
77 mut reader: R,
78 config: &JsonParserConfig,
79 mut on_object: F,
80) -> Result<JsonScanSummary, DataProfilerError>
81where
82 R: BufRead,
83 F: FnMut(&JsonObject),
84{
85 let format = match config.format {
86 Some(JsonFormat::JsonArray) => FileFormat::Json,
87 Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
88 None => match consume_leading_whitespace(&mut reader)? {
89 Some(b'[') => FileFormat::Json,
90 _ => FileFormat::Jsonl,
91 },
92 };
93
94 let mut rows_read = 0;
95 let mut malformed_lines = 0;
96 let mut truncated = false;
97
98 match format {
99 FileFormat::Jsonl => loop {
100 if let Some(max) = config.max_rows
101 && rows_read >= max
102 {
103 truncated = consume_leading_whitespace(&mut reader)?.is_some();
106 break;
107 }
108
109 let mut deserializer = serde_json::Deserializer::from_reader(&mut reader);
110 let value: Value = match Value::deserialize(&mut deserializer) {
111 Ok(v) => v,
112 Err(err) if err.classify() == serde_json::error::Category::Eof => break,
113 Err(_) => {
114 malformed_lines += 1;
115 skip_to_next_line(&mut reader)?;
116 continue;
117 }
118 };
119
120 if let Value::Object(ref obj) = value {
121 on_object(obj);
122 rows_read += 1;
123 }
124 },
125 FileFormat::Json => {
126 let mut found_array = false;
127 loop {
128 let mut consume = 0;
129 {
130 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
131 if buf.is_empty() {
132 break;
133 }
134 for &byte in buf {
135 consume += 1;
136 if byte == b'[' {
137 found_array = true;
138 break;
139 } else if !byte.is_ascii_whitespace() {
140 break;
141 }
142 }
143 }
144 reader.consume(consume);
145 if found_array || consume == 0 {
146 break;
147 }
148 }
149
150 if !found_array {
151 if config.format.is_some() {
152 return Err(DataProfilerError::JsonParsingError {
153 message: "Expected JSON array (starts with '[') but input does not match"
154 .to_string(),
155 });
156 }
157 } else {
158 loop {
159 let mut consume = 0;
160 let mut found_value = false;
161 let mut end_of_array = false;
162
163 {
164 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
165 if buf.is_empty() {
166 break;
167 }
168 for &byte in buf {
169 if byte.is_ascii_whitespace() || byte == b',' {
170 consume += 1;
171 } else if byte == b']' {
172 end_of_array = true;
173 consume += 1;
174 break;
175 } else {
176 found_value = true;
177 break;
178 }
179 }
180 }
181
182 reader.consume(consume);
183 if end_of_array {
184 break;
185 }
186
187 if found_value {
188 if let Some(max) = config.max_rows
189 && rows_read >= max
190 {
191 truncated = true;
194 break;
195 }
196
197 let mut deserializer = serde_json::Deserializer::from_reader(&mut reader);
198 match Value::deserialize(&mut deserializer) {
199 Ok(Value::Object(obj)) => {
200 on_object(&obj);
201 rows_read += 1;
202 }
203 Ok(_) => {}
204 Err(_) => {
205 malformed_lines += 1;
206 break;
207 }
208 }
209 }
210 }
211 }
212 }
213 _ => unreachable!("json scanner only returns json or jsonl formats"),
214 }
215
216 Ok(JsonScanSummary {
217 rows_read,
218 malformed_lines,
219 format,
220 truncated,
221 })
222}
223
224fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
225 loop {
226 let mut bytes_to_consume = 0;
227 let first_non_whitespace = {
228 let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
229 if buf.is_empty() {
230 return Ok(None);
231 }
232
233 let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
234 if first_non_whitespace.is_none() {
235 bytes_to_consume = buf.len();
236 }
237 first_non_whitespace
238 };
239
240 if first_non_whitespace.is_some() {
241 return Ok(first_non_whitespace);
242 }
243
244 reader.consume(bytes_to_consume);
245 }
246}
247
248fn skip_to_next_line<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
249 let mut discarded = Vec::new();
250 reader
251 .read_until(b'\n', &mut discarded)
252 .map_err(DataProfilerError::from)?;
253 Ok(())
254}
255
256fn json_value_to_string(value: &Value) -> String {
258 match value {
259 Value::Null => String::new(),
260 Value::Bool(boolean) => boolean.to_string(),
261 Value::Number(number) => number.to_string(),
262 Value::String(string) => string.to_string(),
263 Value::Array(_) | Value::Object(_) => serde_json::to_string(value).unwrap_or_default(),
264 }
265}
266
267fn feed_json_object(
269 obj: &JsonObject,
270 known_columns: &mut Vec<String>,
271 known_columns_set: &mut HashSet<String>,
272 column_stats: &mut StreamingColumnCollection,
273) {
274 for key in obj.keys() {
275 if known_columns_set.insert(key.clone()) {
276 known_columns.push(key.clone());
277 }
278 }
279
280 let values: Vec<String> = known_columns
281 .iter()
282 .map(|column| {
283 obj.get(column)
284 .map(json_value_to_string)
285 .unwrap_or_default()
286 })
287 .collect();
288
289 column_stats.process_record(known_columns, values);
290}
291
292pub fn analyze_json_from_reader<R: BufRead>(
296 reader: R,
297 config: &JsonParserConfig,
298) -> Result<
299 (
300 Vec<ColumnProfile>,
301 StreamingColumnCollection,
302 usize,
303 usize,
304 FileFormat,
305 ),
306 DataProfilerError,
307> {
308 analyze_json_from_reader_with_hints(reader, config, &SemanticHints::default())
309}
310
311pub fn analyze_json_from_reader_with_hints<R: BufRead>(
312 reader: R,
313 config: &JsonParserConfig,
314 semantic_hints: &SemanticHints,
315) -> Result<
316 (
317 Vec<ColumnProfile>,
318 StreamingColumnCollection,
319 usize,
320 usize,
321 FileFormat,
322 ),
323 DataProfilerError,
324> {
325 let (profiles, stats, rows_read, malformed_lines, format, _truncated) =
326 analyze_json_from_reader_full(reader, config, semantic_hints)?;
327 Ok((profiles, stats, rows_read, malformed_lines, format))
328}
329
330#[allow(clippy::type_complexity)]
333fn analyze_json_from_reader_full<R: BufRead>(
334 reader: R,
335 config: &JsonParserConfig,
336 semantic_hints: &SemanticHints,
337) -> Result<
338 (
339 Vec<ColumnProfile>,
340 StreamingColumnCollection,
341 usize,
342 usize,
343 FileFormat,
344 bool,
345 ),
346 DataProfilerError,
347> {
348 let mut column_stats = StreamingColumnCollection::new();
349 let mut known_columns = Vec::new();
350 let mut known_columns_set = HashSet::new();
351
352 let summary = scan_json_from_reader(reader, config, |obj| {
353 feed_json_object(
354 obj,
355 &mut known_columns,
356 &mut known_columns_set,
357 &mut column_stats,
358 );
359 })?;
360
361 let profiles = profile_builder::profiles_from_streaming_with_hints(
362 &column_stats,
363 false,
364 false,
365 None,
366 semantic_hints,
367 );
368
369 Ok((
370 profiles,
371 column_stats,
372 summary.rows_read,
373 summary.malformed_lines,
374 summary.format,
375 summary.truncated,
376 ))
377}
378
379pub fn analyze_json_file(
381 file_path: &Path,
382 config: &JsonParserConfig,
383) -> Result<ProfileReport, DataProfilerError> {
384 analyze_json_file_with_dimensions(file_path, config, None)
385}
386
387pub fn analyze_json_file_with_dimensions(
389 file_path: &Path,
390 config: &JsonParserConfig,
391 quality_dimensions: Option<&[QualityDimension]>,
392) -> Result<ProfileReport, DataProfilerError> {
393 analyze_json_file_with_dimensions_and_hints(
394 file_path,
395 config,
396 quality_dimensions,
397 &SemanticHints::default(),
398 )
399}
400
401pub fn analyze_json_file_with_dimensions_and_hints(
402 file_path: &Path,
403 config: &JsonParserConfig,
404 quality_dimensions: Option<&[QualityDimension]>,
405 semantic_hints: &SemanticHints,
406) -> Result<ProfileReport, DataProfilerError> {
407 let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
408 let start = std::time::Instant::now();
409
410 let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
411 let buf_reader = std::io::BufReader::new(file);
412
413 let (column_profiles, column_stats, rows_read, malformed_lines, format, truncated) =
414 analyze_json_from_reader_full(buf_reader, config, semantic_hints)?;
415
416 let file_source = DataSource::File {
417 path: file_path.display().to_string(),
418 format,
419 size_bytes: metadata.len(),
420 modified_at: None,
421 parquet_metadata: None,
422 };
423
424 if rows_read == 0 {
425 if malformed_lines > 0 {
426 return Err(DataProfilerError::JsonParsingError {
427 message: "No valid JSON records found in file (malformed JSON encountered)"
428 .to_string(),
429 });
430 }
431 return Ok(ReportAssembler::new(
432 file_source,
433 ExecutionMetadata::new(0, 0, start.elapsed().as_millis()),
434 )
435 .skip_quality()
436 .build());
437 }
438
439 let sample_columns = profile_builder::quality_check_samples(&column_stats);
440 let scan_time_ms = start.elapsed().as_millis();
441 let num_columns = column_profiles.len();
442
443 let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms);
444 if truncated && let Some(max) = config.max_rows {
447 execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
448 }
449
450 let mut assembler = ReportAssembler::new(file_source, execution)
451 .columns(column_profiles)
452 .with_quality_data(sample_columns)
453 .with_semantic_hints(semantic_hints.clone());
454 if let Some(dimensions) = quality_dimensions {
455 assembler = assembler.with_requested_dimensions(dimensions.to_vec());
456 }
457 Ok(assembler.build())
458}
459
460fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
461 if error.kind() == std::io::ErrorKind::NotFound {
462 DataProfilerError::FileNotFound {
463 path: file_path.display().to_string(),
464 }
465 } else {
466 DataProfilerError::from(error)
467 }
468}
469
470#[cfg(test)]
471mod tests {
472 use super::*;
473 use std::io::{Cursor, Write};
474 use tempfile::NamedTempFile;
475
476 fn write_file(content: &str) -> NamedTempFile {
477 let mut file = NamedTempFile::new().unwrap();
478 write!(file, "{}", content).unwrap();
479 file.flush().unwrap();
480 file
481 }
482
483 #[test]
484 fn test_scan_jsonl_detects_format_and_rows() {
485 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
486 let mut rows = 0;
487
488 let summary = scan_json_from_reader(
489 Cursor::new(data.as_ref()),
490 &JsonParserConfig::default(),
491 |_| {
492 rows += 1;
493 },
494 )
495 .unwrap();
496
497 assert_eq!(summary.format, FileFormat::Jsonl);
498 assert_eq!(summary.rows_read, 3);
499 assert_eq!(summary.malformed_lines, 0);
500 assert_eq!(rows, 3);
501 }
502
503 #[test]
504 fn test_scan_json_array_detects_format_and_rows() {
505 let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
506 let mut keys = Vec::new();
507
508 let summary = scan_json_from_reader(
509 Cursor::new(data.as_ref()),
510 &JsonParserConfig::default(),
511 |obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
512 )
513 .unwrap();
514
515 assert_eq!(summary.format, FileFormat::Json);
516 assert_eq!(summary.rows_read, 2);
517 assert_eq!(keys.len(), 2);
518 assert_eq!(keys[0], vec!["name".to_string()]);
519 }
520
521 #[test]
522 fn test_scan_respects_max_rows() {
523 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
524 let config = JsonParserConfig::default().with_max_rows(2);
525 let mut rows = 0;
526
527 let summary =
528 scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
529
530 assert_eq!(summary.rows_read, 2);
531 assert_eq!(rows, 2);
532 }
533
534 #[test]
535 fn test_scan_skips_malformed_jsonl_lines() {
536 let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
537 let mut rows = 0;
538
539 let summary = scan_json_from_reader(
540 Cursor::new(data.as_ref()),
541 &JsonParserConfig::jsonl(),
542 |_| {
543 rows += 1;
544 },
545 )
546 .unwrap();
547
548 assert_eq!(summary.format, FileFormat::Jsonl);
549 assert_eq!(summary.rows_read, 2);
550 assert_eq!(summary.malformed_lines, 1);
551 assert_eq!(rows, 2);
552 }
553
554 #[test]
555 fn test_scan_pretty_printed_root_object_is_one_jsonl_row() {
556 let data = br#"{
557 "type": "FeatureCollection",
558 "features": [
559 {"id": 1},
560 {"id": 2}
561 ]
562}"#;
563 let mut object_field_counts = Vec::new();
564
565 let summary = scan_json_from_reader(
566 Cursor::new(data.as_ref()),
567 &JsonParserConfig::default(),
568 |obj| object_field_counts.push(obj.len()),
569 )
570 .unwrap();
571
572 assert_eq!(summary.format, FileFormat::Jsonl);
573 assert_eq!(summary.rows_read, 1);
574 assert_eq!(summary.malformed_lines, 0);
575 assert_eq!(object_field_counts, vec![2]);
576 }
577
578 #[test]
579 fn test_forced_json_array_requires_opening_bracket() {
580 let data = br#"{"x":1}"#;
581 let err = scan_json_from_reader(
582 Cursor::new(data.as_ref()),
583 &JsonParserConfig::json_array(),
584 |_| {},
585 )
586 .expect_err("forced json array should reject non-array input");
587
588 assert!(err.to_string().contains("Expected JSON array"));
589 }
590
591 #[test]
592 fn test_analyze_json_from_reader_jsonl_streaming() {
593 let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
594 let cursor = Cursor::new(data.as_ref());
595 let config = JsonParserConfig::default();
596
597 let (profiles, _stats, rows, _malformed, format) =
598 analyze_json_from_reader(cursor, &config).unwrap();
599 assert_eq!(format, FileFormat::Jsonl);
600 assert_eq!(rows, 3);
601 assert_eq!(profiles.len(), 2);
602 }
603
604 #[test]
605 fn test_analyze_json_from_reader_json_array() {
606 let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
607 let cursor = Cursor::new(data.as_ref());
608 let config = JsonParserConfig::default();
609
610 let (profiles, _stats, rows, _malformed, format) =
611 analyze_json_from_reader(cursor, &config).unwrap();
612 assert_eq!(format, FileFormat::Json);
613 assert_eq!(rows, 2);
614 assert_eq!(profiles.len(), 2);
615 }
616
617 #[test]
618 fn test_analyze_json_from_reader_max_rows() {
619 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
620 let cursor = Cursor::new(data.as_ref());
621 let config = JsonParserConfig::default().with_max_rows(3);
622
623 let (_profiles, _stats, rows, _malformed, _format) =
624 analyze_json_from_reader(cursor, &config).unwrap();
625 assert_eq!(rows, 3);
626 }
627
628 #[test]
629 fn test_analyze_json_from_reader_missing_fields() {
630 let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
631 let cursor = Cursor::new(data.as_ref());
632 let config = JsonParserConfig::jsonl();
633
634 let (profiles, _stats, rows, _malformed, _format) =
635 analyze_json_from_reader(cursor, &config).unwrap();
636 assert_eq!(rows, 2);
637
638 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
639 assert_eq!(col_b.total_count, 2);
640 }
641
642 #[test]
643 fn test_analyze_json_file_quality_report() {
644 let file = write_file(r#"[{"x":1},{"x":2}]"#);
645 let config = JsonParserConfig::default();
646 let report = analyze_json_file(file.path(), &config).unwrap();
647
648 assert_eq!(report.execution.rows_processed, 2);
649 assert_eq!(report.column_profiles.len(), 1);
650 assert!(report.quality_score().unwrap() >= 0.0);
651 }
652
653 #[test]
654 fn test_jsonl_skips_malformed_lines() {
655 let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
656 let cursor = Cursor::new(data.as_ref());
657 let config = JsonParserConfig::jsonl();
658
659 let (profiles, _stats, rows, _malformed, format) =
660 analyze_json_from_reader(cursor, &config).unwrap();
661 assert_eq!(format, FileFormat::Jsonl);
662 assert_eq!(rows, 2);
663 assert_eq!(profiles[0].total_count, 2);
664 }
665
666 #[test]
667 fn test_analyze_json_with_large_leading_whitespace() {
668 let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
669 let cursor = Cursor::new(data.into_bytes());
670 let config = JsonParserConfig::default();
671
672 let (_profiles, _stats, rows, malformed, format) =
673 analyze_json_from_reader(cursor, &config).unwrap();
674 assert_eq!(format, FileFormat::Json);
675 assert_eq!(rows, 1);
676 assert_eq!(malformed, 0);
677 }
678
679 #[test]
680 fn test_analyze_json_array() {
681 let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
682 let config = JsonParserConfig::default();
683 let report = analyze_json_file(json.path(), &config).unwrap();
684 let profiles = &report.column_profiles;
685
686 assert_eq!(profiles.len(), 2);
687 let names: Vec<&str> = profiles
688 .iter()
689 .map(|profile| profile.name.as_str())
690 .collect();
691 assert!(names.contains(&"name"));
692 assert!(names.contains(&"age"));
693
694 let age = profiles
695 .iter()
696 .find(|profile| profile.name == "age")
697 .unwrap();
698 assert_eq!(age.total_count, 2);
699 assert_eq!(age.null_count, 0);
700 }
701
702 #[test]
703 fn test_analyze_jsonl() {
704 let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
705 let config = JsonParserConfig::default();
706 let report = analyze_json_file(jsonl.path(), &config).unwrap();
707 let profiles = &report.column_profiles;
708
709 assert_eq!(profiles.len(), 1);
710 assert_eq!(profiles[0].name, "x");
711 assert_eq!(profiles[0].total_count, 3);
712 }
713
714 #[test]
715 fn test_analyze_json_with_nulls() {
716 let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
717 let config = JsonParserConfig::default();
718 let report = analyze_json_file(json.path(), &config).unwrap();
719 let profiles = &report.column_profiles;
720
721 let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
722 assert_eq!(col_a.null_count, 1);
723
724 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
725 assert_eq!(col_b.null_count, 1);
726 }
727
728 #[test]
729 fn test_analyze_json_with_missing_fields() {
730 let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
731 let config = JsonParserConfig::default();
732 let report = analyze_json_file(json.path(), &config).unwrap();
733 let profiles = &report.column_profiles;
734
735 let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
736 assert_eq!(col_b.total_count, 2);
737 }
738
739 #[test]
740 fn test_analyze_json_empty_array() {
741 let json = write_file("[]");
742 let config = JsonParserConfig::default();
743 let report = analyze_json_file(json.path(), &config).unwrap();
744 assert!(report.column_profiles.is_empty());
745 }
746
747 #[test]
748 fn test_analyze_json_malformed_returns_error() {
749 let json = write_file("this is entirely invalid json");
750 let config = JsonParserConfig::default();
751 let err = analyze_json_file(json.path(), &config)
752 .expect_err("malformed JSON should return an error");
753
754 let message = err.to_string().to_lowercase();
755 assert!(message.contains("malformed") && message.contains("json"));
756 }
757
758 #[test]
759 fn test_analyze_json_file_detects_format() {
760 let json_array = write_file(r#"[{"x":1}]"#);
761 let config = JsonParserConfig::default();
762 let report = analyze_json_file(json_array.path(), &config).unwrap();
763 assert!(matches!(
764 report.data_source,
765 DataSource::File {
766 format: FileFormat::Json,
767 ..
768 }
769 ));
770
771 let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
772 let report = analyze_json_file(jsonl.path(), &config).unwrap();
773 assert!(matches!(
774 report.data_source,
775 DataSource::File {
776 format: FileFormat::Jsonl,
777 ..
778 }
779 ));
780 }
781
782 #[test]
783 fn test_analyze_json_file_empty() {
784 let json = write_file("");
785 let config = JsonParserConfig::default();
786 let report = analyze_json_file(json.path(), &config).unwrap();
787 assert_eq!(report.execution.rows_processed, 0);
788 assert!(report.column_profiles.is_empty());
789 }
790
791 #[test]
792 fn test_analyze_json_boolean_and_nested() {
793 let json =
794 write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
795 let config = JsonParserConfig::default();
796 let report = analyze_json_file(json.path(), &config).unwrap();
797 let profiles = &report.column_profiles;
798
799 let flag = profiles
800 .iter()
801 .find(|profile| profile.name == "flag")
802 .unwrap();
803 assert_eq!(flag.total_count, 2);
804
805 let nested = profiles
806 .iter()
807 .find(|profile| profile.name == "nested")
808 .unwrap();
809 assert_eq!(nested.total_count, 2);
810 }
811
812 #[test]
813 fn test_single_root_object_compact_yields_one_row() {
814 let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
815 let cursor = Cursor::new(data.as_ref());
816 let config = JsonParserConfig::default();
817
818 let (profiles, _stats, rows, malformed, _format) =
819 analyze_json_from_reader(cursor, &config).unwrap();
820 assert_eq!(rows, 1);
821 assert_eq!(malformed, 0);
822 assert_eq!(profiles.len(), 2);
823
824 let names: Vec<&str> = profiles
825 .iter()
826 .map(|profile| profile.name.as_str())
827 .collect();
828 assert!(names.contains(&"type"));
829 assert!(names.contains(&"features"));
830 }
831
832 #[test]
833 fn test_jsonl_multi_object_still_works() {
834 let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
835 let cursor = Cursor::new(data.as_ref());
836 let config = JsonParserConfig::default();
837
838 let (_profiles, _stats, rows, _malformed, format) =
839 analyze_json_from_reader(cursor, &config).unwrap();
840 assert_eq!(format, FileFormat::Jsonl);
841 assert_eq!(rows, 3);
842 }
843
844 #[test]
845 fn test_single_root_object_via_analyze_json_file() {
846 let file = write_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
847 let config = JsonParserConfig::default();
848 let report = analyze_json_file(file.path(), &config).unwrap();
849
850 assert_eq!(report.execution.rows_processed, 1);
851 assert_eq!(report.column_profiles.len(), 2);
852 }
853
854 #[test]
855 fn test_single_root_object_pretty_printed_yields_one_row() {
856 let data = br#"{
857 "type": "FeatureCollection",
858 "features": [
859 {"id": 1},
860 {"id": 2}
861 ]
862}"#;
863 let cursor = Cursor::new(data.as_ref());
864 let config = JsonParserConfig::default();
865
866 let (profiles, _stats, rows, malformed, format) =
867 analyze_json_from_reader(cursor, &config).unwrap();
868 assert_eq!(format, FileFormat::Jsonl);
869 assert_eq!(rows, 1);
870 assert_eq!(malformed, 0);
871 assert_eq!(profiles.len(), 2);
872 }
873
874 #[test]
875 fn test_single_root_object_pretty_printed_via_analyze_json_file() {
876 let file = write_file(
877 "{\n \"type\": \"FeatureCollection\",\n \"features\": [\n {\"id\": 1},\n {\"id\": 2}\n ]\n}\n",
878 );
879 let config = JsonParserConfig::default();
880 let report = analyze_json_file(file.path(), &config).unwrap();
881
882 assert_eq!(report.execution.rows_processed, 1);
883 assert_eq!(report.column_profiles.len(), 2);
884 }
885}