use std::collections::{HashMap, HashSet};
use std::io::{BufRead, Read};
use std::path::Path;
pub use dataprof_core::JsonErrorPolicy;
#[doc(hidden)]
pub mod record;
use dataprof_core::{
AnalysisOptions, ColumnProfile, DataProfilerError, DataSource, ExecutionMetadata, FileFormat,
TruncationReason, Utf8BomReader,
};
use dataprof_runtime::{
ProfileReport, ReportAssembler, StreamingColumnCollection, profile_builder,
};
use record::{
JsonRecord, drain_to_end, json_document_error, json_value_kind, json_value_text,
malformed_array_error, non_object_record_error, object_record, read_json_record,
read_jsonl_line, rest_is_white_space, skip_ascii_whitespace, skip_leading_white_space,
wide_integer_record_error,
};
use serde_json::Value;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum JsonFormat {
Json,
Jsonl,
}
#[derive(Debug, Clone, Default)]
pub struct JsonParserConfig {
pub format: Option<JsonFormat>,
pub max_rows: Option<usize>,
pub error_policy: JsonErrorPolicy,
}
impl JsonParserConfig {
pub fn with_max_rows(mut self, max_rows: usize) -> Self {
self.max_rows = Some(max_rows);
self
}
pub fn with_error_policy(mut self, policy: JsonErrorPolicy) -> Self {
self.error_policy = policy;
self
}
pub fn jsonl() -> Self {
Self {
format: Some(JsonFormat::Jsonl),
..Default::default()
}
}
pub fn json_document() -> Self {
Self {
format: Some(JsonFormat::Json),
..Default::default()
}
}
pub fn with_format(mut self, format: JsonFormat) -> Self {
self.format = Some(format);
self
}
}
pub type JsonObject = serde_json::Map<String, Value>;
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct JsonScanSummary {
pub rows_read: usize,
pub malformed_lines: usize,
pub format: FileFormat,
pub truncated: bool,
pub unread_after_fault: bool,
}
pub fn scan_json_from_reader<R, F>(
reader: R,
config: &JsonParserConfig,
mut on_object: F,
) -> Result<JsonScanSummary, DataProfilerError>
where
R: BufRead,
F: FnMut(&JsonObject),
{
let mut reader = Utf8BomReader::new(reader).map_err(DataProfilerError::from)?;
let leading = match config.format {
Some(JsonFormat::Jsonl) => None,
_ => Some(skip_leading_white_space(&mut reader)?),
};
let first = leading.as_ref().and_then(|leading| leading.first);
let pending = leading.map(|leading| leading.pending).unwrap_or_default();
let mut reader = std::io::Cursor::new(pending).chain(reader);
let format = match config.format {
Some(JsonFormat::Json) => FileFormat::Json,
Some(JsonFormat::Jsonl) => FileFormat::Jsonl,
None if first == Some(b'[') => FileFormat::Json,
None => FileFormat::Jsonl,
};
let mut rows_read = 0;
let mut malformed_lines = 0;
let mut unread_after_fault = false;
let mut truncated = false;
match format {
FileFormat::Jsonl => {
let mut line = String::new();
let mut line_number = 0;
loop {
if let Some(max) = config.max_rows
&& rows_read >= max
{
truncated = next_record_line(&mut reader, &mut line, &mut line_number)?;
break;
}
if !next_record_line(&mut reader, &mut line, &mut line_number)? {
break;
}
match read_jsonl_line(&line) {
JsonRecord::Object(obj) => {
on_object(&obj);
rows_read += 1;
}
JsonRecord::NonObject(kind) => {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(non_object_record_error(
kind,
rows_read + malformed_lines + 1,
));
}
malformed_lines += 1;
}
JsonRecord::Malformed(err) => {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(record::malformed_jsonl_record_error(
line_number,
&line,
err,
));
}
malformed_lines += 1;
}
JsonRecord::WideInteger => {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(wide_integer_record_error(&format!(
"on line {line_number}"
)));
}
malformed_lines += 1;
}
}
}
}
FileFormat::Json => {
let found_array = first == Some(b'[');
if !found_array {
let (object_rows, object_malformed, object_truncated) =
scan_single_json_object(&mut reader, config, &mut on_object)?;
rows_read += object_rows;
malformed_lines += object_malformed;
truncated |= object_truncated;
} else {
consume_peeked(&mut reader)?;
let mut expect_value = true;
let mut allow_end = true;
let mut array_closed = false;
let mut drain_remainder = false;
loop {
let Some(next) = consume_leading_whitespace(&mut reader)? else {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(malformed_array_error(
"unexpected end of input before closing ']'",
));
}
malformed_lines += 1;
drain_remainder = true;
break;
};
if expect_value {
if next == b']' {
if !allow_end {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(malformed_array_error(
"trailing comma before closing ']'",
));
}
malformed_lines += 1;
drain_remainder = true;
break;
}
consume_peeked(&mut reader)?;
array_closed = true;
break;
}
if let Some(max) = config.max_rows
&& rows_read >= max
{
truncated = true;
break;
}
if next == b',' {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(malformed_array_error(
"unexpected comma where an array value was required",
));
}
malformed_lines += 1;
drain_remainder = true;
break;
}
match read_json_record(&mut reader, next)? {
JsonRecord::Object(obj) => {
on_object(&obj);
rows_read += 1;
}
JsonRecord::NonObject(kind) => {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(non_object_record_error(
kind,
rows_read + malformed_lines + 1,
));
}
malformed_lines += 1;
}
JsonRecord::Malformed(err) => {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(malformed_record_error(err));
}
malformed_lines += 1;
drain_remainder = true;
break;
}
JsonRecord::WideInteger => {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(wide_integer_record_error(&format!(
"at position {}",
rows_read + malformed_lines + 1
)));
}
malformed_lines += 1;
}
}
expect_value = false;
} else {
match next {
b',' => {
consume_peeked(&mut reader)?;
expect_value = true;
allow_end = false;
}
b']' => {
consume_peeked(&mut reader)?;
array_closed = true;
break;
}
_ => {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(malformed_array_error(
"expected ',' or ']' after an array value",
));
}
malformed_lines += 1;
drain_remainder = true;
break;
}
}
}
}
if drain_remainder {
unread_after_fault = !rest_is_white_space(&mut reader)?.0;
drain_to_end(&mut reader)?;
} else if array_closed && !rest_is_white_space(&mut reader)?.0 {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(malformed_array_error(
"non-whitespace content follows the closing ']'",
));
}
malformed_lines += 1;
drain_to_end(&mut reader)?;
}
}
}
_ => unreachable!("json scanner only returns json or jsonl formats"),
}
Ok(JsonScanSummary {
rows_read,
malformed_lines,
format,
truncated,
unread_after_fault,
})
}
fn next_record_line<R: BufRead>(
reader: &mut R,
line: &mut String,
line_number: &mut usize,
) -> Result<bool, DataProfilerError> {
loop {
line.clear();
let read = reader.read_line(line).map_err(DataProfilerError::from)?;
if read == 0 {
return Ok(false);
}
*line_number += 1;
if !line.trim().is_empty() {
return Ok(true);
}
}
}
fn scan_single_json_object<R, F>(
reader: &mut R,
config: &JsonParserConfig,
on_object: &mut F,
) -> Result<(usize, usize, bool), DataProfilerError>
where
R: BufRead,
F: FnMut(&JsonObject),
{
let mut text = String::new();
reader
.read_to_string(&mut text)
.map_err(DataProfilerError::from)?;
if text.trim().is_empty() {
return Ok((0, 0, false));
}
if config.max_rows == Some(0) {
return Ok((0, 0, true));
}
match serde_json::from_str::<Value>(text.trim()) {
Ok(Value::Object(obj)) => match object_record(obj, text.as_bytes()) {
JsonRecord::Object(obj) => {
on_object(&obj);
Ok((1, 0, false))
}
_ => {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(wide_integer_record_error("at position 1"));
}
Ok((0, 1, false))
}
},
Ok(value) => {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(non_object_record_error(json_value_kind(&value), 1));
}
Ok((0, 1, false))
}
Err(err) => {
if config.error_policy == JsonErrorPolicy::Strict {
return Err(json_document_error(err));
}
Ok((0, 1, false))
}
}
}
fn consume_peeked<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
skip_ascii_whitespace(reader)?;
reader.consume(1);
Ok(())
}
fn malformed_record_error(err: serde_json::Error) -> DataProfilerError {
let message = format!("malformed JSON record: {err}");
DataProfilerError::json_parsing_with_source(message, err)
}
fn consume_leading_whitespace<R: BufRead>(reader: &mut R) -> Result<Option<u8>, DataProfilerError> {
loop {
let mut bytes_to_consume = 0;
let first_non_whitespace = {
let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
if buf.is_empty() {
return Ok(None);
}
let first_non_whitespace = buf.iter().find(|byte| !byte.is_ascii_whitespace()).copied();
if first_non_whitespace.is_none() {
bytes_to_consume = buf.len();
}
first_non_whitespace
};
if first_non_whitespace.is_some() {
return Ok(first_non_whitespace);
}
reader.consume(bytes_to_consume);
}
}
fn feed_json_object(
obj: &JsonObject,
prior_rows: usize,
known_columns: &mut Vec<String>,
known_columns_set: &mut HashSet<String>,
column_stats: &mut StreamingColumnCollection,
container_counts: &mut HashMap<String, usize>,
) {
for (key, value) in obj {
if known_columns_set.insert(key.clone()) {
known_columns.push(key.clone());
column_stats.init_column_with_missing(key, prior_rows);
}
if matches!(value, Value::Array(_) | Value::Object(_)) {
*container_counts.entry(key.clone()).or_default() += 1;
}
}
let values: Vec<String> = known_columns
.iter()
.map(|column| {
obj.get(column).map(json_value_text).unwrap_or_default()
})
.collect();
let containers: Vec<usize> = known_columns
.iter()
.enumerate()
.filter(|(_, column)| matches!(obj.get(*column), Some(Value::Array(_) | Value::Object(_))))
.map(|(position, _)| position)
.collect();
column_stats.process_record_with_containers(known_columns, values, &containers);
}
pub fn analyze_json_from_reader<R: BufRead>(
reader: R,
config: &JsonParserConfig,
) -> Result<
(
Vec<ColumnProfile>,
StreamingColumnCollection,
usize,
usize,
FileFormat,
),
DataProfilerError,
> {
let options = AnalysisOptions::default();
let (profiles, stats, rows_read, malformed_lines, format, _truncated, _unread) =
analyze_json_from_reader_full(reader, config, &options)?;
Ok((profiles, stats, rows_read, malformed_lines, format))
}
#[allow(clippy::type_complexity)]
fn analyze_json_from_reader_full<R: BufRead>(
reader: R,
config: &JsonParserConfig,
options: &AnalysisOptions,
) -> Result<
(
Vec<ColumnProfile>,
StreamingColumnCollection,
usize,
usize,
FileFormat,
bool,
bool,
),
DataProfilerError,
> {
let semantic_hints = options.semantic_hints();
let mut column_stats = StreamingColumnCollection::new()
.with_semantic_hints(semantic_hints)
.with_duplicate_rows_exact_limit(options.duplicate_rows_exact_limit());
let mut known_columns = Vec::new();
let mut known_columns_set = HashSet::new();
let mut container_counts = HashMap::new();
let mut rows_seen = 0;
let summary = scan_json_from_reader(reader, config, |obj| {
feed_json_object(
obj,
rows_seen,
&mut known_columns,
&mut known_columns_set,
&mut column_stats,
&mut container_counts,
);
rows_seen += 1;
})?;
if let Some(indices) = options.column_indices(&column_stats.column_names())? {
let available = column_stats.column_names();
let selected = indices
.into_iter()
.map(|index| available[index].clone())
.collect::<Vec<_>>();
column_stats.retain_columns(&selected);
}
let profiles = profile_builder::mark_container_columns(
profile_builder::profiles_from_streaming_with_hints(
&column_stats,
!options.include_statistics(),
!options.include_patterns(),
options.locale(),
semantic_hints,
),
&container_counts,
);
Ok((
profiles,
column_stats,
summary.rows_read,
summary.malformed_lines,
summary.format,
summary.truncated,
summary.unread_after_fault,
))
}
pub fn analyze_json_file(
file_path: &Path,
config: &JsonParserConfig,
) -> Result<ProfileReport, DataProfilerError> {
analyze_json_file_with_options(file_path, config, &AnalysisOptions::default())
}
pub fn analyze_json_file_with_options(
file_path: &Path,
config: &JsonParserConfig,
options: &AnalysisOptions,
) -> Result<ProfileReport, DataProfilerError> {
let mut config = config.clone();
if config.format.is_none() {
config.format = grammar_for_extension(file_path);
}
let config = &config;
let metadata = std::fs::metadata(file_path).map_err(|error| map_io_error(file_path, error))?;
dataprof_core::binary_input::reject_binary_input(file_path)?;
let start = std::time::Instant::now();
let file = std::fs::File::open(file_path).map_err(|error| map_io_error(file_path, error))?;
let buf_reader = std::io::BufReader::new(file);
let (
column_profiles,
column_stats,
rows_read,
malformed_lines,
format,
truncated,
unread_after_fault,
) = analyze_json_from_reader_full(buf_reader, config, options)?;
let read_as_json_document = format == FileFormat::Json;
let file_source = DataSource::File {
path: file_path.display().to_string(),
format,
size_bytes: metadata.len(),
modified_at: None,
parquet_metadata: None,
csv_metadata: None,
};
if rows_read == 0 {
if malformed_lines > 0 {
let hint = if read_as_json_document {
". A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
} else {
""
};
return Err(DataProfilerError::json_parsing_error(&format!(
"No valid JSON records found in file \
(every record was malformed, not a JSON object, or held an integer outside the 64-bit range){hint}"
)));
}
let mut execution =
ExecutionMetadata::new(0, 0, start.elapsed().as_millis()).with_engine("json");
if truncated && let Some(max) = config.max_rows {
execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
}
return Ok(ReportAssembler::new(file_source, execution)
.columns(column_profiles)
.with_quality_data(HashMap::new())
.with_analysis_options(options)
.build());
}
let sample_columns = profile_builder::quality_check_samples(&column_stats);
let scan_time_ms = start.elapsed().as_millis();
let num_columns = column_profiles.len();
let mut execution = ExecutionMetadata::new(rows_read, num_columns, scan_time_ms)
.with_engine("json")
.with_error_count(malformed_lines);
if truncated && let Some(max) = config.max_rows {
execution = execution.with_truncation(TruncationReason::MaxRows(max as u64));
}
if unread_after_fault {
execution = execution.with_source_exhausted(false);
}
Ok(ReportAssembler::new(file_source, execution)
.columns(column_profiles)
.with_quality_data(sample_columns)
.with_row_duplicates(column_stats.row_duplicate_summary())
.with_row_completeness(column_stats.row_completeness_summary())
.with_exact_value_hint_bindings(column_stats.semantic_hint_bindings())
.with_analysis_options(options)
.build())
}
fn grammar_for_extension(file_path: &Path) -> Option<JsonFormat> {
let extension = file_path.extension().and_then(|ext| ext.to_str())?;
if extension.eq_ignore_ascii_case("json") {
Some(JsonFormat::Json)
} else if extension.eq_ignore_ascii_case("jsonl") || extension.eq_ignore_ascii_case("ndjson") {
Some(JsonFormat::Jsonl)
} else {
None
}
}
fn map_io_error(file_path: &Path, error: std::io::Error) -> DataProfilerError {
if error.kind() == std::io::ErrorKind::NotFound {
DataProfilerError::FileNotFound {
path: file_path.display().to_string(),
}
} else {
DataProfilerError::from(error)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::{Cursor, Write};
use tempfile::NamedTempFile;
fn write_file(content: &str) -> NamedTempFile {
let mut file = NamedTempFile::new().unwrap();
write!(file, "{}", content).unwrap();
file.flush().unwrap();
file
}
fn write_json_file(content: &str) -> NamedTempFile {
let mut file = NamedTempFile::with_suffix(".json").unwrap();
write!(file, "{}", content).unwrap();
file.flush().unwrap();
file
}
fn write_bytes(content: &[u8]) -> NamedTempFile {
let mut file = NamedTempFile::new().unwrap();
file.write_all(content).unwrap();
file.flush().unwrap();
file
}
#[test]
fn test_scan_jsonl_detects_format_and_rows() {
let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
let mut rows = 0;
let summary = scan_json_from_reader(
Cursor::new(data.as_ref()),
&JsonParserConfig::default(),
|_| {
rows += 1;
},
)
.unwrap();
assert_eq!(summary.format, FileFormat::Jsonl);
assert_eq!(summary.rows_read, 3);
assert_eq!(summary.malformed_lines, 0);
assert_eq!(rows, 3);
}
#[test]
fn test_scan_json_array_detects_format_and_rows() {
let data = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
let mut keys = Vec::new();
let summary = scan_json_from_reader(
Cursor::new(data.as_ref()),
&JsonParserConfig::default(),
|obj| keys.push(obj.keys().cloned().collect::<Vec<_>>()),
)
.unwrap();
assert_eq!(summary.format, FileFormat::Json);
assert_eq!(summary.rows_read, 2);
assert_eq!(keys.len(), 2);
assert_eq!(keys[0], vec!["name".to_string()]);
}
#[test]
fn test_scan_accepts_one_leading_utf8_bom_for_json_and_jsonl() {
for (data, config, expected_format) in [
(
b"\xEF\xBB\xBF[{\"x\":1},{\"x\":2}]".as_slice(),
JsonParserConfig::default(),
FileFormat::Json,
),
(
b"\xEF\xBB\xBF{\"x\":1}\n{\"x\":2}\n".as_slice(),
JsonParserConfig::default(),
FileFormat::Jsonl,
),
] {
let mut rows = 0;
let summary = scan_json_from_reader(Cursor::new(data), &config, |_| rows += 1)
.expect("leading UTF-8 BOM should be accepted");
assert_eq!(rows, 2);
assert_eq!(summary.rows_read, 2);
assert_eq!(summary.malformed_lines, 0);
assert_eq!(summary.format, expected_format);
}
}
#[test]
fn test_scan_does_not_strip_nonleading_or_second_utf8_bom() {
let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
for data in [
b" \xEF\xBB\xBF{\"x\":1}\n".as_slice(),
b"\xEF\xBB\xBF\xEF\xBB\xBF{\"x\":1}\n".as_slice(),
b"{\"x\":1}\n\xEF\xBB\xBF{\"x\":2}\n".as_slice(),
] {
scan_json_from_reader(Cursor::new(data), &strict, |_| {})
.expect_err("only the BOM at absolute byte offset zero may be stripped");
}
}
#[test]
fn test_bom_only_input_matches_empty_input_policy() {
for config in [JsonParserConfig::default(), JsonParserConfig::jsonl()] {
let empty = scan_json_from_reader(Cursor::new(b""), &config, |_| {}).unwrap();
let bom =
scan_json_from_reader(Cursor::new(b"\xEF\xBB\xBF".as_slice()), &config, |_| {})
.unwrap();
assert_eq!(bom, empty);
}
}
#[test]
fn test_json_array_container_grammar_obeys_error_policy() {
let cases = [
(br#"[{"x":1}"#.as_ref(), 1, "missing closing bracket"),
(br#"[{"x":1},"#.as_ref(), 1, "comma followed by EOF"),
(br#"[{"x":1} {"x":2}]"#.as_ref(), 1, "missing comma"),
(br#"[{"x":1},,{"x":2}]"#.as_ref(), 1, "doubled comma"),
(br#"[,{"x":1}]"#.as_ref(), 0, "leading comma"),
(br#"[{"x":1},]"#.as_ref(), 1, "trailing comma"),
(br#"[{"x":1}] trailing"#.as_ref(), 1, "trailing content"),
(
br#"[{"x":1}][{"x":2}]"#.as_ref(),
1,
"second top-level value",
),
];
for (data, expected_rows, case) in cases {
let skip = JsonParserConfig::json_document();
let summary =
scan_json_from_reader(Cursor::new(data), &skip, |_| {}).expect("tolerant scan");
assert_eq!(summary.rows_read, expected_rows, "{case}");
assert_eq!(summary.malformed_lines, 1, "{case}");
let strict =
JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict);
let err = scan_json_from_reader(Cursor::new(data), &strict, |_| {})
.expect_err("strict mode must reject invalid array grammar");
assert!(
err.to_string()
.to_lowercase()
.contains("malformed json array"),
"{case}: {err}"
);
}
}
#[test]
fn test_json_array_max_rows_does_not_validate_unread_values() {
let data = br#"[{"x":1},not-valid-json"#;
let config = JsonParserConfig::json_document().with_max_rows(1);
let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
assert_eq!(summary.rows_read, 1);
assert_eq!(summary.malformed_lines, 0);
assert!(summary.truncated);
}
#[test]
fn test_scan_respects_max_rows() {
let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n";
let config = JsonParserConfig::default().with_max_rows(2);
let mut rows = 0;
let summary =
scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| rows += 1).unwrap();
assert_eq!(summary.rows_read, 2);
assert_eq!(rows, 2);
}
#[test]
fn test_scan_skips_malformed_jsonl_lines() {
let data = b"{\"x\":1}\n{\"x\":,bad}\n{\"x\":3}\n";
let mut rows = 0;
let summary = scan_json_from_reader(
Cursor::new(data.as_ref()),
&JsonParserConfig::jsonl(),
|_| {
rows += 1;
},
)
.unwrap();
assert_eq!(summary.format, FileFormat::Jsonl);
assert_eq!(summary.rows_read, 2);
assert_eq!(summary.malformed_lines, 1);
assert_eq!(rows, 2);
}
#[test]
fn test_pretty_printed_root_object_is_one_row_under_the_json_grammar() {
let data = br#"{
"type": "FeatureCollection",
"features": [
{"id": 1},
{"id": 2}
]
}"#;
let mut object_field_counts = Vec::new();
let summary = scan_json_from_reader(
Cursor::new(data.as_ref()),
&JsonParserConfig::json_document(),
|obj| object_field_counts.push(obj.len()),
)
.unwrap();
assert_eq!(summary.format, FileFormat::Json);
assert_eq!(summary.rows_read, 1);
assert_eq!(summary.malformed_lines, 0);
assert_eq!(object_field_counts, vec![2]);
}
#[test]
fn test_pretty_printed_root_object_is_not_valid_jsonl() {
let data = br#"{
"type": "FeatureCollection"
}"#;
let err = scan_json_from_reader(
Cursor::new(data.as_ref()),
&JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict),
|_| {},
)
.expect_err("a record spanning lines is not JSONL");
assert!(
err.to_string().contains("malformed JSON record"),
"{err}, expected the malformed category"
);
}
const NON_OBJECT_VALUES: &[(&str, &str)] = &[
("null", "null"),
("boolean", "true"),
("number", "42"),
("number", "-1.5e3"),
("string", r#""text""#),
("array", "[1, 2]"),
];
#[test]
fn test_jsonl_counts_non_object_records_and_keeps_scanning() {
for (kind, value) in NON_OBJECT_VALUES {
let data = format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n");
let mut rows = 0;
let summary = scan_json_from_reader(
Cursor::new(data.as_bytes()),
&JsonParserConfig::jsonl(),
|_| rows += 1,
)
.unwrap();
assert_eq!(summary.rows_read, 2, "{kind}: record after it was lost");
assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
assert_eq!(rows, 2, "{kind}");
}
}
#[test]
fn test_json_array_counts_non_object_elements_and_keeps_scanning() {
for (kind, value) in NON_OBJECT_VALUES {
let data = format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]");
let mut rows = 0;
let summary = scan_json_from_reader(
Cursor::new(data.as_bytes()),
&JsonParserConfig::json_document(),
|_| rows += 1,
)
.unwrap();
assert_eq!(summary.rows_read, 2, "{kind}: element after it was lost");
assert_eq!(summary.malformed_lines, 1, "{kind}: was silently dropped");
assert_eq!(rows, 2, "{kind}");
}
}
#[test]
fn test_strict_policy_rejects_the_first_non_object_record() {
for (kind, value) in NON_OBJECT_VALUES {
for (config, data) in [
(
JsonParserConfig::jsonl(),
format!("{{\"id\":1}}\n{value}\n{{\"id\":2}}\n"),
),
(
JsonParserConfig::json_document(),
format!("[{{\"id\":1}}, {value}, {{\"id\":2}}]"),
),
] {
let config = config.with_error_policy(JsonErrorPolicy::Strict);
let err = scan_json_from_reader(Cursor::new(data.as_bytes()), &config, |_| {})
.expect_err("{kind}: strict mode must reject a non-object record");
let message = err.to_string();
assert!(message.contains("non-object JSON record"), "{message}");
assert!(message.contains("at position 2"), "{message}");
assert!(message.contains(&format!("found {kind}")), "{message}");
}
}
}
#[test]
fn test_number_element_does_not_swallow_the_array_delimiter() {
let data = br#"[{"id":1}, 1, 2, 3, {"id":2}, 4, {"id":3}]"#;
let mut rows = 0;
let summary = scan_json_from_reader(
Cursor::new(data.as_ref()),
&JsonParserConfig::json_document(),
|_| rows += 1,
)
.unwrap();
assert_eq!(summary.rows_read, 3);
assert_eq!(summary.malformed_lines, 4);
assert_eq!(rows, 3);
}
#[test]
fn test_number_shaped_garbage_stays_a_malformed_record() {
let data = b"{\"id\":1}\n1.2.3\n";
let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
let err = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {})
.expect_err("number-shaped garbage should be malformed");
assert!(
err.to_string().contains("malformed JSON record"),
"{err}, expected the malformed category"
);
}
#[test]
fn test_input_of_only_non_object_records_fails() {
let file = write_file("\"just a string\"\n1\n[2]\n");
let err = analyze_json_file(file.path(), &JsonParserConfig::jsonl())
.expect_err("nothing profileable should not return a clean empty profile");
assert!(
err.to_string().contains("No valid JSON records found"),
"{err}"
);
}
#[test]
fn test_json_grammar_reads_a_bare_object_as_one_row() {
let data = br#"{"x":1}"#;
let mut rows = 0;
let summary = scan_json_from_reader(
Cursor::new(data.as_ref()),
&JsonParserConfig::json_document(),
|_| rows += 1,
)
.unwrap();
assert_eq!(summary.rows_read, 1);
assert_eq!(summary.malformed_lines, 0);
assert_eq!(rows, 1);
}
#[test]
fn test_json_grammar_rejects_concatenated_objects() {
let data = br#"{"x":1}{"x":2}"#;
let err = scan_json_from_reader(
Cursor::new(data.as_ref()),
&JsonParserConfig::json_document().with_error_policy(JsonErrorPolicy::Strict),
|_| {},
)
.expect_err("two objects are not one JSON document");
let message = err.to_string();
assert!(message.contains("malformed JSON document"), "{message}");
assert!(message.contains("format=\"jsonl\""), "{message}");
}
#[test]
fn test_analyze_json_from_reader_jsonl_streaming() {
let data = b"{\"x\":1,\"y\":\"a\"}\n{\"x\":2,\"y\":\"b\"}\n{\"x\":3,\"y\":\"c\"}\n";
let cursor = Cursor::new(data.as_ref());
let config = JsonParserConfig::default();
let (profiles, _stats, rows, _malformed, format) =
analyze_json_from_reader(cursor, &config).unwrap();
assert_eq!(format, FileFormat::Jsonl);
assert_eq!(rows, 3);
assert_eq!(profiles.len(), 2);
}
#[test]
fn test_analyze_json_from_reader_json_array() {
let data = br#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#;
let cursor = Cursor::new(data.as_ref());
let config = JsonParserConfig::default();
let (profiles, _stats, rows, _malformed, format) =
analyze_json_from_reader(cursor, &config).unwrap();
assert_eq!(format, FileFormat::Json);
assert_eq!(rows, 2);
assert_eq!(profiles.len(), 2);
}
#[test]
fn test_integer_negative_zero_profiles_as_integer_zero_in_every_shape() {
for (data, config) in [
(
&b"[{\"v\":0},{\"v\":-0}]"[..],
JsonParserConfig::json_document(),
),
(&b"{\"v\":0}\n{\"v\":-0}\n"[..], JsonParserConfig::jsonl()),
(&b"{\"v\":-0}"[..], JsonParserConfig::json_document()),
] {
let (profiles, _stats, _rows, _malformed, _format) =
analyze_json_from_reader(Cursor::new(data), &config).unwrap();
assert_eq!(
profiles[0].data_type,
dataprof_core::DataType::Integer,
"{data:?}"
);
assert_eq!(profiles[0].unique_count, Some(1), "{data:?}");
}
}
#[test]
fn test_analyze_json_from_reader_max_rows() {
let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n{\"x\":4}\n{\"x\":5}\n";
let cursor = Cursor::new(data.as_ref());
let config = JsonParserConfig::default().with_max_rows(3);
let (_profiles, _stats, rows, _malformed, _format) =
analyze_json_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 3);
}
#[test]
fn test_analyze_json_from_reader_missing_fields() {
let data = b"{\"a\":1,\"b\":2}\n{\"a\":3}\n";
let cursor = Cursor::new(data.as_ref());
let config = JsonParserConfig::jsonl();
let (profiles, _stats, rows, _malformed, _format) =
analyze_json_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 2);
let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
assert_eq!(col_b.total_count, 2);
}
#[test]
fn test_identical_records_count_as_duplicates_across_a_late_key() {
let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"x\"}\n";
let cursor = Cursor::new(data.as_ref());
let config = JsonParserConfig::jsonl();
let (_profiles, stats, rows, _malformed, _format) =
analyze_json_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 3);
let summary = stats
.row_duplicate_summary()
.expect("three rows were observed");
assert_eq!(summary.rows_checked, 3);
assert_eq!(summary.duplicate_rows, 1);
assert!(!summary.approximate);
}
#[test]
fn test_a_late_key_does_not_invent_duplicates() {
let data = b"{\"a\": \"x\"}\n{\"a\": \"x\", \"b\": \"y\"}\n{\"a\": \"z\"}\n";
let cursor = Cursor::new(data.as_ref());
let config = JsonParserConfig::jsonl();
let (_profiles, stats, _rows, _malformed, _format) =
analyze_json_from_reader(cursor, &config).unwrap();
let summary = stats
.row_duplicate_summary()
.expect("three rows were observed");
assert_eq!(summary.duplicate_rows, 0);
}
#[test]
fn test_analyze_json_file_quality_report() {
let file = write_file(r#"[{"x":1},{"x":2}]"#);
let config = JsonParserConfig::default();
let report = analyze_json_file(file.path(), &config).unwrap();
assert_eq!(report.execution.rows_processed, 2);
assert_eq!(report.column_profiles.len(), 1);
assert!(report.quality_score().unwrap() >= 0.0);
}
#[test]
fn test_jsonl_skips_malformed_lines() {
let data = b"{\"x\":1}\n{\"x\":,malformed}\n{\"x\":3}\n";
let cursor = Cursor::new(data.as_ref());
let config = JsonParserConfig::jsonl();
let (profiles, _stats, rows, _malformed, format) =
analyze_json_from_reader(cursor, &config).unwrap();
assert_eq!(format, FileFormat::Jsonl);
assert_eq!(rows, 2);
assert_eq!(profiles[0].total_count, 2);
}
#[test]
fn test_strict_policy_errors_on_malformed_jsonl() {
for data in [
b"not-json\n{\"x\":1}\n{\"x\":2}\n".as_ref(),
b"{\"x\":1}\nnot-json\n{\"x\":2}\n".as_ref(),
b"{\"x\":1}\n{\"x\":2}\nnot-json\n".as_ref(),
] {
let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
let err = scan_json_from_reader(Cursor::new(data), &config, |_| {})
.expect_err("strict mode must reject malformed records");
let message = err.to_string().to_lowercase();
assert!(message.contains("malformed json record"), "got: {message}");
assert!(
!err.to_string().contains("not-json"),
"leaked record: {err}"
);
}
}
#[test]
fn test_tolerant_policy_counts_skipped_records() {
let data = b"{\"x\":1}\nnot-json\n{\"x\":2}\n";
let config = JsonParserConfig::jsonl(); let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
assert_eq!(summary.rows_read, 2);
assert_eq!(summary.malformed_lines, 1);
}
#[test]
fn test_blank_lines_are_not_counted_as_malformed() {
let data = b"{\"x\":1}\n\n \n{\"x\":2}\n";
for policy in [JsonErrorPolicy::Skip, JsonErrorPolicy::Strict] {
let config = JsonParserConfig::jsonl().with_error_policy(policy);
let summary =
scan_json_from_reader(Cursor::new(data.as_ref()), &config, |_| {}).unwrap();
assert_eq!(summary.rows_read, 2, "policy {policy:?}");
assert_eq!(summary.malformed_lines, 0, "policy {policy:?}");
}
}
#[test]
fn test_truncated_final_record_obeys_error_policy() {
let data = b"{\"x\":1}\n{\"x\":2";
let skip = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Skip);
let summary = scan_json_from_reader(Cursor::new(data.as_ref()), &skip, |_| {}).unwrap();
assert_eq!(summary.rows_read, 1);
assert_eq!(summary.malformed_lines, 1);
let strict = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
let err = scan_json_from_reader(Cursor::new(data.as_ref()), &strict, |_| {})
.expect_err("strict mode must reject an incomplete trailing record");
assert!(
err.to_string()
.to_lowercase()
.contains("malformed json record")
);
}
#[test]
fn test_all_malformed_file_fails() {
let file = write_file("not-json\nalso-bad\n");
let config = JsonParserConfig::jsonl();
let err = analyze_json_file(file.path(), &config)
.expect_err("a file with no valid records must fail");
let message = err.to_string().to_lowercase();
assert!(message.contains("no valid json records"), "got: {message}");
}
#[test]
fn test_tolerant_file_reports_error_count() {
let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
let config = JsonParserConfig::jsonl();
let report = analyze_json_file(file.path(), &config).unwrap();
assert_eq!(report.execution.rows_processed, 2);
assert_eq!(report.execution.error_count, 1);
}
#[test]
fn test_strict_file_errors_on_malformed() {
let file = write_file("{\"x\":1}\nnot-json\n{\"x\":2}\n");
let config = JsonParserConfig::jsonl().with_error_policy(JsonErrorPolicy::Strict);
let err = analyze_json_file(file.path(), &config)
.expect_err("strict mode must reject the malformed record");
assert!(
err.to_string()
.to_lowercase()
.contains("malformed json record")
);
}
#[test]
fn test_analyze_json_with_large_leading_whitespace() {
let data = format!("{}[{{\"x\":1}}]", " ".repeat(10_000));
let cursor = Cursor::new(data.into_bytes());
let config = JsonParserConfig::default();
let (_profiles, _stats, rows, malformed, format) =
analyze_json_from_reader(cursor, &config).unwrap();
assert_eq!(format, FileFormat::Json);
assert_eq!(rows, 1);
assert_eq!(malformed, 0);
}
#[test]
fn test_analyze_json_array() {
let json = write_file(r#"[{"name":"Alice","age":25},{"name":"Bob","age":30}]"#);
let config = JsonParserConfig::default();
let report = analyze_json_file(json.path(), &config).unwrap();
let profiles = &report.column_profiles;
assert_eq!(profiles.len(), 2);
let names: Vec<&str> = profiles
.iter()
.map(|profile| profile.name.as_str())
.collect();
assert!(names.contains(&"name"));
assert!(names.contains(&"age"));
let age = profiles
.iter()
.find(|profile| profile.name == "age")
.unwrap();
assert_eq!(age.total_count, 2);
assert_eq!(age.null_count, 0);
}
#[test]
fn test_analyze_jsonl() {
let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n");
let config = JsonParserConfig::default();
let report = analyze_json_file(jsonl.path(), &config).unwrap();
let profiles = &report.column_profiles;
assert_eq!(profiles.len(), 1);
assert_eq!(profiles[0].name, "x");
assert_eq!(profiles[0].total_count, 3);
}
#[test]
fn test_analyze_json_with_nulls() {
let json = write_file(r#"[{"a":"hello","b":1},{"a":null,"b":2},{"a":"world","b":null}]"#);
let config = JsonParserConfig::default();
let report = analyze_json_file(json.path(), &config).unwrap();
let profiles = &report.column_profiles;
let col_a = profiles.iter().find(|profile| profile.name == "a").unwrap();
assert_eq!(col_a.null_count, 1);
let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
assert_eq!(col_b.null_count, 1);
}
#[test]
fn test_analyze_json_with_missing_fields() {
let json = write_file(r#"[{"a":1,"b":2},{"a":3}]"#);
let config = JsonParserConfig::default();
let report = analyze_json_file(json.path(), &config).unwrap();
let profiles = &report.column_profiles;
let col_b = profiles.iter().find(|profile| profile.name == "b").unwrap();
assert_eq!(col_b.total_count, 2);
}
const UNSORTED_FIELDS: &str = r#"{"id":1,"amount":12.5,"active":true,"date":"2026-07-23"}"#;
const UNSORTED_ORDER: [&str; 4] = ["id", "amount", "active", "date"];
fn profile_names(report: &ProfileReport) -> Vec<&str> {
report
.column_profiles
.iter()
.map(|profile| profile.name.as_str())
.collect()
}
#[test]
fn test_json_column_order_follows_source_not_alphabet() {
let array = write_file(&format!("[{UNSORTED_FIELDS},{UNSORTED_FIELDS}]"));
let jsonl = write_file(&format!("{UNSORTED_FIELDS}\n{UNSORTED_FIELDS}\n"));
for file in [&array, &jsonl] {
let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
assert_eq!(profile_names(&report), UNSORTED_ORDER);
}
}
#[test]
fn test_json_column_order_appends_later_keys_in_first_seen_order() {
let data = b"{\"zulu\":1,\"mike\":2}\n{\"zulu\":3,\"mike\":4,\"delta\":5,\"alpha\":6}\n";
let file = write_bytes(data);
let report = analyze_json_file(file.path(), &JsonParserConfig::jsonl()).unwrap();
assert_eq!(profile_names(&report), ["zulu", "mike", "delta", "alpha"]);
}
#[test]
fn test_json_column_order_survives_report_serialization() {
let file = write_file(&format!("[{UNSORTED_FIELDS}]"));
let report = analyze_json_file(file.path(), &JsonParserConfig::default()).unwrap();
let json = serde_json::to_string(&report).unwrap();
let restored: Value = serde_json::from_str(&json).unwrap();
let names: Vec<&str> = restored["column_profiles"]
.as_array()
.unwrap()
.iter()
.map(|profile| profile["name"].as_str().unwrap())
.collect();
assert_eq!(names, UNSORTED_ORDER);
}
#[test]
fn test_analyze_json_backfills_columns_discovered_after_first_row() {
let json = write_file(r#"[{"a":1},{"a":2,"b":3}]"#);
let config = JsonParserConfig::default();
let report = analyze_json_file(json.path(), &config).unwrap();
let col_b = report
.column_profiles
.iter()
.find(|profile| profile.name == "b")
.unwrap();
assert_eq!(col_b.total_count, 2);
assert_eq!(col_b.null_count, 1);
assert_eq!(col_b.unique_count, Some(1));
}
#[test]
fn test_analyze_json_empty_array() {
let json = write_file("[]");
let config = JsonParserConfig::default();
let report = analyze_json_file(json.path(), &config).unwrap();
assert!(report.column_profiles.is_empty());
assert_eq!(report.quality_score(), None);
}
#[test]
fn test_analyze_json_malformed_returns_error() {
let json = write_file("this is entirely invalid json");
let config = JsonParserConfig::default();
let err = analyze_json_file(json.path(), &config)
.expect_err("malformed JSON should return an error");
let message = err.to_string().to_lowercase();
assert!(message.contains("malformed") && message.contains("json"));
}
#[test]
fn test_analyze_json_file_detects_format() {
let json_array = write_file(r#"[{"x":1}]"#);
let config = JsonParserConfig::default();
let report = analyze_json_file(json_array.path(), &config).unwrap();
assert!(matches!(
report.data_source,
DataSource::File {
format: FileFormat::Json,
..
}
));
let jsonl = write_file("{\"x\":1}\n{\"x\":2}\n");
let report = analyze_json_file(jsonl.path(), &config).unwrap();
assert!(matches!(
report.data_source,
DataSource::File {
format: FileFormat::Jsonl,
..
}
));
}
#[test]
fn test_bom_prefixed_file_preserves_source_size() {
let data = b"\xEF\xBB\xBF[{\"x\":1}]";
let json = write_bytes(data);
let report = analyze_json_file(json.path(), &JsonParserConfig::json_document()).unwrap();
assert_eq!(report.execution.rows_processed, 1);
assert!(matches!(
report.data_source,
DataSource::File {
format: FileFormat::Json,
size_bytes,
..
} if size_bytes == data.len() as u64
));
}
#[test]
fn test_analyze_json_file_empty() {
let json = write_file("");
let config = JsonParserConfig::default();
let report = analyze_json_file(json.path(), &config).unwrap();
assert_eq!(report.execution.rows_processed, 0);
assert!(report.column_profiles.is_empty());
}
#[test]
fn test_analyze_json_boolean_and_nested() {
let json =
write_file(r#"[{"flag":true,"nested":{"a":1}},{"flag":false,"nested":{"b":2}}]"#);
let config = JsonParserConfig::default();
let report = analyze_json_file(json.path(), &config).unwrap();
let profiles = &report.column_profiles;
let flag = profiles
.iter()
.find(|profile| profile.name == "flag")
.unwrap();
assert_eq!(flag.total_count, 2);
let nested = profiles
.iter()
.find(|profile| profile.name == "nested")
.unwrap();
assert_eq!(nested.total_count, 2);
}
#[test]
fn test_single_root_object_compact_yields_one_row() {
let data = br#"{"type":"FeatureCollection","features":[1,2,3]}"#;
let cursor = Cursor::new(data.as_ref());
let config = JsonParserConfig::default();
let (profiles, _stats, rows, malformed, _format) =
analyze_json_from_reader(cursor, &config).unwrap();
assert_eq!(rows, 1);
assert_eq!(malformed, 0);
assert_eq!(profiles.len(), 2);
let names: Vec<&str> = profiles
.iter()
.map(|profile| profile.name.as_str())
.collect();
assert!(names.contains(&"type"));
assert!(names.contains(&"features"));
}
#[test]
fn test_jsonl_multi_object_still_works() {
let data = b"{\"x\":1}\n{\"x\":2}\n{\"x\":3}\n";
let cursor = Cursor::new(data.as_ref());
let config = JsonParserConfig::default();
let (_profiles, _stats, rows, _malformed, format) =
analyze_json_from_reader(cursor, &config).unwrap();
assert_eq!(format, FileFormat::Jsonl);
assert_eq!(rows, 3);
}
#[test]
fn test_single_root_object_via_analyze_json_file() {
let file =
write_json_file(r#"{"type":"FeatureCollection","features":[{"id":1},{"id":2}]}"#);
let config = JsonParserConfig::default();
let report = analyze_json_file(file.path(), &config).unwrap();
assert_eq!(report.execution.rows_processed, 1);
assert_eq!(report.column_profiles.len(), 2);
}
#[test]
fn test_single_root_object_pretty_printed_yields_one_row() {
let data = br#"{
"type": "FeatureCollection",
"features": [
{"id": 1},
{"id": 2}
]
}"#;
let cursor = Cursor::new(data.as_ref());
let config = JsonParserConfig::json_document();
let (profiles, _stats, rows, malformed, format) =
analyze_json_from_reader(cursor, &config).unwrap();
assert_eq!(format, FileFormat::Json);
assert_eq!(rows, 1);
assert_eq!(malformed, 0);
assert_eq!(profiles.len(), 2);
}
#[test]
fn test_single_root_object_pretty_printed_via_analyze_json_file() {
let file = write_json_file(
"{\n \"type\": \"FeatureCollection\",\n \"features\": [\n {\"id\": 1},\n {\"id\": 2}\n ]\n}\n",
);
let config = JsonParserConfig::default();
let report = analyze_json_file(file.path(), &config).unwrap();
assert_eq!(report.execution.rows_processed, 1);
assert_eq!(report.column_profiles.len(), 2);
}
}