use std::io::{BufRead, Read};
use dataprof_core::DataProfilerError;
use serde::Deserialize;
use serde_json::Value;
use crate::JsonObject;
pub enum JsonRecord {
Object(JsonObject),
NonObject(&'static str),
Malformed(serde_json::Error),
WideInteger,
}
pub fn read_json_record<R: BufRead>(
reader: &mut R,
first: u8,
) -> Result<JsonRecord, DataProfilerError> {
if first == b'-' || first.is_ascii_digit() {
let token = read_number_token(reader)?;
return Ok(match serde_json::from_str::<Value>(&token) {
Ok(_) => JsonRecord::NonObject("number"),
Err(err) => JsonRecord::Malformed(err),
});
}
let mut recorded = RecordingReader {
inner: reader,
bytes: Vec::new(),
};
let parsed = {
let mut deserializer = serde_json::Deserializer::from_reader(&mut recorded);
Value::deserialize(&mut deserializer)
};
Ok(match parsed {
Ok(Value::Object(obj)) => object_record(obj, &recorded.bytes),
Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
Err(err) if err.is_io() => {
let message = format!("JSON record read failed: {err}");
return Err(DataProfilerError::json_parsing_with_source(
message,
std::io::Error::from(err),
));
}
Err(err) => JsonRecord::Malformed(err),
})
}
struct RecordingReader<'a, R> {
inner: &'a mut R,
bytes: Vec<u8>,
}
impl<R: Read> Read for RecordingReader<'_, R> {
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
let read = self.inner.read(buffer)?;
self.bytes.extend_from_slice(&buffer[..read]);
Ok(read)
}
}
pub fn may_hold_wide_integer(value: &Value) -> bool {
match value {
Value::Number(number) => {
number.as_i64().is_none()
&& number.as_u64().is_none()
&& number
.as_f64()
.is_some_and(|float| float.abs() >= 2f64.powi(63))
}
Value::Array(items) => items.iter().any(may_hold_wide_integer),
Value::Object(fields) => fields.values().any(may_hold_wide_integer),
_ => false,
}
}
pub fn has_wide_integer_literal(raw: &[u8]) -> bool {
integer_literals(raw).any(|(_, token)| is_wide_integer(token))
}
fn integer_literals(raw: &[u8]) -> impl Iterator<Item = (usize, &[u8])> {
let mut index = 0;
std::iter::from_fn(move || {
while index < raw.len() {
match raw[index] {
b'"' => {
index += 1;
while index < raw.len() && raw[index] != b'"' {
index += if raw[index] == b'\\' { 2 } else { 1 };
}
index += 1;
}
byte if byte == b'-' || byte.is_ascii_digit() => {
let start = index;
while index < raw.len() && is_json_number_byte(raw[index]) {
index += 1;
}
let token = &raw[start..index];
if !token.iter().any(|byte| matches!(byte, b'.' | b'e' | b'E')) {
return Some((start, token));
}
}
_ => index += 1,
}
}
None
})
}
fn is_wide_integer(token: &[u8]) -> bool {
let digits = token.len() - usize::from(token.first() == Some(&b'-'));
if digits < 19 {
return false;
}
let text = std::str::from_utf8(token).expect("JSON number bytes are ASCII");
text.parse::<i64>().is_err() && text.parse::<u64>().is_err()
}
pub fn object_record(obj: JsonObject, raw: &[u8]) -> JsonRecord {
let mut negative_zeros = Vec::new();
for (start, token) in integer_literals(raw) {
if is_wide_integer(token) {
return JsonRecord::WideInteger;
}
if token == b"-0" {
negative_zeros.push(start);
}
}
if negative_zeros.is_empty() {
return JsonRecord::Object(obj);
}
let mut unsigned = raw.to_vec();
for start in negative_zeros {
unsigned[start] = b' ';
}
JsonRecord::Object(
serde_json::from_slice::<JsonObject>(&unsigned)
.expect("an object with its `-0` signs blanked is still an object"),
)
}
pub fn wide_integer_record_error(location: &str) -> DataProfilerError {
DataProfilerError::json_parsing_error(&format!(
"JSON record {location} holds an integer outside the 64-bit range, which would be rounded to a float and could merge distinct values; write such integers as strings to profile them"
))
}
fn read_number_token<R: BufRead>(reader: &mut R) -> Result<String, DataProfilerError> {
skip_ascii_whitespace(reader)?;
let mut token = String::new();
loop {
let (consume, token_ended) = {
let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
if buf.is_empty() {
break;
}
let consume = buf
.iter()
.take_while(|byte| is_json_number_byte(**byte))
.count();
token.push_str(
std::str::from_utf8(&buf[..consume]).expect("JSON number bytes are ASCII"),
);
(consume, consume < buf.len())
};
reader.consume(consume);
if token_ended {
break;
}
}
Ok(token)
}
fn is_json_number_byte(byte: u8) -> bool {
byte.is_ascii_digit() || matches!(byte, b'-' | b'+' | b'.' | b'e' | b'E')
}
pub(crate) fn skip_ascii_whitespace<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
loop {
let (consume, done) = {
let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
if buf.is_empty() {
return Ok(());
}
let consume = buf
.iter()
.take_while(|byte| byte.is_ascii_whitespace())
.count();
(consume, consume < buf.len())
};
reader.consume(consume);
if done {
return Ok(());
}
}
}
pub fn json_value_text(value: &Value) -> String {
match value {
Value::Null => String::new(),
Value::Bool(boolean) => boolean.to_string(),
Value::Number(number) => number.to_string(),
Value::String(string) => string.to_string(),
Value::Array(_) | Value::Object(_) => {
let mut text = String::new();
write_canonical_json(value, &mut text);
text
}
}
}
fn write_canonical_json(value: &Value, out: &mut String) {
match value {
Value::Array(items) => {
out.push('[');
for (index, item) in items.iter().enumerate() {
if index > 0 {
out.push(',');
}
write_canonical_json(item, out);
}
out.push(']');
}
Value::Object(fields) => {
let mut entries: Vec<_> = fields.iter().collect();
entries.sort_unstable_by_key(|(key, _)| *key);
out.push('{');
for (index, (key, field)) in entries.into_iter().enumerate() {
if index > 0 {
out.push(',');
}
out.push_str(&scalar_json(key));
out.push(':');
write_canonical_json(field, out);
}
out.push('}');
}
scalar => out.push_str(&scalar_json(scalar)),
}
}
fn scalar_json<T: serde::Serialize + ?Sized>(value: &T) -> String {
serde_json::to_string(value).expect("re-serializing a parsed JSON value cannot fail")
}
pub fn json_value_kind(value: &Value) -> &'static str {
match value {
Value::Null => "null",
Value::Bool(_) => "boolean",
Value::Number(_) => "number",
Value::String(_) => "string",
Value::Array(_) => "array",
Value::Object(_) => "object",
}
}
pub fn non_object_record_error(kind: &str, position: usize) -> DataProfilerError {
DataProfilerError::json_parsing_error(&format!(
"non-object JSON record at position {position}: \
expected an object with fields to profile, found {kind}"
))
}
pub fn malformed_array_error(message: &str) -> DataProfilerError {
DataProfilerError::json_parsing_error(&format!("malformed JSON array: {message}"))
}
pub fn json_document_error(err: serde_json::Error) -> DataProfilerError {
let message = format!(
"malformed JSON document: {err}. A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
);
DataProfilerError::json_parsing_with_source(message, err)
}
pub fn drain_to_end<R: BufRead>(reader: &mut R) -> Result<usize, DataProfilerError> {
let mut consumed = 0;
loop {
let available = reader.fill_buf().map_err(DataProfilerError::from)?.len();
if available == 0 {
return Ok(consumed);
}
reader.consume(available);
consumed += available;
}
}
pub fn read_jsonl_line(line: &str) -> JsonRecord {
match serde_json::from_str::<Value>(line.trim()) {
Ok(Value::Object(obj)) => object_record(obj, line.as_bytes()),
Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
Err(_) if has_wide_integer_literal(line.as_bytes()) => JsonRecord::WideInteger,
Err(err) => JsonRecord::Malformed(err),
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::error::Error as _;
use std::io::{self, BufReader, Cursor, Read};
#[derive(Debug)]
struct LocalReadFailure;
impl std::fmt::Display for LocalReadFailure {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("local read failure")
}
}
impl std::error::Error for LocalReadFailure {}
impl Read for LocalReadFailure {
fn read(&mut self, _buffer: &mut [u8]) -> io::Result<usize> {
Err(io::Error::new(io::ErrorKind::PermissionDenied, Self))
}
}
#[test]
fn integer_negative_zero_reads_as_zero_and_float_negative_zero_keeps_its_sign() {
let text = r#"{"v":-0,"w":[-0,{"k":-0}],"s":"-0","-0":1,"f":-0.0,"e":-0e0}"#;
let from_line = match read_jsonl_line(text) {
JsonRecord::Object(obj) => obj,
_ => panic!("a JSONL object"),
};
let mut reader = Cursor::new(text.as_bytes());
let from_array = match read_json_record(&mut reader, b'{').unwrap() {
JsonRecord::Object(obj) => obj,
_ => panic!("an array element object"),
};
for obj in [from_line, from_array] {
assert_eq!(json_value_text(&obj["v"]), "0");
assert_eq!(json_value_text(&obj["w"]), r#"[0,{"k":0}]"#);
assert_eq!(json_value_text(&obj["s"]), "-0");
assert!(obj.contains_key("-0"));
assert_eq!(json_value_text(&obj["f"]), "-0.0");
assert_eq!(json_value_text(&obj["e"]), "-0.0");
}
}
#[test]
fn json_record_read_failure_keeps_the_original_io_source() {
let mut reader = BufReader::new(Cursor::new(b"{\"x\":\"").chain(LocalReadFailure));
let error = read_json_record(&mut reader, b'{')
.err()
.expect("a read failure must abort before a caller can skip a malformed record");
assert!(
error
.to_string()
.contains("JSON record read failed: local read failure")
);
let source = error
.source()
.and_then(|source| source.downcast_ref::<io::Error>())
.expect("the original I/O error must survive serde's wrapper");
assert_eq!(source.kind(), io::ErrorKind::PermissionDenied);
assert!(source.get_ref().unwrap().is::<LocalReadFailure>());
}
}