use super::*;
fn valid(input: &str, depth: Option<usize>) -> bool {
let memory = MemoryBudget::new(1024 * 1024);
let cancellation = CancellationToken::new();
let accepted = {
let mut reader = JsonReader::new(input, &memory, &cancellation);
reader.depth_limit = depth;
loop {
match reader.next_event() {
Ok(Some(_)) => {}
Ok(None) => break true,
Err(JsonReadError::InvalidJson) => break false,
Err(error) => panic!("unexpected control failure: {error}"),
}
}
};
assert_eq!(memory.used(), 0);
accepted
}
#[test]
fn grammar_matches_serde_for_scalars_nested_fields_and_malformed_delimiters() {
let tokens = [
"null",
"true",
"false",
"0",
"-0",
"18446744073709551616",
"1e400",
"-1.25E-10",
"\"νκΈπ\"",
"\"a\\n\\u0000\\uD83D\\uDE03\"",
"[]",
"{}",
"[1,{\"a\":true}]",
"{\"a\":1,\"a\":2}",
"",
"+1",
"01",
"-01",
".1",
"1.",
"1e",
"1e+",
"--1",
"tru",
"True",
"[1,]",
"[,1]",
"{,}",
"{\"a\",1}",
"{\"a\":1,}",
"{1:2}",
"[}",
"\"unterminated",
"\"\\x\"",
"\"\\uD800\"",
"\"\\uDC00\"",
"\"\\uD800\\u0041\"",
"\"\\uZZZZ\"",
"\"\n\"",
];
for token in tokens {
for input in [
token.to_owned(),
format!(" \n{token}\t "),
format!("[{token}]"),
format!("{{\"v\":{token}}}"),
format!("{token} null"),
] {
assert_eq!(
valid(&input, None),
serde_json::from_str::<serde_json::Value>(&input).is_ok(),
"{input:?}"
);
}
}
}
#[test]
fn borrowed_events_preserve_token_offsets_and_duplicate_key_order() {
let input = r#" { "a": [true, "\u0062"], "a": -1e+2 } "#;
let memory = MemoryBudget::new(4096);
let cancellation = CancellationToken::new();
let mut reader = JsonReader::new(input, &memory, &cancellation);
let mut events = Vec::new();
while let Some(event) = reader.next_event().unwrap() {
let raw = &input[event.range.clone()];
match event.token {
JsonToken::Number(text) => {
assert_eq!(text.as_ptr(), raw.as_ptr());
assert_eq!(text, raw);
}
JsonToken::String(encoded) | JsonToken::Key(encoded) => {
assert_eq!(encoded.as_ptr(), raw.as_ptr());
assert_eq!(encoded, raw.as_bytes());
}
_ => {}
}
events.push(event.token);
}
assert_eq!(
events,
vec![
JsonToken::StartObject,
JsonToken::Key(br#""a""#),
JsonToken::StartArray,
JsonToken::Bool(true),
JsonToken::String(br#""\u0062""#),
JsonToken::EndArray,
JsonToken::Key(br#""a""#),
JsonToken::Number("-1e+2"),
JsonToken::EndObject
]
);
drop(reader);
assert_eq!(memory.used(), 0);
}
#[test]
fn configurable_nesting_matches_values_without_restricting_jsonb_syntax() {
for depth in 1..=130 {
let input = format!("{}0{}", "[".repeat(depth), "]".repeat(depth));
assert_eq!(
valid(&input, Some(127)),
serde_json::from_str::<serde_json::Value>(&input).is_ok(),
"depth {depth}"
);
assert!(valid(&input, None));
}
let input = format!("{}0{}", "[".repeat(4096), "]".repeat(4096));
assert!(valid(&input, None));
}
#[test]
fn nesting_reserves_before_growth_and_releases_after_failure() {
let memory = MemoryBudget::new(0);
let cancellation = CancellationToken::new();
let mut scalar = JsonReader::new("123456789012345678901234567890", &memory, &cancellation);
assert!(matches!(
scalar.next_event().unwrap().unwrap().token,
JsonToken::Number(_)
));
assert!(scalar.next_event().unwrap().is_none());
let mut container = JsonReader::new("[]", &memory, &cancellation);
assert!(matches!(
container.next_event(),
Err(JsonReadError::Memory(MemoryError::Limit { .. }))
));
assert_eq!(memory.used(), 0);
}
#[test]
fn cancellation_keeps_prior_retained_strings_and_releases_reader_scratch() {
let memory = MemoryBudget::new(4096);
let cancellation = CancellationToken::new();
let retained = decode_json_string(r#""prior""#, &memory, &cancellation).unwrap();
let before = memory.used();
let mut reader = JsonReader::new("[1,2]", &memory, &cancellation);
reader.next_event().unwrap();
assert!(memory.used() > before);
cancellation.cancel();
assert!(matches!(
reader.next_event(),
Err(JsonReadError::Cancelled(_))
));
assert!(matches!(
decode_json_string(r#""next""#, &memory, &cancellation),
Err(JsonReadError::Cancelled(_))
));
drop(reader);
assert_eq!(memory.used(), before);
assert_eq!(&**retained, "prior");
drop(retained);
assert_eq!(memory.used(), 0);
}
#[test]
fn string_quota_precedes_decoding_and_final_lease_charges_owned_capacity() {
let cancellation = CancellationToken::new();
for encoded in [r#""plain""#, r#""a\n\uD83D\uDE03\u0000""#, r#""""#] {
let rejected = MemoryBudget::new(0);
if encoded == r#""""# {
assert!(decode_json_string(encoded, &rejected, &cancellation)
.unwrap()
.is_empty());
} else {
assert!(matches!(
decode_json_string(encoded, &rejected, &cancellation),
Err(JsonReadError::Memory(MemoryError::Limit { .. }))
));
}
assert_eq!(rejected.used(), 0);
let memory = MemoryBudget::new(4096);
let value = decode_json_string(encoded, &memory, &cancellation).unwrap();
assert_eq!(&**value, serde_json::from_str::<String>(encoded).unwrap());
assert_eq!(memory.used(), value.capacity());
assert_eq!(value.reserved_bytes(), value.capacity());
drop(value);
assert_eq!(memory.used(), 0);
}
let memory = MemoryBudget::new(4096);
assert!(matches!(
decode_json_string(r#""\uD800""#, &memory, &cancellation),
Err(JsonReadError::InvalidJson)
));
assert_eq!(memory.used(), 0);
}
#[test]
fn ignored_escape_scanning_matches_serde_without_weakening_owned_strings() {
use serde::Deserialize;
let cancellation = CancellationToken::new();
for input in [
r#"{"unused":"\uD800"}"#,
r#"{"unused":{"\uDC00":["\uD800\u0041"]}}"#,
r#"{"unused":"\uD800\uDC00"}"#,
r#"{"unused":"\uZZZZ"}"#,
r#"{"unused":"\q"}"#,
"{\"unused\":\"\n\"}",
] {
let memory = MemoryBudget::new(4096);
let result = {
let mut reader =
JsonReader::new(input, &memory, &cancellation).with_ignored_string_escapes();
loop {
match reader.next_event() {
Ok(Some(_)) => {}
Ok(None) => break true,
Err(JsonReadError::InvalidJson) => break false,
Err(error) => panic!("unexpected control failure: {error}"),
}
}
};
let mut reference = serde_json::Deserializer::from_str(input);
let expected = serde::de::IgnoredAny::deserialize(&mut reference)
.and_then(|_| reference.end())
.is_ok();
assert_eq!(result, expected, "{input}");
assert_eq!(memory.used(), 0);
}
assert!(!valid(r#""\uD800""#, None));
assert!(matches!(
decode_json_string(r#""\uD800""#, &MemoryBudget::new(4096), &cancellation),
Err(JsonReadError::InvalidJson)
));
}
#[test]
fn borrowed_byte_tokens_preserve_ignored_input_without_lossy_utf8_conversion() {
use serde::Deserialize;
let cancellation = CancellationToken::new();
for input in [
b"{\"unused\":\"\xff\"}".as_slice(),
b"{\"unused\":{\"\xff\":1}}",
b"{\"unused\":\"\xc0\xaf\"}",
b"{\"unused\":\"\xff\\q\"}",
b"{\"unused\":\"\xff\n\"}",
b"{\"unused\":\xff}",
] {
let memory = MemoryBudget::new(4096);
let result = {
let mut reader =
JsonReader::from_slice(input, &memory, &cancellation).with_ignored_string_escapes();
loop {
match reader.next_event() {
Ok(Some(event)) => {
if let JsonToken::String(bytes) | JsonToken::Key(bytes) = event.token {
assert_eq!(bytes.as_ptr(), input[event.range.clone()].as_ptr());
assert_eq!(bytes, &input[event.range]);
}
}
Ok(None) => break true,
Err(JsonReadError::InvalidJson) => break false,
Err(error) => panic!("unexpected control failure: {error}"),
}
}
};
let mut reference = serde_json::Deserializer::from_slice(input);
let expected = serde::de::IgnoredAny::deserialize(&mut reference)
.and_then(|_| reference.end())
.is_ok();
assert_eq!(result, expected, "{input:?}");
assert_eq!(memory.used(), 0);
}
let memory = MemoryBudget::new(4096);
let mut strict = JsonReader::from_slice(b"\"\xff\"", &memory, &cancellation);
assert!(matches!(
strict.next_event(),
Err(JsonReadError::InvalidJson)
));
assert!(matches!(
decode_json_string(b"\"\xff\"", &memory, &cancellation),
Err(JsonReadError::InvalidJson)
));
assert_eq!(memory.used(), 0);
}
#[test]
fn controlled_strings_match_serde_escapes_and_preserve_both_cancellation_owners() {
let memory = MemoryBudget::new(4096);
let original = CancellationToken::new();
let invoking = CancellationToken::new();
let control = ProductionControl::new(&memory, &original, &invoking);
for text in [
r#""\"\\\/\b\f\n\r\t\u0000\uD83D\uDE03νκΈ""#,
" \"literal\" ",
] {
let value = decode_json_string_with_control(text, &control).unwrap();
assert_eq!(&**value, serde_json::from_str::<String>(text).unwrap());
assert_eq!(value.reserved_bytes(), value.capacity());
drop(value);
assert_eq!(memory.used(), 0);
}
for cancel_original in [false, true] {
let prior = decode_json_string_with_control(r#""prior""#, &control).unwrap();
let before = memory.used();
let mut reader = JsonReader::with_control("[1,2]", &control);
reader.next_event().unwrap();
if cancel_original {
original.cancel();
} else {
invoking.cancel();
}
assert!(matches!(
reader.next_event(),
Err(JsonReadError::Cancelled(_))
));
assert!(matches!(
decode_json_string_with_control(r#""next""#, &control),
Err(JsonReadError::Cancelled(_))
));
drop(reader);
assert_eq!(memory.used(), before);
assert_eq!(&**prior, "prior");
original.reset();
invoking.reset();
drop(prior);
assert_eq!(memory.used(), 0);
}
}