use std::{
cmp::Ordering,
fmt::{Display, Write},
};
use crate::{
args::{ArgValues, FromArgs},
bytecode::{ContainsVM, VM},
defer_drop_mut, defer_drop_vm, defer_drop_vm_mut,
exception_private::{ExcType, RunResult},
heap::{ContainsHeap, Heap, HeapData, HeapGuard, HeapId, HeapRead, HeapReadOutput},
resource::ResourceTracker,
sorting::{apply_permutation, sort_indices},
types::{Dict, PyTrait, long_int::check_bigint_str_digits_limit, str::allocate_string},
value::Value,
};
struct JsonDumpsConfig {
indent: Option<String>,
item_separator: String,
key_separator: String,
flags: u8,
}
impl Default for JsonDumpsConfig {
fn default() -> Self {
Self {
indent: None,
item_separator: ", ".to_owned(),
key_separator: ": ".to_owned(),
flags: Self::ENSURE_ASCII | Self::ALLOW_NAN,
}
}
}
impl JsonDumpsConfig {
const SORT_KEYS: u8 = 1 << 0;
const ENSURE_ASCII: u8 = 1 << 1;
const ALLOW_NAN: u8 = 1 << 2;
const SKIPKEYS: u8 = 1 << 3;
fn sort_keys(&self) -> bool {
self.flags & Self::SORT_KEYS != 0
}
fn ensure_ascii(&self) -> bool {
self.flags & Self::ENSURE_ASCII != 0
}
fn allow_nan(&self) -> bool {
self.flags & Self::ALLOW_NAN != 0
}
fn skipkeys(&self) -> bool {
self.flags & Self::SKIPKEYS != 0
}
fn from_macro_args(args: JsonDumpsArgs, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult<(Value, Self)> {
let JsonDumpsArgs {
obj,
indent,
sort_keys,
ensure_ascii,
allow_nan,
separators,
skipkeys,
} = args;
let mut obj_guard = HeapGuard::new(obj, vm);
let vm = obj_guard.heap();
let mut config = Self::default();
let indent = parse_indent_value(indent, vm)?;
config.indent = indent;
config.flags = apply_bool_flag(config.flags, Self::SORT_KEYS, sort_keys, vm);
config.flags = apply_bool_flag(config.flags, Self::ENSURE_ASCII, ensure_ascii, vm);
config.flags = apply_bool_flag(config.flags, Self::ALLOW_NAN, allow_nan, vm);
config.flags = apply_bool_flag(config.flags, Self::SKIPKEYS, skipkeys, vm);
let separators_were_set = if let Some((item, key)) = parse_separators_value(separators, vm)? {
config.item_separator = item;
config.key_separator = key;
true
} else {
false
};
if config.indent.is_some() && !separators_were_set {
",".clone_into(&mut config.item_separator);
": ".clone_into(&mut config.key_separator);
}
Ok((obj_guard.into_inner(), config))
}
}
pub(super) fn call_dumps(vm: &mut VM<'_, impl ResourceTracker>, args: ArgValues) -> RunResult<Value> {
let macro_args = JsonDumpsArgs::from_args(args, vm)?;
let (obj, config) = JsonDumpsConfig::from_macro_args(macro_args, vm)?;
let mut obj_guard = HeapGuard::new(obj, vm);
let mut output = String::new();
let mut active_containers = Vec::new();
{
let (obj, vm) = obj_guard.as_parts_mut();
let mut encoder = Encoder {
out: &mut output,
config: &config,
active_containers: &mut active_containers,
vm,
};
encoder.serialize_value(obj, 0)?;
}
let (obj, vm) = obj_guard.into_parts();
obj.drop_with_heap(vm);
Ok(allocate_string(output, vm.heap)?)
}
#[derive(FromArgs)]
#[from_args(name = "dumps", style = def, kwarg_error_name = "JSONEncoder.__init__")]
struct JsonDumpsArgs {
obj: Value,
#[from_args(kw_only, default = Value::None)]
indent: Value,
#[from_args(kw_only, default = Value::Bool(false))]
sort_keys: Value,
#[from_args(kw_only, default = Value::Bool(true))]
ensure_ascii: Value,
#[from_args(kw_only, default = Value::Bool(true))]
allow_nan: Value,
#[from_args(kw_only, default = Value::None)]
separators: Value,
#[from_args(kw_only, default = Value::Bool(false))]
skipkeys: Value,
}
fn apply_bool_flag(flags: u8, bit: u8, value: Value, vm: &mut VM<'_, impl ResourceTracker>) -> u8 {
let new_flags = if value.py_bool(vm) { flags | bit } else { flags & !bit };
value.drop_with_heap(vm);
new_flags
}
fn parse_indent_value(value: Value, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult<Option<String>> {
let mut value_guard = HeapGuard::new(value, vm);
let (value, vm) = value_guard.as_parts_mut();
match value {
Value::None => Ok(None),
Value::Bool(flag) => Ok(Some(" ".repeat(usize::from(*flag)))),
Value::Int(count) => spaces_from_indent_count(*count),
Value::InternString(string_id) => Ok(Some(vm.interns.get_str(*string_id).to_owned())),
Value::Ref(heap_id) => match vm.heap.read(*heap_id) {
HeapReadOutput::Str(string) => Ok(Some(string.get(vm.heap).as_str().to_owned())),
HeapReadOutput::LongInt(long_int) => {
spaces_from_indent_count(long_int.get(vm.heap).to_i64().ok_or_else(ExcType::overflow_c_ssize_t)?)
}
_ => Err(ExcType::type_error("indent must be None, an integer or a string")),
},
_ => Err(ExcType::type_error("indent must be None, an integer or a string")),
}
}
fn spaces_from_indent_count(count: i64) -> RunResult<Option<String>> {
if count <= 0 {
Ok(Some(String::new()))
} else {
match usize::try_from(count) {
Ok(count) => Ok(Some(" ".repeat(count))),
Err(_) => Err(ExcType::overflow_c_ssize_t()),
}
}
}
fn parse_separators_value(value: Value, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult<Option<(String, String)>> {
let mut value_guard = HeapGuard::new(value, vm);
let (value, vm) = value_guard.as_parts_mut();
if matches!(value, Value::None) {
return Ok(None);
}
let pair = match value {
Value::Ref(heap_id) => match vm.heap.read(*heap_id) {
HeapReadOutput::Tuple(tuple) => {
let items = tuple.get(vm.heap).as_slice();
check_separators_length(items.len())?;
(
json_separator_to_string(&items[0], "item_separator", vm)?,
json_separator_to_string(&items[1], "key_separator", vm)?,
)
}
HeapReadOutput::List(list) => {
let items = list.get(vm.heap).as_slice();
check_separators_length(items.len())?;
(
json_separator_to_string(&items[0], "item_separator", vm)?,
json_separator_to_string(&items[1], "key_separator", vm)?,
)
}
_ => {
return Err(ExcType::type_error(format!(
"cannot unpack non-iterable {} object",
value.py_type_name(vm)
)));
}
},
_ => {
return Err(ExcType::type_error(format!(
"cannot unpack non-iterable {} object",
value.py_type_name(vm)
)));
}
};
Ok(Some(pair))
}
fn check_separators_length(len: usize) -> RunResult<()> {
match len.cmp(&2) {
Ordering::Greater => Err(ExcType::value_error(format!(
"too many values to unpack (expected 2, got {len})"
))),
Ordering::Less => Err(ExcType::value_error(format!(
"not enough values to unpack (expected 2, got {len})"
))),
Ordering::Equal => Ok(()),
}
}
fn json_separator_to_string(value: &Value, role: &str, vm: &VM<'_, impl ResourceTracker>) -> RunResult<String> {
let arg_num = if role == "item_separator" { 6 } else { 5 };
match value {
Value::InternString(string_id) => Ok(vm.interns.get_str(*string_id).to_owned()),
Value::Ref(heap_id) => match vm.heap.get(*heap_id) {
HeapData::Str(string) => Ok(string.as_str().to_owned()),
_ => Err(ExcType::type_error(format!(
"make_encoder() argument {arg_num} must be str, not {}",
value.py_type_name(vm)
))),
},
_ => Err(ExcType::type_error(format!(
"make_encoder() argument {arg_num} must be str, not {}",
value.py_type_name(vm)
))),
}
}
struct Encoder<'a, 'h, R: ResourceTracker> {
out: &'a mut String,
config: &'a JsonDumpsConfig,
active_containers: &'a mut Vec<HeapId>,
vm: &'a mut VM<'h, R>,
}
impl<R: ResourceTracker> ContainsHeap for Encoder<'_, '_, R> {
type ResourceTracker = R;
fn heap(&self) -> &Heap<R> {
self.vm.heap()
}
fn heap_mut(&mut self) -> &mut Heap<R> {
self.vm.heap_mut()
}
}
impl<'h, R: ResourceTracker> ContainsVM<'h> for Encoder<'_, 'h, R> {
type Tracker = R;
fn vm(&mut self) -> &mut VM<'h, Self::Tracker> {
self.vm
}
}
impl<'h, R: ResourceTracker> Encoder<'_, 'h, R> {
fn serialize_value(&mut self, value: &Value, depth: usize) -> RunResult<()> {
match value {
Value::None => {
self.out.push_str("null");
Ok(())
}
Value::Bool(true) => {
self.out.push_str("true");
Ok(())
}
Value::Bool(false) => {
self.out.push_str("false");
Ok(())
}
Value::Int(value) => {
write!(self.out, "{value}").expect("writing to String cannot fail");
Ok(())
}
Value::Float(value) => serialize_float(*value, self.out, self.config),
Value::InternString(string_id) => {
write_json_string(
self.vm.interns.get_str(*string_id),
self.out,
self.config.ensure_ascii(),
);
Ok(())
}
Value::InternLongInt(long_int_id) => {
let value = self.vm.interns.get_long_int(*long_int_id);
check_bigint_str_digits_limit(value)?;
write!(self.out, "{value}").expect("writing to String cannot fail");
Ok(())
}
Value::Ref(heap_id) => match self.vm.heap.read(*heap_id) {
HeapReadOutput::Str(string) => {
write_json_string(string.get(self.vm.heap).as_str(), self.out, self.config.ensure_ascii());
Ok(())
}
HeapReadOutput::LongInt(long_int) => {
long_int.get(self.vm.heap).check_str_digits_limit()?;
write!(self.out, "{}", long_int.get(self.vm.heap).inner()).expect("writing to String cannot fail");
Ok(())
}
HeapReadOutput::List(list) => self.with_entered_container(*heap_id, |enc| {
let iter = list.iter(enc.vm)?;
defer_drop_vm_mut!(iter, enc);
enc.serialize_array(depth, |enc, depth| {
if let Some(item) = iter.next(enc.vm)? {
enc.serialize_value(item, depth)?;
Ok(true)
} else {
Ok(false)
}
})
}),
HeapReadOutput::Tuple(tuple) => self.with_entered_container(*heap_id, |enc| {
let iter = tuple.iter(enc.vm)?;
defer_drop_vm_mut!(iter, enc);
enc.serialize_array(depth, |enc, depth| {
if let Some(item) = iter.next(enc.vm)? {
enc.serialize_value(item, depth)?;
Ok(true)
} else {
Ok(false)
}
})
}),
HeapReadOutput::Dict(dict) => {
let entries = self.collect_dict_entries(&dict);
let this = self;
defer_drop_mut!(entries, this);
let token = this.vm.recursion_token()?;
defer_drop_vm!(token, this);
this.with_entered_container(*heap_id, |enc| enc.serialize_dict(entries, depth))
}
_ => Err(ExcType::json_not_serializable_error(&value.py_type_name(self.vm))),
},
_ => Err(ExcType::json_not_serializable_error(&value.py_type_name(self.vm))),
}
}
fn serialize_array(
&mut self,
depth: usize,
mut write_next: impl FnMut(&mut Self, usize) -> RunResult<bool>,
) -> RunResult<()> {
self.out.push('[');
let pretty = self.config.indent.is_some();
let mut wrote_any = false;
loop {
let prefix_start = self.out.len();
if wrote_any {
self.out.push_str(&self.config.item_separator);
}
if pretty {
self.out.push('\n');
write_indent(self.out, self.config, depth + 1);
}
let body_start = self.out.len();
if !write_next(self, depth + 1)? {
self.out.truncate(prefix_start);
break;
}
debug_assert!(
self.out.len() > body_start,
"write_next reported true but wrote nothing"
);
wrote_any = true;
}
if pretty && wrote_any {
self.out.push('\n');
write_indent(self.out, self.config, depth);
}
self.out.push(']');
Ok(())
}
fn collect_dict_entries(&mut self, dict: &HeapRead<'h, Dict>) -> Vec<(Value, Value)> {
dict.get(self.vm.heap)
.iter()
.map(|(k, v)| (k.clone_with_heap(self.vm.heap), v.clone_with_heap(self.vm.heap)))
.collect::<Vec<_>>()
}
fn serialize_dict(&mut self, entries: &mut Vec<(Value, Value)>, depth: usize) -> RunResult<()> {
if self.config.skipkeys() {
skip_disallowed_dict_keys(entries, self.vm);
} else if let Some((key, _)) = entries.iter().find(|(key, _)| !is_json_key_allowed(key, self.vm)) {
return Err(ExcType::json_invalid_key_error(&key.py_type_name(self.vm)));
}
if self.config.sort_keys() {
sort_dict_entries(entries, self.vm)?;
}
self.out.push('{');
let pretty = self.config.indent.is_some();
for (index, (key, value)) in entries.iter().enumerate() {
if index != 0 {
self.out.push_str(&self.config.item_separator);
}
if pretty {
self.out.push('\n');
write_indent(self.out, self.config, depth + 1);
}
write_json_key(key, self.out, self.config, self.vm)?;
self.out.push_str(&self.config.key_separator);
self.serialize_value(value, depth + 1)?;
}
if pretty && !entries.is_empty() {
self.out.push('\n');
write_indent(self.out, self.config, depth);
}
self.out.push('}');
Ok(())
}
fn with_entered_container<T>(
&mut self,
heap_id: HeapId,
f: impl FnOnce(&mut Self) -> RunResult<T>,
) -> RunResult<T> {
if self.active_containers.contains(&heap_id) {
return Err(ExcType::json_circular_reference_error());
}
self.active_containers.push(heap_id);
let result = f(self);
self.active_containers
.pop()
.expect("entered container missing from JSON serialization stack");
result
}
}
fn sort_dict_entries(entries: &mut Vec<(Value, Value)>, vm: &mut VM<'_, impl ResourceTracker>) -> RunResult<()> {
let mut indices: Vec<usize> = (0..entries.len()).collect();
let compare_values: Vec<Value> = entries.iter().map(|(key, _)| key.clone_with_heap(vm)).collect();
let mut compare_values_guard = HeapGuard::new(compare_values, vm);
let (compare_values, vm) = compare_values_guard.as_parts_mut();
sort_indices(&mut indices, compare_values.as_slice(), false, vm)?;
apply_permutation(entries.as_mut_slice(), &mut indices);
Ok(())
}
fn skip_disallowed_dict_keys(entries: &mut Vec<(Value, Value)>, vm: &mut VM<'_, impl ResourceTracker>) {
let mut write = 0;
for read in 0..entries.len() {
if is_json_key_allowed(&entries[read].0, vm) {
if write != read {
entries.swap(write, read);
}
write += 1;
}
}
for (key, value) in entries.drain(write..) {
key.drop_with_heap(vm);
value.drop_with_heap(vm);
}
}
fn is_json_key_allowed(value: &Value, vm: &VM<'_, impl ResourceTracker>) -> bool {
matches!(
value,
Value::None | Value::Bool(_) | Value::Int(_) | Value::Float(_) | Value::InternString(_)
) || matches!(value, Value::Ref(heap_id) if matches!(vm.heap.get(*heap_id), HeapData::Str(_) | HeapData::LongInt(_)))
}
fn write_json_key(
key: &Value,
out: &mut String,
config: &JsonDumpsConfig,
vm: &VM<'_, impl ResourceTracker>,
) -> RunResult<()> {
let ensure_ascii = config.ensure_ascii();
match key {
Value::None => write_json_ascii_key("null", out),
Value::Bool(true) => write_json_ascii_key("true", out),
Value::Bool(false) => write_json_ascii_key("false", out),
Value::Int(value) => write_json_display_key(value, out),
Value::Float(value) => {
serialize_float_key(*value, out, config)?;
}
Value::InternString(string_id) => write_json_string(vm.interns.get_str(*string_id), out, ensure_ascii),
Value::Ref(heap_id) => match vm.heap.get(*heap_id) {
HeapData::Str(string) => write_json_string(string.as_str(), out, ensure_ascii),
HeapData::LongInt(long_int) => {
long_int.check_str_digits_limit()?;
write_json_display_key(long_int.inner(), out);
}
_ => return Err(ExcType::json_invalid_key_error(&key.py_type_name(vm))),
},
_ => return Err(ExcType::json_invalid_key_error(&key.py_type_name(vm))),
}
Ok(())
}
fn write_json_ascii_key(value: &str, out: &mut String) {
out.push('"');
out.push_str(value);
out.push('"');
}
fn write_json_display_key(value: impl Display, out: &mut String) {
out.push('"');
write!(out, "{value}").expect("writing to String cannot fail");
out.push('"');
}
fn serialize_float_key(value: f64, out: &mut String, config: &JsonDumpsConfig) -> RunResult<()> {
out.push('"');
if value.is_nan() {
if !config.allow_nan() {
return Err(ExcType::json_nan_error("nan"));
}
out.push_str("NaN");
} else if value == f64::INFINITY {
if !config.allow_nan() {
return Err(ExcType::json_nan_error("inf"));
}
out.push_str("Infinity");
} else if value == f64::NEG_INFINITY {
if !config.allow_nan() {
return Err(ExcType::json_nan_error("-inf"));
}
out.push_str("-Infinity");
} else {
write_json_float_text(value, out);
}
out.push('"');
Ok(())
}
fn serialize_float(value: f64, out: &mut String, config: &JsonDumpsConfig) -> RunResult<()> {
if value.is_nan() {
if config.allow_nan() {
out.push_str("NaN");
Ok(())
} else {
Err(ExcType::json_nan_error("nan"))
}
} else if value == f64::INFINITY {
if config.allow_nan() {
out.push_str("Infinity");
Ok(())
} else {
Err(ExcType::json_nan_error("inf"))
}
} else if value == f64::NEG_INFINITY {
if config.allow_nan() {
out.push_str("-Infinity");
Ok(())
} else {
Err(ExcType::json_nan_error("-inf"))
}
} else {
write_json_float_text(value, out);
Ok(())
}
}
fn write_json_float_text(value: f64, out: &mut String) {
let abs = value.abs();
if abs != 0.0 && !(1e-4..1e16).contains(&abs) {
let start = out.len();
write!(out, "{value:e}").expect("writing to String cannot fail");
let e_pos = out[start..].find('e').expect("scientific format must contain 'e'") + start;
let exponent: i32 = out[e_pos + 1..].parse().expect("exponent must be a valid integer");
out.truncate(e_pos);
let exp_sign = if exponent >= 0 { '+' } else { '-' };
write!(out, "e{exp_sign}{:02}", exponent.unsigned_abs()).expect("writing to String cannot fail");
} else {
let start = out.len();
write!(out, "{value}").expect("writing to String cannot fail");
if !out[start..].contains('.') {
out.push_str(".0");
}
}
}
fn write_indent(out: &mut String, config: &JsonDumpsConfig, depth: usize) {
if let Some(indent) = &config.indent {
for _ in 0..depth {
out.push_str(indent);
}
}
}
fn write_json_string(value: &str, out: &mut String, ensure_ascii: bool) {
out.push('"');
let bytes = value.as_bytes();
let mut start = 0;
let mut i = 0;
while i < bytes.len() {
let byte = bytes[i];
if ensure_ascii && byte >= 0x7F {
out.push_str(&value[start..i]);
if byte == 0x7F {
out.push_str("\\u007f");
i += 1;
} else {
let ch = value[i..].chars().next().expect("valid UTF-8");
write_json_escape_for_non_ascii(ch, out);
i += ch.len_utf8();
}
start = i;
continue;
}
let escape = ESCAPE_TABLE[byte as usize];
if escape == 0 {
i += 1;
continue;
}
out.push_str(&value[start..i]);
match escape {
b'b' => out.push_str("\\b"),
b't' => out.push_str("\\t"),
b'n' => out.push_str("\\n"),
b'f' => out.push_str("\\f"),
b'r' => out.push_str("\\r"),
b'"' => out.push_str("\\\""),
b'\\' => out.push_str("\\\\"),
b'u' => {
write!(out, "\\u{:04x}", u32::from(byte)).expect("writing to String cannot fail");
}
_ => unreachable!(),
}
i += 1;
start = i;
}
out.push_str(&value[start..]);
out.push('"');
}
#[rustfmt::skip]
static ESCAPE_TABLE: [u8; 256] = {
let mut table = [0u8; 256];
let mut i = 0;
while i < 0x20 {
table[i] = b'u';
i += 1;
}
table[0x08] = b'b'; table[0x09] = b't'; table[0x0A] = b'n'; table[0x0C] = b'f'; table[0x0D] = b'r'; table[0x22] = b'"'; table[0x5C] = b'\\'; table
};
fn write_json_escape_for_non_ascii(ch: char, out: &mut String) {
let code = ch as u32;
if code <= 0xFFFF {
write!(out, "\\u{code:04x}").expect("writing to String cannot fail");
} else {
let code = code - 0x1_0000;
let high = 0xD800 + (code >> 10);
let low = 0xDC00 + (code & 0x3FF);
write!(out, "\\u{high:04x}\\u{low:04x}").expect("writing to String cannot fail");
}
}