Skip to main content

unityrs_export/
pyjson.rs

1//! JSON as `CPython` 3.11's `json.dumps(value, indent=2)` writes it at its
2//! other defaults (`docs/plans/extract.md` E24): `ensure_ascii`, so every
3//! character outside `' '..='~'` is a `\uXXXX` escape (lowercase hex, a
4//! character past U+FFFF as its surrogate pair) but `\b`, `\f`, `\n`, `\r`
5//! and `\t`, which have short forms, as `"` and `\` do; `allow_nan`, so a
6//! NaN is `NaN` and the infinities `Infinity` and `-Infinity`; a float as
7//! its `repr`; the separators `,` and `: ` that `indent` implies; an empty
8//! list `[]` and an empty object `{}`, each on its line.
9//!
10//! A value is a typetree value as UnityPy 1.25.2's `read_typetree()` gives
11//! it ([`unityrs_core::Value`]): a dict as an object in its field order, a
12//! pair or any list as an array, an `f32` widened exactly to the double, a
13//! string decoded from its bytes as Python's `surrogateescape` decodes them
14//! (each byte of an invalid UTF-8 sequence becomes the lone surrogate
15//! `U+DC00 + byte`, written `\udcXX`). Bytes (`TypelessData`) are a value
16//! `json.dumps` cannot write ([`Unwritable`]).
17//!
18//! [`Options::MONO_BEHAVIOUR`] selects four spaces and `ensure_ascii=False`.
19//! In this mode valid non-ASCII characters remain UTF-8 (including DEL and
20//! U+2028); escaped source bytes are written back verbatim, modeling the
21//! final UTF-8 surrogateescape encoding. Only U+DC80..U+DCFF can arise from
22//! a raw byte string; Python's encoder would raise on other lone surrogates,
23//! which this value model cannot represent.
24//!
25//! Every write goes through a [`Sink`]: [`Count`] measures a document
26//! before anything is allocated for it, so the caller charges the exact
27//! size first, then writes into a buffer of that capacity. The value walk
28//! keeps an explicit stack, one frame per open container, charged to the
29//! caller's meter before it grows, so no value's depth costs the thread's
30//! stack.
31
32use std::sync::Arc;
33
34use unityrs_core::limits::{ChargeError, Meter, MeteredVec};
35use unityrs_core::pyfloat::py_float_repr;
36use unityrs_core::{Map, Parsed, PrimArray, Value};
37
38/// Where the writer puts its bytes.
39pub trait Sink {
40    /// Appends `bytes`.
41    fn put(&mut self, bytes: &[u8]);
42    /// A refused sink write. Traversal stops before formatting more values.
43    fn check(&self) -> Result<(), ChargeError> {
44        Ok(())
45    }
46}
47
48/// A [`Sink`] that only counts.
49#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
50pub struct Count(pub u64);
51
52impl Sink for Count {
53    fn put(&mut self, bytes: &[u8]) {
54        let n = u64::try_from(bytes.len()).unwrap_or(u64::MAX);
55        self.0 = self.0.saturating_add(n);
56    }
57}
58
59impl Sink for Vec<u8> {
60    fn put(&mut self, bytes: &[u8]) {
61        self.extend_from_slice(bytes);
62    }
63}
64
65/// A value `json.dumps` raises on: bytes (`TypelessData`), which E25 makes
66/// the atlas JSON's skip, `json_unwritable`.
67#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
68#[error("a bytes value, which json.dumps cannot write")]
69pub struct Unwritable;
70
71/// JSON formatting; `MONO_BEHAVIOUR` includes the extractor's final
72/// UTF-8 `surrogateescape` encoding, so malformed source bytes round-trip.
73#[derive(Clone, Copy, Debug)]
74pub struct Options {
75    pub indent: usize,
76    pub ascii_only: bool,
77}
78impl Options {
79    pub const DEFAULT: Self = Self {
80        indent: 2,
81        ascii_only: true,
82    };
83    pub const MONO_BEHAVIOUR: Self = Self {
84        indent: 4,
85        ascii_only: false,
86    };
87}
88fn newline_with(level: usize, options: Options, out: &mut impl Sink) {
89    out.put(b"\n");
90    for _ in 0..level {
91        for _ in 0..options.indent {
92            if out.check().is_err() {
93                return;
94            }
95            out.put(b" ");
96        }
97    }
98}
99
100/// A newline and `level` levels of two spaces.
101pub fn newline(level: usize, out: &mut impl Sink) {
102    out.put(b"\n");
103    for _ in 0..level {
104        out.put(b"  ");
105    }
106}
107
108/// Python's `repr` of a double, as `json.dumps` writes a float.
109pub fn float(x: f64, out: &mut impl Sink) {
110    super::pyjson_numbers::float(x, out);
111}
112
113/// Python's `repr(x)` for a float as `json.dumps` writes it: the core's
114/// [`py_float_repr`] (the nearest of the shortest digits that read back,
115/// ties to even), but JSON's `NaN`, `Infinity` and `-Infinity`.
116#[must_use]
117pub fn float_repr(x: f64) -> String {
118    if x.is_nan() {
119        return "NaN".to_owned();
120    }
121    if x.is_infinite() {
122        return if x < 0.0 { "-Infinity" } else { "Infinity" }.to_owned();
123    }
124    py_float_repr(x)
125}
126
127/// `bytes` decoded as Python's `surrogateescape` decodes them: each valid
128/// UTF-8 character as itself (`Ok`), each byte of an invalid sequence as one
129/// item of its own (`Err(byte)`), in order.
130pub fn decode_escaped(bytes: &[u8], mut each: impl FnMut(Result<char, u8>)) {
131    decode_escaped_while(bytes, |c| {
132        each(c);
133        true
134    });
135}
136
137// Bound validation ahead of a sink refusal, including within a large string.
138fn decode_escaped_while(bytes: &[u8], mut each: impl FnMut(Result<char, u8>) -> bool) {
139    let mut rest = bytes;
140    while !rest.is_empty() {
141        let (chunk, after_chunk) = rest.split_at(rest.len().min(1024));
142        match std::str::from_utf8(chunk) {
143            Ok(s) => {
144                for c in s.chars() {
145                    if !each(Ok(c)) {
146                        return;
147                    }
148                }
149                rest = after_chunk;
150            }
151            Err(e) => {
152                let valid = e.valid_up_to();
153                let (good, tail) = rest.split_at(valid);
154                if let Ok(s) = std::str::from_utf8(good) {
155                    for c in s.chars() {
156                        if !each(Ok(c)) {
157                            return;
158                        }
159                    }
160                }
161                // An incomplete codepoint at a chunk boundary belongs to
162                // the next chunk, rather than becoming surrogateescape bytes.
163                if e.error_len().is_none() && chunk.len() < rest.len() {
164                    rest = tail;
165                    continue;
166                }
167                let bad = e.error_len().unwrap_or(tail.len()).min(tail.len());
168                let (invalid, after) = tail.split_at(bad);
169                for &b in invalid {
170                    if !each(Err(b)) {
171                        return;
172                    }
173                }
174                rest = after;
175            }
176        }
177    }
178}
179
180/// A JSON string of `bytes`, decoded as [`decode_escaped`] decodes them and
181/// escaped as `ensure_ascii` escapes (module docs).
182pub fn string(bytes: &[u8], out: &mut impl Sink) {
183    string_with(bytes, true, out);
184}
185
186fn string_with(bytes: &[u8], ascii_only: bool, out: &mut impl Sink) {
187    out.put(b"\"");
188    decode_escaped_while(bytes, |item| {
189        if out.check().is_err() {
190            return false;
191        }
192        match item {
193            Ok(c) if !ascii_only && c >= ' ' => match c {
194                '"' | '\\' => char_escaped(c, out),
195                _ => {
196                    let mut buf = [0; 4];
197                    out.put(c.encode_utf8(&mut buf).as_bytes());
198                }
199            },
200            Ok(c) => char_escaped(c, out),
201            Err(b) if !ascii_only => out.put(&[b]),
202            Err(b) => unit(0xDC00 + u32::from(b), out),
203        }
204        out.check().is_ok()
205    });
206    out.put(b"\"");
207}
208
209fn char_escaped(c: char, out: &mut impl Sink) {
210    match c {
211        '"' => out.put(b"\\\""),
212        '\\' => out.put(b"\\\\"),
213        '\n' => out.put(b"\\n"),
214        '\r' => out.put(b"\\r"),
215        '\t' => out.put(b"\\t"),
216        '\u{8}' => out.put(b"\\b"),
217        '\u{c}' => out.put(b"\\f"),
218        ' '..='~' => {
219            let mut buf = [0u8; 4];
220            out.put(c.encode_utf8(&mut buf).as_bytes());
221        }
222        _ => {
223            let mut units = [0u16; 2];
224            for u in c.encode_utf16(&mut units) {
225                unit(u32::from(*u), out);
226            }
227        }
228    }
229}
230
231/// `\uXXXX`, lowercase.
232fn unit(u: u32, out: &mut impl Sink) {
233    let hex = b"0123456789abcdef";
234    let mut bytes = *b"\\u0000";
235    for (i, dst) in bytes.iter_mut().skip(2).enumerate() {
236        *dst = hex
237            .get(((u >> (12 - i * 4)) & 15) as usize)
238            .copied()
239            .unwrap_or(b'0');
240    }
241    out.put(&bytes);
242}
243
244/// A decimal integer.
245pub fn int(i: i128, out: &mut impl Sink) {
246    super::pyjson_numbers::int(i, out);
247}
248
249fn f32_value(bits: u32) -> f64 {
250    f64::from(f32::from_bits(bits))
251}
252
253/// One element of a primitive run.
254fn prim(arr: &PrimArray, i: usize, out: &mut impl Sink) {
255    match arr {
256        PrimArray::Bool(v) => {
257            if let Some(&b) = v.get(i) {
258                out.put(if b { b"true" } else { b"false" });
259            }
260        }
261        PrimArray::U8(v) => v.get(i).into_iter().for_each(|&x| int(x.into(), out)),
262        PrimArray::I8(v) => v.get(i).into_iter().for_each(|&x| int(x.into(), out)),
263        PrimArray::U16(v) => v.get(i).into_iter().for_each(|&x| int(x.into(), out)),
264        PrimArray::I16(v) => v.get(i).into_iter().for_each(|&x| int(x.into(), out)),
265        PrimArray::U32(v) => v.get(i).into_iter().for_each(|&x| int(x.into(), out)),
266        PrimArray::I32(v) => v.get(i).into_iter().for_each(|&x| int(x.into(), out)),
267        PrimArray::U64(v) => v.get(i).into_iter().for_each(|&x| int(x.into(), out)),
268        PrimArray::I64(v) => v.get(i).into_iter().for_each(|&x| int(x.into(), out)),
269        PrimArray::F32(v) => v.get(i).into_iter().for_each(|&x| float(f32_value(x), out)),
270        PrimArray::F64(v) => v
271            .get(i)
272            .into_iter()
273            .for_each(|&x| float(f64::from_bits(x), out)),
274    }
275}
276
277/// An open container's elements.
278enum Items<'a> {
279    Seq(&'a [Value]),
280    Dict(&'a Map),
281    Prims(&'a PrimArray),
282}
283
284impl Items<'_> {
285    fn len(&self) -> usize {
286        match self {
287            Self::Seq(v) => v.len(),
288            Self::Dict(m) => m.len(),
289            Self::Prims(p) => p.len(),
290        }
291    }
292}
293
294/// An open container: its elements, the next one, its level.
295struct Frame<'a> {
296    items: Items<'a>,
297    next: usize,
298    level: usize,
299}
300
301/// What a value opens: a container with elements, or nothing.
302fn open(value: &Value) -> Option<Items<'_>> {
303    let items = match value {
304        Value::List(v) => Items::Seq(v),
305        Value::Tuple(v) => Items::Seq(v),
306        Value::Dict(m) => Items::Dict(m),
307        Value::Prims(p) => Items::Prims(p),
308        _ => return None,
309    };
310    Some(items)
311}
312
313/// Writes a scalar, an empty container, or a container's opening bracket
314/// (returning its frame).
315fn begin<'a>(
316    parsed: &Parsed,
317    value: &'a Value,
318    level: usize,
319    options: Options,
320    out: &mut impl Sink,
321) -> Result<Option<Frame<'a>>, WriteError> {
322    out.check()?;
323    let _ = parsed;
324    match value {
325        Value::Bool(b) => out.put(if *b { b"true" } else { b"false" }),
326        Value::Int(i) => int((*i).into(), out),
327        Value::UInt(u) => int((*u).into(), out),
328        Value::F32(bits) => float(f32_value(*bits), out),
329        Value::F64(bits) => float(f64::from_bits(*bits), out),
330        Value::Str(s) => string_with(s.as_bytes(), options.ascii_only, out),
331        Value::Bytes(_) => return Err(Unwritable.into()),
332        _ => {
333            let Some(items) = open(value) else {
334                return Err(Unwritable.into());
335            };
336            let dict = matches!(items, Items::Dict(_));
337            if items.len() == 0 {
338                out.put(if dict { b"{}" } else { b"[]" });
339                return Ok(None);
340            }
341            out.put(if dict { b"{" } else { b"[" });
342            return Ok(Some(Frame {
343                items,
344                next: 0,
345                level,
346            }));
347        }
348    }
349    out.check()?;
350    Ok(None)
351}
352
353/// Why a value was not written: a value `json.dumps` cannot write, or the
354/// refused charge of the walk's stack.
355#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
356pub enum WriteError {
357    /// [`Unwritable`].
358    #[error(transparent)]
359    Unwritable(#[from] Unwritable),
360    /// The stack's growth, refused by `meter` (or by the allocator, which
361    /// the core reports as a budget's refusal).
362    #[error("the JSON writer's stack: {0}")]
363    Charge(#[from] ChargeError),
364}
365
366/// Writes `value`, read with `parsed`'s node lists (which name a dict's
367/// keys), as it sits at `level` levels of indentation: its first line
368/// continues the current one, its nested lines are indented below it. The
369/// walk's stack, a frame per open container, is charged to `meter` before
370/// it grows (review round 2's S3), and returned when the write ends.
371pub fn value(
372    parsed: &Parsed,
373    value: &Value,
374    level: usize,
375    out: &mut impl Sink,
376    meter: &Arc<Meter>,
377) -> Result<(), WriteError> {
378    value_with(parsed, value, level, Options::DEFAULT, out, meter)
379}
380
381/// [`value`] with explicit formatting and final UTF-8 surrogateescape encoding.
382pub fn value_with(
383    parsed: &Parsed,
384    value: &Value,
385    level: usize,
386    options: Options,
387    out: &mut impl Sink,
388    meter: &Arc<Meter>,
389) -> Result<(), WriteError> {
390    let mut stack: MeteredVec<Frame<'_>> = MeteredVec::new(meter);
391    if let Some(f) = begin(parsed, value, level, options, out)? {
392        out.check()?;
393        stack.push(f)?;
394    }
395    while let Some(top) = stack.as_mut_slice().last_mut() {
396        out.check()?;
397        let i = top.next;
398        let inner = top.level + 1;
399        if i >= top.items.len() {
400            let close: &[u8] = if matches!(top.items, Items::Dict(_)) {
401                b"}"
402            } else {
403                b"]"
404            };
405            let lvl = top.level;
406            stack.truncate(stack.len().saturating_sub(1));
407            newline_with(lvl, options, out);
408            out.put(close);
409            continue;
410        }
411        top.next += 1;
412        if i > 0 {
413            out.put(b",");
414        }
415        newline_with(inner, options, out);
416        let child = match &top.items {
417            Items::Prims(p) => {
418                prim(p, i, out);
419                None
420            }
421            Items::Seq(v) => v.get(i),
422            Items::Dict(m) => match m.entries().get(i) {
423                Some((k, v)) => {
424                    string_with(parsed.key_name(m, *k), options.ascii_only, out);
425                    out.put(b": ");
426                    Some(v)
427                }
428                None => None,
429            },
430        };
431        if let Some(v) = child
432            && let Some(f) = begin(parsed, v, inner, options, out)?
433        {
434            out.check()?;
435            stack.push(f)?;
436        }
437    }
438    out.check()?;
439    Ok(())
440}
441
442#[cfg(test)]
443mod tests {
444    use super::{Count, decode_escaped, float_repr, string};
445
446    #[test]
447    fn chunk_boundaries_preserve_utf8_and_surrogateescape() {
448        for prefix_len in 1020..1026 {
449            let prefix = "a".repeat(prefix_len);
450            let mut input = format!("{prefix}é😀\u{2028}\n").into_bytes();
451            input.extend_from_slice(b"\xe4\xb8\xff");
452            assert_eq!(
453                json(&input),
454                format!("\"{prefix}\\u00e9\\ud83d\\ude00\\u2028\\n\\udce4\\udcb8\\udcff\"")
455            );
456            let mut output = Vec::new();
457            super::string_with(&input, false, &mut output);
458            let mut expected = format!("\"{prefix}é😀\u{2028}\\n").into_bytes();
459            expected.extend_from_slice(b"\xe4\xb8\xff\"");
460            assert_eq!(output, expected);
461        }
462    }
463
464    #[test]
465    fn floats_are_written_as_python_reprs_them() {
466        // Each pair is what CPython 3.11.18 in the oracle image gives for
467        // `json.dumps(x)` (an `f32` as `struct` widens it).
468        for (x, want) in [
469            (1.0, "1.0"),
470            (0.1, "0.1"),
471            (1e-05, "1e-05"),
472            (0.0001, "0.0001"),
473            (1e16, "1e+16"),
474            (1e15, "1000000000000000.0"),
475            (-0.0, "-0.0"),
476            (0.0, "0.0"),
477            (f64::NAN, "NaN"),
478            (f64::INFINITY, "Infinity"),
479            (f64::NEG_INFINITY, "-Infinity"),
480            (f64::from(f32::MAX), "3.4028234663852886e+38"),
481            (f64::from(0.1f32), "0.10000000149011612"),
482            (f64::from(1e-5f32), "9.999999747378752e-06"),
483            (16_777_216.0, "16777216.0"),
484            (5e-324, "5e-324"),
485            (1.5, "1.5"),
486            (-2.5e-7, "-2.5e-07"),
487            (f64::MAX, "1.7976931348623157e+308"),
488        ] {
489            assert_eq!(float_repr(x), want, "{x:e}");
490        }
491    }
492
493    #[test]
494    fn ties_go_to_the_even_digits_and_powers_of_two_to_the_nearest() {
495        // Doubles written exactly: two shortest digit strings equally near,
496        // where CPython takes the even one (`{:e}` broke the tie upwards),
497        // and powers of two, whose nearest shortest digits can fail to read
498        // back (the dump fixture's `index.json` "str" cases).
499        for (exact, want) in [
500            ("123456789012345680.0", "1.2345678901234568e+17"),
501            ("1551563132437320.25", "1551563132437320.2"),
502            ("-1551563132437320.25", "-1551563132437320.2"),
503            ("79686618578806.625", "79686618578806.62"),
504            ("-22620844907483.3125", "-22620844907483.312"),
505            ("2.98023223876953125e-8", "2.9802322387695312e-08"),
506            ("-5.9604644775390625e-7", "-5.960464477539062e-07"),
507            ("9007199254740992.0", "9007199254740992.0"),
508        ] {
509            let x: f64 = exact.parse().unwrap();
510            assert_eq!(float_repr(x), want, "{exact}");
511        }
512    }
513
514    fn json(bytes: &[u8]) -> String {
515        let mut out = Vec::new();
516        string(bytes, &mut out);
517        String::from_utf8(out).unwrap()
518    }
519
520    #[test]
521    fn strings_escape_as_ensure_ascii_with_surrogateescape() {
522        // CPython 3.11.18's `json.dumps` of each, decoded with
523        // `surrogateescape`. `B` is a backslash, spelled so that no tool
524        // reads the escapes as characters.
525        const B: char = '\\';
526        assert_eq!(json(b"a\"b\\c"), r#""a\"b\\c""#);
527        assert_eq!(
528            json(b"\n\r\t\x08\x0c\x01\x7f"),
529            r#""\n\r\t\b\f\u0001\u007f""#
530        );
531        assert_eq!(
532            json("\u{e4}\u{7cbe}".as_bytes()),
533            format!("\"{B}u00e4{B}u7cbe\"")
534        );
535        assert_eq!(
536            json("\u{1f600}".as_bytes()),
537            format!("\"{B}ud83d{B}ude00\"")
538        );
539        assert_eq!(json(b"bad\xff"), r#""bad\udcff""#);
540        // A truncated sequence is one escape per byte; decoding resumes.
541        assert_eq!(json(b"\xe4\xb8a"), r#""\udce4\udcb8a""#);
542    }
543
544    #[test]
545    fn decoding_gives_one_item_per_invalid_byte() {
546        let mut n = 0;
547        decode_escaped(b"\xf0\x80x\xc3", |_| n += 1);
548        assert_eq!(n, 4);
549        let mut c = Count::default();
550        string(b"ab", &mut c);
551        assert_eq!(c.0, 4);
552    }
553}