secs_project/
pyjson.rs

1//! Python-compatible JSON values, parser, and pretty-printer.
2
3use std::collections::HashMap;
4use std::collections::hash_map::Entry;
5use std::fmt;
6
7/// A JSON value exactly as Python's `json` module reads and writes it.
8#[derive(Clone, Debug, PartialEq)]
9pub enum PyJson {
10    Null,
11    Bool(bool),
12    /// Any integer, as canonical decimal text.
13    Int(String),
14    /// Any `f64`, including NaN, infinities, and negative zero.
15    Float(f64),
16    Str(String),
17    Array(Vec<PyJson>),
18    /// Object members in insertion order.
19    Object(Vec<(String, PyJson)>),
20}
21
22impl PyJson {
23    pub(crate) fn object(&self) -> Option<&[(String, PyJson)]> {
24        match self {
25            Self::Object(value) => Some(value),
26            _ => None,
27        }
28    }
29
30    pub(crate) fn array(&self) -> Option<&[PyJson]> {
31        match self {
32            Self::Array(value) => Some(value),
33            _ => None,
34        }
35    }
36
37    pub(crate) fn str(&self) -> Option<&str> {
38        match self {
39            Self::Str(value) => Some(value),
40            _ => None,
41        }
42    }
43
44    pub(crate) fn bool(&self) -> Option<bool> {
45        match self {
46            Self::Bool(value) => Some(*value),
47            _ => None,
48        }
49    }
50
51    pub(crate) fn get(&self, key: &str) -> Option<&PyJson> {
52        self.object()?
53            .iter()
54            .find(|(name, _)| name == key)
55            .map(|(_, value)| value)
56    }
57
58    pub(crate) fn object_mut(&mut self) -> Option<&mut Vec<(String, PyJson)>> {
59        match self {
60            Self::Object(value) => Some(value),
61            _ => None,
62        }
63    }
64
65    /// Python's `bool(value)`: `None`, `False`, zero and empty containers are false.
66    pub(crate) fn is_truthy(&self) -> bool {
67        match self {
68            Self::Null => false,
69            Self::Bool(value) => *value,
70            Self::Int(value) => value != "0",
71            Self::Float(value) => *value != 0.0,
72            Self::Str(value) => !value.is_empty(),
73            Self::Array(value) => !value.is_empty(),
74            Self::Object(value) => !value.is_empty(),
75        }
76    }
77}
78
79/// How a value the writer rebuilds from a typed field meets the value the
80/// file held, so saving keeps keys this version does not know (ADR 0004,
81/// docs/DEVIATIONS.md F16).
82#[derive(Debug)]
83pub(crate) enum Shape {
84    /// An object with fixed field names: the writer's fields in its order,
85    /// then the file's other fields in theirs. `optional` names fields the
86    /// writer leaves out on purpose, which therefore do not come back from the
87    /// file. `nested` gives the shape of fields merged in turn; the writer's
88    /// value replaces every other field.
89    Record {
90        optional: &'static [&'static str],
91        nested: &'static [(&'static str, Shape)],
92    },
93    /// An object whose keys are data (roles, bundles): only the writer's
94    /// entries, each merged with the file's entry of the same key.
95    Map(&'static Shape),
96    /// An array merged element by element when both sides have the same
97    /// length; otherwise the writer's array.
98    List(&'static Shape),
99}
100
101/// Merges `new`, rebuilt by the writer, with `old`, read from the file.
102pub(crate) fn merge_with_file(new: PyJson, old: &PyJson, shape: &Shape) -> PyJson {
103    match (shape, new, old) {
104        (
105            Shape::Record { optional, nested },
106            PyJson::Object(fields),
107            PyJson::Object(old_fields),
108        ) => {
109            let mut out = fields
110                .into_iter()
111                .map(|(key, value)| {
112                    let value = match (
113                        nested.iter().find(|(name, _)| *name == key),
114                        old_fields.iter().find(|(name, _)| *name == key),
115                    ) {
116                        (Some((_, shape)), Some((_, old))) => merge_with_file(value, old, shape),
117                        _ => value,
118                    };
119                    (key, value)
120                })
121                .collect::<Vec<_>>();
122            let unknown = old_fields
123                .iter()
124                .filter(|(key, _)| {
125                    !optional.contains(&key.as_str()) && !out.iter().any(|(name, _)| name == key)
126                })
127                .cloned()
128                .collect::<Vec<_>>();
129            out.extend(unknown);
130            PyJson::Object(out)
131        }
132        (Shape::Map(entry), PyJson::Object(fields), PyJson::Object(old_fields)) => PyJson::Object(
133            fields
134                .into_iter()
135                .map(|(key, value)| {
136                    let value = match old_fields.iter().find(|(name, _)| *name == key) {
137                        Some((_, old)) => merge_with_file(value, old, entry),
138                        None => value,
139                    };
140                    (key, value)
141                })
142                .collect(),
143        ),
144        (Shape::List(item), PyJson::Array(items), PyJson::Array(old_items))
145            if items.len() == old_items.len() =>
146        {
147            PyJson::Array(
148                items
149                    .into_iter()
150                    .zip(old_items)
151                    .map(|(value, old)| merge_with_file(value, old, item))
152                    .collect(),
153            )
154        }
155        (_, new, _) => new,
156    }
157}
158
159#[derive(Clone, Debug, PartialEq, Eq)]
160pub struct PyJsonError {
161    offset: usize,
162    message: String,
163}
164
165impl PyJsonError {
166    fn new(offset: usize, message: impl Into<String>) -> Self {
167        Self {
168            offset,
169            message: message.into(),
170        }
171    }
172}
173
174impl fmt::Display for PyJsonError {
175    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
176        write!(f, "{} at byte {}", self.message, self.offset)
177    }
178}
179
180impl std::error::Error for PyJsonError {}
181
182/// Parses the same value language as CPython's `json.loads`.
183pub fn loads(text: &str) -> Result<PyJson, PyJsonError> {
184    let mut parser = Parser {
185        text,
186        bytes: text.as_bytes(),
187        pos: 0,
188    };
189    parser.whitespace();
190    let value = parser.value(0)?;
191    parser.whitespace();
192    if parser.pos != parser.bytes.len() {
193        return Err(parser.error("extra data"));
194    }
195    Ok(value)
196}
197
198struct Parser<'a> {
199    text: &'a str,
200    bytes: &'a [u8],
201    pos: usize,
202}
203
204impl Parser<'_> {
205    fn error(&self, message: impl Into<String>) -> PyJsonError {
206        PyJsonError::new(self.pos, message)
207    }
208
209    fn whitespace(&mut self) {
210        while matches!(self.bytes.get(self.pos), Some(b' ' | b'\n' | b'\r' | b'\t')) {
211            self.pos += 1;
212        }
213    }
214
215    fn value(&mut self, depth: usize) -> Result<PyJson, PyJsonError> {
216        match self.bytes.get(self.pos).copied() {
217            Some(b'n') => self.keyword(b"null", PyJson::Null),
218            Some(b't') => self.keyword(b"true", PyJson::Bool(true)),
219            Some(b'f') => self.keyword(b"false", PyJson::Bool(false)),
220            Some(b'N') => self.keyword(b"NaN", PyJson::Float(f64::NAN)),
221            Some(b'I') => self.keyword(b"Infinity", PyJson::Float(f64::INFINITY)),
222            Some(b'-')
223                if self
224                    .bytes
225                    .get(self.pos..)
226                    .is_some_and(|s| s.starts_with(b"-Infinity")) =>
227            {
228                self.keyword(b"-Infinity", PyJson::Float(f64::NEG_INFINITY))
229            }
230            Some(b'"') => self.string().map(PyJson::Str),
231            Some(b'[') => self.array_value(depth),
232            Some(b'{') => self.object_value(depth),
233            Some(b'-' | b'0'..=b'9') => self.number(),
234            _ => Err(self.error("expecting value")),
235        }
236    }
237
238    fn keyword(&mut self, word: &[u8], value: PyJson) -> Result<PyJson, PyJsonError> {
239        if self
240            .bytes
241            .get(self.pos..)
242            .is_some_and(|rest| rest.starts_with(word))
243        {
244            self.pos += word.len();
245            Ok(value)
246        } else {
247            Err(self.error("expecting value"))
248        }
249    }
250
251    fn nested(&self, depth: usize) -> Result<usize, PyJsonError> {
252        let next = depth + 1;
253        if next > 1000 {
254            Err(self.error("maximum JSON nesting depth exceeded"))
255        } else {
256            Ok(next)
257        }
258    }
259
260    fn array_value(&mut self, depth: usize) -> Result<PyJson, PyJsonError> {
261        let depth = self.nested(depth)?;
262        self.pos += 1;
263        self.whitespace();
264        let mut values = Vec::new();
265        if self.bytes.get(self.pos) == Some(&b']') {
266            self.pos += 1;
267            return Ok(PyJson::Array(values));
268        }
269        loop {
270            values.push(self.value(depth)?);
271            self.whitespace();
272            match self.bytes.get(self.pos) {
273                Some(b',') => {
274                    self.pos += 1;
275                    self.whitespace();
276                }
277                Some(b']') => {
278                    self.pos += 1;
279                    return Ok(PyJson::Array(values));
280                }
281                _ => return Err(self.error("expecting ',' or ']'")),
282            }
283        }
284    }
285
286    fn object_value(&mut self, depth: usize) -> Result<PyJson, PyJsonError> {
287        let depth = self.nested(depth)?;
288        self.pos += 1;
289        self.whitespace();
290        let mut values: Vec<(String, PyJson)> = Vec::new();
291        if self.bytes.get(self.pos) == Some(&b'}') {
292            self.pos += 1;
293            return Ok(PyJson::Object(values));
294        }
295        // Key -> position in `values`, only while this object is parsed.
296        let mut index: HashMap<String, usize> = HashMap::new();
297        loop {
298            if self.bytes.get(self.pos) != Some(&b'"') {
299                return Err(self.error("expecting property name"));
300            }
301            let key = self.string()?;
302            self.whitespace();
303            if self.bytes.get(self.pos) != Some(&b':') {
304                return Err(self.error("expecting ':'"));
305            }
306            self.pos += 1;
307            self.whitespace();
308            let value = self.value(depth)?;
309            // Like a Python dict: a repeated key keeps its first position and
310            // takes the last value.
311            match index.entry(key) {
312                Entry::Occupied(slot) => values[*slot.get()].1 = value,
313                Entry::Vacant(slot) => {
314                    values.push((slot.key().clone(), value));
315                    slot.insert(values.len() - 1);
316                }
317            }
318            self.whitespace();
319            match self.bytes.get(self.pos) {
320                Some(b',') => {
321                    self.pos += 1;
322                    self.whitespace();
323                }
324                Some(b'}') => {
325                    self.pos += 1;
326                    return Ok(PyJson::Object(values));
327                }
328                _ => return Err(self.error("expecting ',' or '}'")),
329            }
330        }
331    }
332
333    fn string(&mut self) -> Result<String, PyJsonError> {
334        self.pos += 1;
335        let mut out = String::new();
336        let mut raw_start = self.pos;
337        while let Some(&byte) = self.bytes.get(self.pos) {
338            match byte {
339                b'"' => {
340                    out.push_str(&self.text[raw_start..self.pos]);
341                    self.pos += 1;
342                    return Ok(out);
343                }
344                b'\\' => {
345                    out.push_str(&self.text[raw_start..self.pos]);
346                    self.pos += 1;
347                    let escaped = *self
348                        .bytes
349                        .get(self.pos)
350                        .ok_or_else(|| self.error("unterminated string"))?;
351                    self.pos += 1;
352                    match escaped {
353                        b'"' => out.push('"'),
354                        b'\\' => out.push('\\'),
355                        b'/' => out.push('/'),
356                        b'b' => out.push('\u{8}'),
357                        b'f' => out.push('\u{c}'),
358                        b'n' => out.push('\n'),
359                        b'r' => out.push('\r'),
360                        b't' => out.push('\t'),
361                        b'u' => {
362                            let first = self.hex_unit()?;
363                            let scalar = if (0xd800..=0xdbff).contains(&first) {
364                                if self.bytes.get(self.pos..self.pos.saturating_add(2))
365                                    != Some(b"\\u")
366                                {
367                                    return Err(self.error("unpaired UTF-16 surrogate"));
368                                }
369                                self.pos += 2;
370                                let second = self.hex_unit()?;
371                                if !(0xdc00..=0xdfff).contains(&second) {
372                                    return Err(self.error("unpaired UTF-16 surrogate"));
373                                }
374                                0x10000
375                                    + ((u32::from(first) - 0xd800) << 10)
376                                    + (u32::from(second) - 0xdc00)
377                            } else if (0xdc00..=0xdfff).contains(&first) {
378                                return Err(self.error("unpaired UTF-16 surrogate"));
379                            } else {
380                                u32::from(first)
381                            };
382                            out.push(
383                                char::from_u32(scalar)
384                                    .ok_or_else(|| self.error("invalid Unicode escape"))?,
385                            );
386                        }
387                        _ => return Err(self.error("invalid escape")),
388                    }
389                    raw_start = self.pos;
390                }
391                0..=0x1f => return Err(self.error("invalid control character")),
392                _ => {
393                    let ch = self.text[self.pos..]
394                        .chars()
395                        .next()
396                        .ok_or_else(|| self.error("unterminated string"))?;
397                    self.pos += ch.len_utf8();
398                }
399            }
400        }
401        Err(self.error("unterminated string"))
402    }
403
404    fn hex_unit(&mut self) -> Result<u16, PyJsonError> {
405        let end = self
406            .pos
407            .checked_add(4)
408            .ok_or_else(|| self.error("invalid Unicode escape"))?;
409        let digits = self
410            .text
411            .get(self.pos..end)
412            .ok_or_else(|| self.error("invalid Unicode escape"))?;
413        if !digits.bytes().all(|b| b.is_ascii_hexdigit()) {
414            return Err(self.error("invalid Unicode escape"));
415        }
416        self.pos = end;
417        u16::from_str_radix(digits, 16).map_err(|_| self.error("invalid Unicode escape"))
418    }
419
420    fn number(&mut self) -> Result<PyJson, PyJsonError> {
421        let start = self.pos;
422        if self.bytes.get(self.pos) == Some(&b'-') {
423            self.pos += 1;
424        }
425        match self.bytes.get(self.pos) {
426            Some(b'0') => self.pos += 1,
427            Some(b'1'..=b'9') => {
428                self.pos += 1;
429                while self.bytes.get(self.pos).is_some_and(u8::is_ascii_digit) {
430                    self.pos += 1;
431                }
432            }
433            _ => return Err(self.error("invalid number")),
434        }
435        let mut float = false;
436        if self.bytes.get(self.pos) == Some(&b'.') {
437            float = true;
438            self.pos += 1;
439            let before = self.pos;
440            while self.bytes.get(self.pos).is_some_and(u8::is_ascii_digit) {
441                self.pos += 1;
442            }
443            if self.pos == before {
444                return Err(self.error("invalid number"));
445            }
446        }
447        if matches!(self.bytes.get(self.pos), Some(b'e' | b'E')) {
448            float = true;
449            self.pos += 1;
450            if matches!(self.bytes.get(self.pos), Some(b'+' | b'-')) {
451                self.pos += 1;
452            }
453            let before = self.pos;
454            while self.bytes.get(self.pos).is_some_and(u8::is_ascii_digit) {
455                self.pos += 1;
456            }
457            if self.pos == before {
458                return Err(self.error("invalid number"));
459            }
460        }
461        let raw = &self.text[start..self.pos];
462        if float {
463            let value = raw
464                .parse::<f64>()
465                .map_err(|_| self.error("invalid number"))?;
466            Ok(PyJson::Float(value))
467        } else if raw == "-0" {
468            Ok(PyJson::Int("0".into()))
469        } else {
470            Ok(PyJson::Int(raw.into()))
471        }
472    }
473}
474
475/// Formats like `json.dumps(value, ensure_ascii=False, indent=2)`.
476pub fn dumps(value: &PyJson) -> String {
477    let mut out = String::new();
478    write_value(value, 0, &mut out);
479    out
480}
481
482/// Formats like `json.dumps(value, ensure_ascii=False)`.
483pub fn dumps_compact(value: &PyJson) -> String {
484    let mut out = String::new();
485    write_compact(value, &mut out);
486    out
487}
488
489fn write_compact(value: &PyJson, out: &mut String) {
490    match value {
491        PyJson::Null => out.push_str("null"),
492        PyJson::Bool(value) => out.push_str(if *value { "true" } else { "false" }),
493        PyJson::Int(value) => out.push_str(value),
494        PyJson::Float(value) => out.push_str(&python_float(*value)),
495        PyJson::Str(value) => write_string(value, out),
496        PyJson::Array(values) => {
497            out.push('[');
498            for (index, value) in values.iter().enumerate() {
499                if index != 0 {
500                    out.push_str(", ");
501                }
502                write_compact(value, out);
503            }
504            out.push(']');
505        }
506        PyJson::Object(values) => {
507            out.push('{');
508            for (index, (key, value)) in values.iter().enumerate() {
509                if index != 0 {
510                    out.push_str(", ");
511                }
512                write_string(key, out);
513                out.push_str(": ");
514                write_compact(value, out);
515            }
516            out.push('}');
517        }
518    }
519}
520
521fn write_value(value: &PyJson, depth: usize, out: &mut String) {
522    match value {
523        PyJson::Null => out.push_str("null"),
524        PyJson::Bool(value) => out.push_str(if *value { "true" } else { "false" }),
525        PyJson::Int(value) => out.push_str(value),
526        PyJson::Float(value) => out.push_str(&python_float(*value)),
527        PyJson::Str(value) => write_string(value, out),
528        PyJson::Array(values) => {
529            if values.is_empty() {
530                out.push_str("[]");
531                return;
532            }
533            out.push_str("[\n");
534            for (index, value) in values.iter().enumerate() {
535                out.push_str(&" ".repeat((depth + 1) * 2));
536                write_value(value, depth + 1, out);
537                if index + 1 != values.len() {
538                    out.push(',');
539                }
540                out.push('\n');
541            }
542            out.push_str(&" ".repeat(depth * 2));
543            out.push(']');
544        }
545        PyJson::Object(values) => {
546            if values.is_empty() {
547                out.push_str("{}");
548                return;
549            }
550            out.push_str("{\n");
551            for (index, (key, value)) in values.iter().enumerate() {
552                out.push_str(&" ".repeat((depth + 1) * 2));
553                write_string(key, out);
554                out.push_str(": ");
555                write_value(value, depth + 1, out);
556                if index + 1 != values.len() {
557                    out.push(',');
558                }
559                out.push('\n');
560            }
561            out.push_str(&" ".repeat(depth * 2));
562            out.push('}');
563        }
564    }
565}
566
567fn write_string(value: &str, out: &mut String) {
568    out.push('"');
569    for ch in value.chars() {
570        match ch {
571            '"' => out.push_str("\\\""),
572            '\\' => out.push_str("\\\\"),
573            '\u{8}' => out.push_str("\\b"),
574            '\u{c}' => out.push_str("\\f"),
575            '\n' => out.push_str("\\n"),
576            '\r' => out.push_str("\\r"),
577            '\t' => out.push_str("\\t"),
578            '\0'..='\u{1f}' => {
579                use std::fmt::Write as _;
580                write!(out, "\\u{:04x}", ch as u32).expect("writing to String cannot fail");
581            }
582            _ => out.push(ch),
583        }
584    }
585    out.push('"');
586}
587
588fn python_float(value: f64) -> String {
589    if value.is_nan() {
590        return "NaN".into();
591    }
592    if value == f64::INFINITY {
593        return "Infinity".into();
594    }
595    if value == f64::NEG_INFINITY {
596        return "-Infinity".into();
597    }
598    if value == 0.0 && value.is_sign_negative() {
599        return "-0.0".into();
600    }
601    let mut text = ryu::Buffer::new().format_finite(value).to_owned();
602    let absolute = value.abs();
603    if !text.contains(['e', 'E']) && absolute != 0.0 && (absolute < 1e-4 || absolute >= 1e16) {
604        let negative = text.starts_with('-');
605        let unsigned = text.trim_start_matches('-');
606        let (digits, exponent) = if absolute < 1.0 {
607            let fractional = unsigned.trim_start_matches("0.");
608            let zeroes = fractional.chars().take_while(|ch| *ch == '0').count();
609            (
610                fractional.trim_start_matches('0').to_owned(),
611                -(zeroes as i32) - 1,
612            )
613        } else {
614            let whole = unsigned.find('.').unwrap_or(unsigned.len());
615            (unsigned.replace('.', ""), whole as i32 - 1)
616        };
617        let (first, rest) = digits.split_at(1);
618        let mantissa = if rest.is_empty() {
619            first.into()
620        } else {
621            format!("{first}.{rest}")
622        };
623        text = format!("{}{mantissa}e{exponent:+}", if negative { "-" } else { "" });
624    }
625    if !text.contains(['.', 'e', 'E']) {
626        text.push_str(".0");
627    }
628    if let Some(index) = text.find('e') {
629        let exponent = text[index + 1..]
630            .parse::<i32>()
631            .expect("ryu exponent is an integer");
632        text = format!(
633            "{}e{}{:02}",
634            &text[..index],
635            if exponent >= 0 { "+" } else { "-" },
636            exponent.abs()
637        );
638    }
639    text
640}
641
642#[cfg(test)]
643mod tests {
644    use super::*;
645
646    #[test]
647    fn scanner_and_encoder_edge_cases() {
648        let input = r#" {"a":-0,"b":-0.0,"c":1e400,"d":340282366920938463463374607431768211456,"a":2,"s":"한글 😀\n😀"} "#;
649        let value = loads(input).unwrap();
650        assert_eq!(value.get("a"), Some(&PyJson::Int("2".into())));
651        assert_eq!(value.get("b"), Some(&PyJson::Float(-0.0)));
652        assert_eq!(value.get("c"), Some(&PyJson::Float(f64::INFINITY)));
653        assert!(dumps(&value).contains("340282366920938463463374607431768211456"));
654        assert!(dumps(&value).contains("한글 😀\\n😀"));
655    }
656
657    #[test]
658    fn rejects_controls_surrogates_and_excess_depth() {
659        assert!(loads("\"line\n\"").is_err());
660        assert!(loads(r#""\ud800""#).is_err());
661        assert!(loads(&format!("{}0{}", "[".repeat(1000), "]".repeat(1000))).is_ok());
662        assert!(loads(&format!("{}0{}", "[".repeat(1001), "]".repeat(1001))).is_err());
663    }
664}
secs_project/pyjson.rs

secs_project/
pyjson.rs
