Skip to main content

yog_data/forms/
text.rs

1//! The text syntax: JSON, plus `//` and `/* */` comments, trailing commas
2//! and unquoted keys (the part of JSON5 editors already understand).
3//!
4//! [`parse`] keeps where every value is ([`Spans`]), so a schema error at
5//! `entries[2].pages[0].item` can point at its line. Typed values stay
6//! strings here — the schema decides what a string means.
7
8use std::fmt;
9
10/// `\\u` escapes are hexadecimal.
11const HEX_RADIX: u32 = 16;
12
13use crate::model::value::{Map, Path, Seg, Value};
14
15mod comments;
16
17pub use comments::with_comments;
18
19/// Deeper than this is an error, not a stack overflow.
20pub const MAX_DEPTH: usize = 256;
21
22/// A byte range of the source.
23#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
24pub struct Span {
25    pub start: u32,
26    pub end: u32,
27}
28
29impl Span {
30    /// `start..end`; offsets past `u32` saturate.
31    pub fn new(start: usize, end: usize) -> Span {
32        Span {
33            start: u32::try_from(start).unwrap_or(u32::MAX),
34            end: u32::try_from(end).unwrap_or(u32::MAX),
35        }
36    }
37
38    /// From this one's start to `other`'s end.
39    #[must_use]
40    pub fn to(self, other: Span) -> Span {
41        Span {
42            start: self.start,
43            end: other.end.max(self.end),
44        }
45    }
46}
47
48/// Where each value of a parsed document is: the value's span, its key's
49/// (map entries), and its children's, in the same order as the value's.
50#[derive(Debug, Clone, Default, PartialEq)]
51pub struct Spans {
52    pub span: Span,
53    pub key: Option<Span>,
54    pub children: Vec<Spans>,
55}
56
57/// A parsed document: the value and where each of its parts is.
58#[derive(Debug, Clone, PartialEq)]
59pub struct Doc {
60    pub value: Value,
61    pub spans: Spans,
62}
63
64impl Doc {
65    /// The spans of the value at `path` — the closest existing parent's if
66    /// the path goes further than the document.
67    pub fn spans_at(&self, path: &Path) -> &Spans {
68        let mut v = &self.value;
69        let mut s = &self.spans;
70        for seg in path.segments() {
71            let i = match (seg, v) {
72                (Seg::Key(k), Value::Map(m)) => m.index_of(k),
73                (Seg::Index(i), Value::List(l)) if *i < l.len() => Some(*i),
74                _ => None,
75            };
76            let Some(i) = i else { break };
77            let Some(child) = s.children.get(i) else {
78                break;
79            };
80            v = match v {
81                Value::Map(m) => m.value_at(i).unwrap_or(v),
82                Value::List(l) => &l[i],
83                _ => v,
84            };
85            s = child;
86        }
87        s
88    }
89
90    /// The line and column of the value at `path` in `text` (the source this was parsed from).
91    pub fn locate(&self, text: &str, path: &Path) -> LineCol {
92        LineCol::of(text, self.spans_at(path).span.start as usize)
93    }
94}
95
96/// A 1-based line and column (in characters).
97#[derive(Debug, Clone, Copy, PartialEq, Eq)]
98pub struct LineCol {
99    pub line: u32,
100    pub col: u32,
101}
102
103impl LineCol {
104    /// Where byte `offset` of `text` is.
105    pub fn of(text: &str, offset: usize) -> LineCol {
106        let offset = offset.min(text.len());
107        let before = &text.as_bytes()[..offset];
108        let line = before.iter().filter(|&&b| b == b'\n').count() as u32 + 1;
109        let line_start = before
110            .iter()
111            .rposition(|&b| b == b'\n')
112            .map_or(0, |i| i + 1);
113        let col = text
114            .get(line_start..offset)
115            .map_or(offset - line_start, |s| s.chars().count()) as u32
116            + 1;
117        LineCol { line, col }
118    }
119}
120
121impl fmt::Display for LineCol {
122    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
123        write!(f, "{}:{}", self.line, self.col)
124    }
125}
126
127/// A syntax error.
128#[derive(Debug, Clone, PartialEq, Eq)]
129pub struct SyntaxError {
130    pub message: String,
131    pub at: LineCol,
132    pub offset: usize,
133}
134
135impl fmt::Display for SyntaxError {
136    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
137        write!(f, "{}: {}", self.at, self.message)
138    }
139}
140
141impl std::error::Error for SyntaxError {}
142
143/// Parses Yog Data text.
144pub fn parse(text: &str) -> Result<Doc, SyntaxError> {
145    let (value, spans) = run::<true>(text)?;
146    Ok(Doc { value, spans })
147}
148
149/// Parses Yog Data text, without the positions (faster).
150pub fn parse_value(text: &str) -> Result<Value, SyntaxError> {
151    run::<false>(text).map(|(v, _)| v)
152}
153
154fn run<const SPANS: bool>(text: &str) -> Result<(Value, Spans), SyntaxError> {
155    let mut p = Parser::<SPANS> {
156        src: text.as_bytes(),
157        text,
158        i: 0,
159        depth: 0,
160    };
161    if text.starts_with('\u{feff}') {
162        p.i = 3;
163    }
164    p.ws()?;
165    let (value, spans) = p.value()?;
166    p.ws()?;
167    if p.i < p.src.len() {
168        return Err(p.error("unexpected text after the value"));
169    }
170    Ok((value, spans))
171}
172
173/// `SPANS`: whether to record where each value is.
174struct Parser<'a, const SPANS: bool> {
175    src: &'a [u8],
176    text: &'a str,
177    i: usize,
178    depth: usize,
179}
180
181impl<const SPANS: bool> Parser<'_, SPANS> {
182    fn error(&self, message: impl Into<String>) -> SyntaxError {
183        self.error_at(self.i, message)
184    }
185
186    fn error_at(&self, offset: usize, message: impl Into<String>) -> SyntaxError {
187        let mut message = message.into();
188        if offset >= self.src.len() {
189            message.push_str(", found the end of the file");
190        }
191        SyntaxError {
192            message,
193            at: LineCol::of(self.text, offset),
194            offset,
195        }
196    }
197
198    fn peek(&self) -> Option<u8> {
199        self.src.get(self.i).copied()
200    }
201
202    /// Skips whitespace and comments.
203    fn ws(&mut self) -> Result<(), SyntaxError> {
204        loop {
205            let rest = &self.src[self.i..];
206            self.i += rest
207                .iter()
208                .position(|&b| !matches!(b, b' ' | b'\t' | b'\n' | b'\r'))
209                .unwrap_or(rest.len());
210            if self.peek() != Some(b'/') {
211                return Ok(());
212            }
213            match self.src.get(self.i + 1) {
214                Some(b'/') => {
215                    let rest = &self.src[self.i..];
216                    self.i += rest.iter().position(|&b| b == b'\n').unwrap_or(rest.len());
217                }
218                Some(b'*') => {
219                    let start = self.i;
220                    match self.text[self.i + 2..].find("*/") {
221                        Some(end) => self.i += 2 + end + 2,
222                        None => return Err(self.error_at(start, "unclosed `/*` comment")),
223                    }
224                }
225                _ => return Ok(()),
226            }
227        }
228    }
229
230    fn value(&mut self) -> Result<(Value, Spans), SyntaxError> {
231        let start = self.i;
232        let (value, children) = match self.peek() {
233            Some(b'{') => self.nested(|p| p.map())?,
234            Some(b'[') => self.nested(|p| p.list())?,
235            Some(b'"') => (Value::Str(self.string()?), Vec::new()),
236            Some(b'-' | b'0'..=b'9') => (self.number()?, Vec::new()),
237            Some(b't') if self.src[self.i..].starts_with(b"true") => {
238                self.i += 4;
239                (Value::Bool(true), Vec::new())
240            }
241            Some(b'f') if self.src[self.i..].starts_with(b"false") => {
242                self.i += 5;
243                (Value::Bool(false), Vec::new())
244            }
245            Some(b'n') if self.src[self.i..].starts_with(b"null") => {
246                self.i += 4;
247                (Value::Null, Vec::new())
248            }
249            Some(b'\'') => return Err(self.error("strings use double quotes")),
250            _ => return Err(self.error("expected a value")),
251        };
252        let span = Span {
253            start: start as u32,
254            end: self.i as u32,
255        };
256        Ok((
257            value,
258            Spans {
259                span,
260                key: None,
261                children,
262            },
263        ))
264    }
265
266    fn nested(
267        &mut self,
268        f: impl FnOnce(&mut Self) -> Result<(Value, Vec<Spans>), SyntaxError>,
269    ) -> Result<(Value, Vec<Spans>), SyntaxError> {
270        self.depth += 1;
271        if self.depth > MAX_DEPTH {
272            return Err(self.error(format!("nested deeper than {MAX_DEPTH} levels")));
273        }
274        let r = f(self);
275        self.depth -= 1;
276        r
277    }
278
279    fn list(&mut self) -> Result<(Value, Vec<Spans>), SyntaxError> {
280        self.i += 1;
281        let mut items = Vec::new();
282        let mut spans = Vec::new();
283        loop {
284            self.ws()?;
285            if self.peek() == Some(b']') {
286                self.i += 1;
287                break;
288            }
289            let (v, s) = self.value()?;
290            items.push(v);
291            if SPANS {
292                spans.push(s);
293            }
294            self.ws()?;
295            match self.peek() {
296                Some(b',') => self.i += 1,
297                Some(b']') => {
298                    self.i += 1;
299                    break;
300                }
301                _ => return Err(self.error("expected `,` or `]`")),
302            }
303        }
304        Ok((Value::List(items), spans))
305    }
306
307    fn map(&mut self) -> Result<(Value, Vec<Spans>), SyntaxError> {
308        self.i += 1;
309        let mut map = Map::new();
310        let mut spans = Vec::new();
311        // Duplicate keys: a scan for small objects, a set once they grow.
312        let mut seen: Option<std::collections::HashSet<String>> = None;
313        loop {
314            self.ws()?;
315            if self.peek() == Some(b'}') {
316                self.i += 1;
317                break;
318            }
319            let key_start = self.i;
320            let key = match self.peek() {
321                Some(b'"') => self.string()?,
322                Some(c) if c.is_ascii_alphabetic() || c == b'_' || c == b'$' => {
323                    while self
324                        .peek()
325                        .is_some_and(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'$')
326                    {
327                        self.i += 1;
328                    }
329                    self.text[key_start..self.i].to_string()
330                }
331                _ => return Err(self.error("expected a key")),
332            };
333            let key_span = Span {
334                start: key_start as u32,
335                end: self.i as u32,
336            };
337            let duplicate = match &mut seen {
338                Some(set) => !set.insert(key.clone()),
339                None if map.len() < 16 => map.get(&key).is_some(),
340                None => {
341                    let mut set: std::collections::HashSet<String> =
342                        map.keys().map(str::to_string).collect();
343                    let dup = !set.insert(key.clone());
344                    seen = Some(set);
345                    dup
346                }
347            };
348            if duplicate {
349                return Err(
350                    self.error_at(key_start, format!("`{key}` is already set in this object"))
351                );
352            }
353            self.ws()?;
354            if self.peek() != Some(b':') {
355                return Err(self.error("expected `:` after the key"));
356            }
357            self.i += 1;
358            self.ws()?;
359            let (v, mut s) = self.value()?;
360            s.key = Some(key_span);
361            map.push_unchecked(key.into(), v);
362            if SPANS {
363                spans.push(s);
364            }
365            self.ws()?;
366            match self.peek() {
367                Some(b',') => self.i += 1,
368                Some(b'}') => {
369                    self.i += 1;
370                    break;
371                }
372                _ => return Err(self.error("expected `,` or `}`")),
373            }
374        }
375        Ok((Value::Map(map), spans))
376    }
377
378    fn string(&mut self) -> Result<String, SyntaxError> {
379        let open = self.i;
380        self.i += 1;
381        let mut out = String::new();
382        loop {
383            let rest = &self.src[self.i..];
384            let Some(n) = rest
385                .iter()
386                .position(|&b| b == b'"' || b == b'\\' || b < 0x20)
387            else {
388                return Err(self.error_at(open, "unclosed string"));
389            };
390            out.push_str(&self.text[self.i..self.i + n]);
391            self.i += n;
392            match self.src[self.i] {
393                b'"' => {
394                    self.i += 1;
395                    return Ok(out);
396                }
397                b'\\' => self.escape(&mut out)?,
398                b'\n' => return Err(self.error("line break in a string (write `\\n`)")),
399                _ => return Err(self.error("control character in a string")),
400            }
401        }
402    }
403
404    fn escape(&mut self, out: &mut String) -> Result<(), SyntaxError> {
405        let at = self.i;
406        self.i += 1;
407        let c = self.peek().ok_or_else(|| self.error("unclosed string"))?;
408        self.i += 1;
409        match c {
410            b'"' => out.push('"'),
411            b'\\' => out.push('\\'),
412            b'/' => out.push('/'),
413            b'b' => out.push('\u{8}'),
414            b'f' => out.push('\u{c}'),
415            b'n' => out.push('\n'),
416            b'r' => out.push('\r'),
417            b't' => out.push('\t'),
418            b'u' => {
419                let hi = self.hex4(at)?;
420                let ch = if (0xD800..0xDC00).contains(&hi) {
421                    if !self.src[self.i..].starts_with(b"\\u") {
422                        return Err(self.error_at(at, "unpaired surrogate in `\\u` escape"));
423                    }
424                    self.i += 2;
425                    let lo = self.hex4(at)?;
426                    if !(0xDC00..0xE000).contains(&lo) {
427                        return Err(self.error_at(at, "unpaired surrogate in `\\u` escape"));
428                    }
429                    char::from_u32(0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00))
430                } else {
431                    char::from_u32(hi)
432                };
433                out.push(
434                    ch.ok_or_else(|| self.error_at(at, "unpaired surrogate in `\\u` escape"))?,
435                );
436            }
437            _ => {
438                return Err(self.error_at(
439                    at,
440                    "unknown escape (`\\n`, `\\t`, `\\\"`, `\\\\`, `\\uXXXX`, ...)",
441                ));
442            }
443        }
444        Ok(())
445    }
446
447    fn hex4(&mut self, at: usize) -> Result<u32, SyntaxError> {
448        let digits = self
449            .text
450            .get(self.i..self.i + 4)
451            .filter(|d| d.bytes().all(|b| b.is_ascii_hexdigit()));
452        let d = digits.ok_or_else(|| self.error_at(at, "`\\u` needs four hex digits"))?;
453        let code = u32::from_str_radix(d, HEX_RADIX)
454            .map_err(|_| self.error_at(at, "`\\u` needs four hex digits"))?;
455        self.i += 4;
456        Ok(code)
457    }
458
459    fn number(&mut self) -> Result<Value, SyntaxError> {
460        let start = self.i;
461        if self.peek() == Some(b'-') {
462            self.i += 1;
463        }
464        let digits = |p: &mut Self| {
465            let s = p.i;
466            while p.peek().is_some_and(|c| c.is_ascii_digit()) {
467                p.i += 1;
468            }
469            p.i - s
470        };
471        let int_start = self.i;
472        if digits(self) == 0 {
473            return Err(self.error("expected digits"));
474        }
475        if self.src[int_start] == b'0' && self.i - int_start > 1 {
476            return Err(self.error_at(int_start, "numbers don't start with 0"));
477        }
478        let mut float = false;
479        if self.peek() == Some(b'.') {
480            self.i += 1;
481            float = true;
482            if digits(self) == 0 {
483                return Err(self.error("expected digits after `.`"));
484            }
485        }
486        if matches!(self.peek(), Some(b'e' | b'E')) {
487            self.i += 1;
488            float = true;
489            if matches!(self.peek(), Some(b'+' | b'-')) {
490                self.i += 1;
491            }
492            if digits(self) == 0 {
493                return Err(self.error("expected digits in the exponent"));
494            }
495        }
496        let s = &self.text[start..self.i];
497        if !float && let Ok(n) = s.parse::<i64>() {
498            return Ok(Value::Int(n));
499        }
500        Ok(Value::Float(
501            s.parse::<f64>()
502                .map_err(|_| self.error_at(start, "invalid number"))?,
503        ))
504    }
505}
506
507// ── Writing ─────────────────────────────────────────────────────────────────
508
509/// `value` as compact text (typed values in their string forms).
510pub fn to_string(value: &Value) -> String {
511    let mut out = String::with_capacity(256);
512    write(value, None, 0, &mut out);
513    out
514}
515
516/// `value` as text indented by two spaces, one entry per line.
517pub fn to_string_pretty(value: &Value) -> String {
518    let mut out = String::new();
519    write(value, Some("  "), 0, &mut out);
520    out.push('\n');
521    out
522}
523
524fn write(value: &Value, indent: Option<&str>, level: usize, out: &mut String) {
525    let newline = |out: &mut String, level: usize| {
526        if let Some(unit) = indent {
527            out.push('\n');
528            for _ in 0..level {
529                out.push_str(unit);
530            }
531        }
532    };
533    match value {
534        Value::Null => out.push_str("null"),
535        Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
536        Value::Int(n) => {
537            use std::fmt::Write as _;
538            let _ = write!(out, "{n}");
539        }
540        Value::Float(f) => write_float(*f, out),
541        Value::List(items) => {
542            if items.is_empty() {
543                out.push_str("[]");
544                return;
545            }
546            out.push('[');
547            for (i, v) in items.iter().enumerate() {
548                if i > 0 {
549                    out.push(',');
550                }
551                newline(out, level + 1);
552                write(v, indent, level + 1, out);
553            }
554            newline(out, level);
555            out.push(']');
556        }
557        Value::Map(map) => {
558            if map.is_empty() {
559                out.push_str("{}");
560                return;
561            }
562            out.push('{');
563            for (i, (k, v)) in map.iter().enumerate() {
564                if i > 0 {
565                    out.push(',');
566                }
567                newline(out, level + 1);
568                write_string(k, out);
569                out.push_str(if indent.is_some() { ": " } else { ":" });
570                write(v, indent, level + 1, out);
571            }
572            newline(out, level);
573            out.push('}');
574        }
575        Value::Str(s) | Value::Text(s) => {
576            if s.starts_with('@') {
577                out.push('"');
578                out.push('@');
579                write_string_body(s, out);
580                out.push('"');
581            } else {
582                write_string(s, out);
583            }
584        }
585        typed => write_string(&crate::typed::typed_string(typed).unwrap_or_default(), out),
586    }
587}
588
589fn write_float(f: f64, out: &mut String) {
590    if !f.is_finite() {
591        out.push_str("null");
592    } else if f.fract() == 0.0 && f.abs() < 1e16 {
593        out.push_str(&format!("{f:.1}"));
594    } else {
595        out.push_str(&f.to_string());
596    }
597}
598
599/// `s` as a JSON string literal.
600pub fn write_string(s: &str, out: &mut String) {
601    out.push('"');
602    write_string_body(s, out);
603    out.push('"');
604}
605
606fn write_string_body(s: &str, out: &mut String) {
607    let bytes = s.as_bytes();
608    let mut run = 0;
609    while let Some(n) = bytes[run..]
610        .iter()
611        .position(|&b| b == b'"' || b == b'\\' || b < 0x20)
612    {
613        let i = run + n;
614        out.push_str(&s[run..i]);
615        match bytes[i] {
616            b'"' => out.push_str("\\\""),
617            b'\\' => out.push_str("\\\\"),
618            b'\n' => out.push_str("\\n"),
619            b'\r' => out.push_str("\\r"),
620            b'\t' => out.push_str("\\t"),
621            b => {
622                use std::fmt::Write as _;
623                let _ = write!(out, "\\u{b:04x}");
624            }
625        }
626        run = i + 1;
627    }
628    out.push_str(&s[run..]);
629}
630
631#[cfg(test)]
632mod tests {
633    use super::*;
634
635    #[test]
636    fn json5_subset() {
637        let doc = parse("\u{feff}{\n  // comment\n  name: \"Ruby\", /* inline */ $schema: 1,\n  \"list\": [1, 2.5, -3e2, true, null,],\n}").unwrap();
638        let m = doc.value.as_map().unwrap();
639        assert_eq!(m.keys().collect::<Vec<_>>(), ["name", "$schema", "list"]);
640        assert_eq!(
641            doc.value.get("list").unwrap(),
642            &Value::List(vec![
643                Value::Int(1),
644                Value::Float(2.5),
645                Value::Float(-300.0),
646                Value::Bool(true),
647                Value::Null
648            ])
649        );
650    }
651
652    #[test]
653    fn locates_values_and_keys() {
654        let text = "{\n  \"a\": [\n    1,\n    {\"b\": \"x\"}\n  ]\n}";
655        let doc = parse(text).unwrap();
656        let path = Path::root().key("a").index(1).key("b");
657        assert_eq!(doc.locate(text, &path), LineCol { line: 4, col: 11 });
658        assert_eq!(
659            LineCol::of(text, doc.spans_at(&path).key.unwrap().start as usize),
660            LineCol { line: 4, col: 6 }
661        );
662        // A path past the document: its deepest existing part.
663        assert_eq!(
664            doc.locate(text, &Path::root().key("a").index(9)),
665            LineCol { line: 2, col: 8 }
666        );
667    }
668
669    #[test]
670    fn errors_say_where() {
671        let cases = [
672            ("{\"a\": 1,, }", "1:9", "expected a key"),
673            ("{\"a\": 1 \"b\": 2}", "1:9", "expected `,` or `}`"),
674            ("{\"a\": 1, \"a\": 2}", "1:10", "already set"),
675            ("[1, 2", "1:6", "found the end of the file"),
676            ("\"ab\ncd\"", "1:4", "line break"),
677            ("'x'", "1:1", "double quotes"),
678            ("01", "1:1", "start with 0"),
679            ("{} x", "1:4", "unexpected text"),
680            ("/* never closed", "1:1", "unclosed"),
681            ("\"\\q\"", "1:2", "unknown escape"),
682            ("\"\\ud800\"", "1:2", "surrogate"),
683            ("{\n  \"é\": tru }", "2:8", "expected a value"),
684        ];
685        for (text, at, msg) in cases {
686            let e = parse(text).unwrap_err();
687            assert_eq!(e.at.to_string(), at, "{text:?}: {e}");
688            assert!(e.message.contains(msg), "{text:?}: {e}");
689        }
690        assert!(
691            parse(&"[".repeat(MAX_DEPTH + 1))
692                .unwrap_err()
693                .message
694                .contains("nested deeper")
695        );
696    }
697
698    #[test]
699    fn strings_and_numbers() {
700        let v = parse_value(
701            r#"["a\"b\\c\n\u00e9\ud83d\ude00", 9223372036854775807, 9223372036854775808, 0.1]"#,
702        )
703        .unwrap();
704        let l = v.as_list().unwrap();
705        assert_eq!(l[0], Value::Str("a\"b\\c\né😀".into()));
706        assert_eq!(l[1], Value::Int(i64::MAX));
707        assert_eq!(l[2], Value::Float(9223372036854775808.0));
708        assert_eq!(l[3], Value::Float(0.1));
709    }
710
711    #[test]
712    fn writes_what_it_reads() {
713        let text = "{\n  \"name\": \"Ruby \\\"red\\\"\\n\",\n  \"n\": [1, 2.0, 0.1, -5],\n  \"empty\": {},\n  \"none\": [],\n  \"tab\\u0001\": null\n}\n";
714        let v = parse_value(text).unwrap();
715        assert_eq!(parse_value(&to_string(&v)).unwrap(), v);
716        let pretty = to_string_pretty(&v);
717        assert_eq!(parse_value(&pretty).unwrap(), v);
718        assert!(
719            pretty.starts_with(
720                "{\n  \"name\": \"Ruby \\\"red\\\"\\n\",\n  \"n\": [\n    1,\n    2.0,"
721            ),
722            "{pretty}"
723        );
724        assert_eq!(
725            to_string(&v),
726            r#"{"name":"Ruby \"red\"\n","n":[1,2.0,0.1,-5],"empty":{},"none":[],"tab\u0001":null}"#
727        );
728    }
729
730    #[test]
731    fn typed_values_are_strings_in_text() {
732        let v = Value::List(vec![
733            Value::Ref(crate::Ref::new("item", "minecraft:stick")),
734            Value::Color(0xFF112233),
735            Value::Str("@not a ref".into()),
736        ]);
737        assert_eq!(
738            to_string(&v),
739            r##"["@item:minecraft:stick","#112233","@@not a ref"]"##
740        );
741    }
742}