yog_data/
text.rs

1//! The text syntax: JSON, plus `//` and `/* */` comments, trailing commas
2//! and unquoted keys (the part of JSON5 editors already understand).
3//!
4//! [`parse`] keeps where every value is ([`Spans`]), so a schema error at
5//! `entries[2].pages[0].item` can point at its line. Typed values stay
6//! strings here — the schema decides what a string means.
7
8use std::fmt;
9
10/// `\\u` escapes are hexadecimal.
11const HEX_RADIX: u32 = 16;
12
13use crate::value::{Map, Path, Seg, Value};
14
15mod comments;
16
17pub use comments::with_comments;
18
19/// Deeper than this is an error, not a stack overflow.
20pub const MAX_DEPTH: usize = 256;
21
22/// A byte range of the source.
23#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
24pub struct Span {
25    pub start: u32,
26    pub end: u32,
27}
28
29/// Where each value of a parsed document is: the value's span, its key's
30/// (map entries), and its children's, in the same order as the value's.
31#[derive(Debug, Clone, Default, PartialEq)]
32pub struct Spans {
33    pub span: Span,
34    pub key: Option<Span>,
35    pub children: Vec<Spans>,
36}
37
38/// A parsed document: the value and where each of its parts is.
39#[derive(Debug, Clone, PartialEq)]
40pub struct Doc {
41    pub value: Value,
42    pub spans: Spans,
43}
44
45impl Doc {
46    /// The spans of the value at `path` — the closest existing parent's if
47    /// the path goes further than the document.
48    pub fn spans_at(&self, path: &Path) -> &Spans {
49        let mut v = &self.value;
50        let mut s = &self.spans;
51        for seg in path.segments() {
52            let i = match (seg, v) {
53                (Seg::Key(k), Value::Map(m)) => m.index_of(k),
54                (Seg::Index(i), Value::List(l)) if *i < l.len() => Some(*i),
55                _ => None,
56            };
57            let Some(i) = i else { break };
58            let Some(child) = s.children.get(i) else {
59                break;
60            };
61            v = match v {
62                Value::Map(m) => m.value_at(i).unwrap_or(v),
63                Value::List(l) => &l[i],
64                _ => v,
65            };
66            s = child;
67        }
68        s
69    }
70
71    /// The line and column of the value at `path` in `text` (the source this was parsed from).
72    pub fn locate(&self, text: &str, path: &Path) -> LineCol {
73        LineCol::of(text, self.spans_at(path).span.start as usize)
74    }
75}
76
77/// A 1-based line and column (in characters).
78#[derive(Debug, Clone, Copy, PartialEq, Eq)]
79pub struct LineCol {
80    pub line: u32,
81    pub col: u32,
82}
83
84impl LineCol {
85    /// Where byte `offset` of `text` is.
86    pub fn of(text: &str, offset: usize) -> LineCol {
87        let offset = offset.min(text.len());
88        let before = &text.as_bytes()[..offset];
89        let line = before.iter().filter(|&&b| b == b'\n').count() as u32 + 1;
90        let line_start = before
91            .iter()
92            .rposition(|&b| b == b'\n')
93            .map_or(0, |i| i + 1);
94        let col = text
95            .get(line_start..offset)
96            .map_or(offset - line_start, |s| s.chars().count()) as u32
97            + 1;
98        LineCol { line, col }
99    }
100}
101
102impl fmt::Display for LineCol {
103    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
104        write!(f, "{}:{}", self.line, self.col)
105    }
106}
107
108/// A syntax error.
109#[derive(Debug, Clone, PartialEq, Eq)]
110pub struct SyntaxError {
111    pub message: String,
112    pub at: LineCol,
113    pub offset: usize,
114}
115
116impl fmt::Display for SyntaxError {
117    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
118        write!(f, "{}: {}", self.at, self.message)
119    }
120}
121
122impl std::error::Error for SyntaxError {}
123
124/// Parses Yog Data text.
125pub fn parse(text: &str) -> Result<Doc, SyntaxError> {
126    let (value, spans) = run::<true>(text)?;
127    Ok(Doc { value, spans })
128}
129
130/// Parses Yog Data text, without the positions (faster).
131pub fn parse_value(text: &str) -> Result<Value, SyntaxError> {
132    run::<false>(text).map(|(v, _)| v)
133}
134
135fn run<const SPANS: bool>(text: &str) -> Result<(Value, Spans), SyntaxError> {
136    let mut p = Parser::<SPANS> {
137        src: text.as_bytes(),
138        text,
139        i: 0,
140        depth: 0,
141    };
142    if text.starts_with('\u{feff}') {
143        p.i = 3;
144    }
145    p.ws()?;
146    let (value, spans) = p.value()?;
147    p.ws()?;
148    if p.i < p.src.len() {
149        return Err(p.error("unexpected text after the value"));
150    }
151    Ok((value, spans))
152}
153
154/// `SPANS`: whether to record where each value is.
155struct Parser<'a, const SPANS: bool> {
156    src: &'a [u8],
157    text: &'a str,
158    i: usize,
159    depth: usize,
160}
161
162impl<const SPANS: bool> Parser<'_, SPANS> {
163    fn error(&self, message: impl Into<String>) -> SyntaxError {
164        self.error_at(self.i, message)
165    }
166
167    fn error_at(&self, offset: usize, message: impl Into<String>) -> SyntaxError {
168        let mut message = message.into();
169        if offset >= self.src.len() {
170            message.push_str(", found the end of the file");
171        }
172        SyntaxError {
173            message,
174            at: LineCol::of(self.text, offset),
175            offset,
176        }
177    }
178
179    fn peek(&self) -> Option<u8> {
180        self.src.get(self.i).copied()
181    }
182
183    /// Skips whitespace and comments.
184    fn ws(&mut self) -> Result<(), SyntaxError> {
185        loop {
186            let rest = &self.src[self.i..];
187            self.i += rest
188                .iter()
189                .position(|&b| !matches!(b, b' ' | b'\t' | b'\n' | b'\r'))
190                .unwrap_or(rest.len());
191            if self.peek() != Some(b'/') {
192                return Ok(());
193            }
194            match self.src.get(self.i + 1) {
195                Some(b'/') => {
196                    let rest = &self.src[self.i..];
197                    self.i += rest.iter().position(|&b| b == b'\n').unwrap_or(rest.len());
198                }
199                Some(b'*') => {
200                    let start = self.i;
201                    match self.text[self.i + 2..].find("*/") {
202                        Some(end) => self.i += 2 + end + 2,
203                        None => return Err(self.error_at(start, "unclosed `/*` comment")),
204                    }
205                }
206                _ => return Ok(()),
207            }
208        }
209    }
210
211    fn value(&mut self) -> Result<(Value, Spans), SyntaxError> {
212        let start = self.i;
213        let (value, children) = match self.peek() {
214            Some(b'{') => self.nested(|p| p.map())?,
215            Some(b'[') => self.nested(|p| p.list())?,
216            Some(b'"') => (Value::Str(self.string()?), Vec::new()),
217            Some(b'-' | b'0'..=b'9') => (self.number()?, Vec::new()),
218            Some(b't') if self.src[self.i..].starts_with(b"true") => {
219                self.i += 4;
220                (Value::Bool(true), Vec::new())
221            }
222            Some(b'f') if self.src[self.i..].starts_with(b"false") => {
223                self.i += 5;
224                (Value::Bool(false), Vec::new())
225            }
226            Some(b'n') if self.src[self.i..].starts_with(b"null") => {
227                self.i += 4;
228                (Value::Null, Vec::new())
229            }
230            Some(b'\'') => return Err(self.error("strings use double quotes")),
231            _ => return Err(self.error("expected a value")),
232        };
233        let span = Span {
234            start: start as u32,
235            end: self.i as u32,
236        };
237        Ok((
238            value,
239            Spans {
240                span,
241                key: None,
242                children,
243            },
244        ))
245    }
246
247    fn nested(
248        &mut self,
249        f: impl FnOnce(&mut Self) -> Result<(Value, Vec<Spans>), SyntaxError>,
250    ) -> Result<(Value, Vec<Spans>), SyntaxError> {
251        self.depth += 1;
252        if self.depth > MAX_DEPTH {
253            return Err(self.error(format!("nested deeper than {MAX_DEPTH} levels")));
254        }
255        let r = f(self);
256        self.depth -= 1;
257        r
258    }
259
260    fn list(&mut self) -> Result<(Value, Vec<Spans>), SyntaxError> {
261        self.i += 1;
262        let mut items = Vec::new();
263        let mut spans = Vec::new();
264        loop {
265            self.ws()?;
266            if self.peek() == Some(b']') {
267                self.i += 1;
268                break;
269            }
270            let (v, s) = self.value()?;
271            items.push(v);
272            if SPANS {
273                spans.push(s);
274            }
275            self.ws()?;
276            match self.peek() {
277                Some(b',') => self.i += 1,
278                Some(b']') => {
279                    self.i += 1;
280                    break;
281                }
282                _ => return Err(self.error("expected `,` or `]`")),
283            }
284        }
285        Ok((Value::List(items), spans))
286    }
287
288    fn map(&mut self) -> Result<(Value, Vec<Spans>), SyntaxError> {
289        self.i += 1;
290        let mut map = Map::new();
291        let mut spans = Vec::new();
292        // Duplicate keys: a scan for small objects, a set once they grow.
293        let mut seen: Option<std::collections::HashSet<String>> = None;
294        loop {
295            self.ws()?;
296            if self.peek() == Some(b'}') {
297                self.i += 1;
298                break;
299            }
300            let key_start = self.i;
301            let key = match self.peek() {
302                Some(b'"') => self.string()?,
303                Some(c) if c.is_ascii_alphabetic() || c == b'_' || c == b'$' => {
304                    while self
305                        .peek()
306                        .is_some_and(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'$')
307                    {
308                        self.i += 1;
309                    }
310                    self.text[key_start..self.i].to_string()
311                }
312                _ => return Err(self.error("expected a key")),
313            };
314            let key_span = Span {
315                start: key_start as u32,
316                end: self.i as u32,
317            };
318            let duplicate = match &mut seen {
319                Some(set) => !set.insert(key.clone()),
320                None if map.len() < 16 => map.get(&key).is_some(),
321                None => {
322                    let mut set: std::collections::HashSet<String> =
323                        map.keys().map(str::to_string).collect();
324                    let dup = !set.insert(key.clone());
325                    seen = Some(set);
326                    dup
327                }
328            };
329            if duplicate {
330                return Err(
331                    self.error_at(key_start, format!("`{key}` is already set in this object"))
332                );
333            }
334            self.ws()?;
335            if self.peek() != Some(b':') {
336                return Err(self.error("expected `:` after the key"));
337            }
338            self.i += 1;
339            self.ws()?;
340            let (v, mut s) = self.value()?;
341            s.key = Some(key_span);
342            map.push_unchecked(key.into(), v);
343            if SPANS {
344                spans.push(s);
345            }
346            self.ws()?;
347            match self.peek() {
348                Some(b',') => self.i += 1,
349                Some(b'}') => {
350                    self.i += 1;
351                    break;
352                }
353                _ => return Err(self.error("expected `,` or `}`")),
354            }
355        }
356        Ok((Value::Map(map), spans))
357    }
358
359    fn string(&mut self) -> Result<String, SyntaxError> {
360        let open = self.i;
361        self.i += 1;
362        let mut out = String::new();
363        loop {
364            let rest = &self.src[self.i..];
365            let Some(n) = rest
366                .iter()
367                .position(|&b| b == b'"' || b == b'\\' || b < 0x20)
368            else {
369                return Err(self.error_at(open, "unclosed string"));
370            };
371            out.push_str(&self.text[self.i..self.i + n]);
372            self.i += n;
373            match self.src[self.i] {
374                b'"' => {
375                    self.i += 1;
376                    return Ok(out);
377                }
378                b'\\' => self.escape(&mut out)?,
379                b'\n' => return Err(self.error("line break in a string (write `\\n`)")),
380                _ => return Err(self.error("control character in a string")),
381            }
382        }
383    }
384
385    fn escape(&mut self, out: &mut String) -> Result<(), SyntaxError> {
386        let at = self.i;
387        self.i += 1;
388        let c = self.peek().ok_or_else(|| self.error("unclosed string"))?;
389        self.i += 1;
390        match c {
391            b'"' => out.push('"'),
392            b'\\' => out.push('\\'),
393            b'/' => out.push('/'),
394            b'b' => out.push('\u{8}'),
395            b'f' => out.push('\u{c}'),
396            b'n' => out.push('\n'),
397            b'r' => out.push('\r'),
398            b't' => out.push('\t'),
399            b'u' => {
400                let hi = self.hex4(at)?;
401                let ch = if (0xD800..0xDC00).contains(&hi) {
402                    if !self.src[self.i..].starts_with(b"\\u") {
403                        return Err(self.error_at(at, "unpaired surrogate in `\\u` escape"));
404                    }
405                    self.i += 2;
406                    let lo = self.hex4(at)?;
407                    if !(0xDC00..0xE000).contains(&lo) {
408                        return Err(self.error_at(at, "unpaired surrogate in `\\u` escape"));
409                    }
410                    char::from_u32(0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00))
411                } else {
412                    char::from_u32(hi)
413                };
414                out.push(
415                    ch.ok_or_else(|| self.error_at(at, "unpaired surrogate in `\\u` escape"))?,
416                );
417            }
418            _ => {
419                return Err(self.error_at(
420                    at,
421                    "unknown escape (`\\n`, `\\t`, `\\\"`, `\\\\`, `\\uXXXX`, ...)",
422                ));
423            }
424        }
425        Ok(())
426    }
427
428    fn hex4(&mut self, at: usize) -> Result<u32, SyntaxError> {
429        let digits = self
430            .text
431            .get(self.i..self.i + 4)
432            .filter(|d| d.bytes().all(|b| b.is_ascii_hexdigit()));
433        let d = digits.ok_or_else(|| self.error_at(at, "`\\u` needs four hex digits"))?;
434        let code = u32::from_str_radix(d, HEX_RADIX)
435            .map_err(|_| self.error_at(at, "`\\u` needs four hex digits"))?;
436        self.i += 4;
437        Ok(code)
438    }
439
440    fn number(&mut self) -> Result<Value, SyntaxError> {
441        let start = self.i;
442        if self.peek() == Some(b'-') {
443            self.i += 1;
444        }
445        let digits = |p: &mut Self| {
446            let s = p.i;
447            while p.peek().is_some_and(|c| c.is_ascii_digit()) {
448                p.i += 1;
449            }
450            p.i - s
451        };
452        let int_start = self.i;
453        if digits(self) == 0 {
454            return Err(self.error("expected digits"));
455        }
456        if self.src[int_start] == b'0' && self.i - int_start > 1 {
457            return Err(self.error_at(int_start, "numbers don't start with 0"));
458        }
459        let mut float = false;
460        if self.peek() == Some(b'.') {
461            self.i += 1;
462            float = true;
463            if digits(self) == 0 {
464                return Err(self.error("expected digits after `.`"));
465            }
466        }
467        if matches!(self.peek(), Some(b'e' | b'E')) {
468            self.i += 1;
469            float = true;
470            if matches!(self.peek(), Some(b'+' | b'-')) {
471                self.i += 1;
472            }
473            if digits(self) == 0 {
474                return Err(self.error("expected digits in the exponent"));
475            }
476        }
477        let s = &self.text[start..self.i];
478        if !float && let Ok(n) = s.parse::<i64>() {
479            return Ok(Value::Int(n));
480        }
481        Ok(Value::Float(
482            s.parse::<f64>()
483                .map_err(|_| self.error_at(start, "invalid number"))?,
484        ))
485    }
486}
487
488// ── Writing ─────────────────────────────────────────────────────────────────
489
490/// `value` as compact text (typed values in their string forms).
491pub fn to_string(value: &Value) -> String {
492    let mut out = String::with_capacity(256);
493    write(value, None, 0, &mut out);
494    out
495}
496
497/// `value` as text indented by two spaces, one entry per line.
498pub fn to_string_pretty(value: &Value) -> String {
499    let mut out = String::new();
500    write(value, Some("  "), 0, &mut out);
501    out.push('\n');
502    out
503}
504
505fn write(value: &Value, indent: Option<&str>, level: usize, out: &mut String) {
506    let newline = |out: &mut String, level: usize| {
507        if let Some(unit) = indent {
508            out.push('\n');
509            for _ in 0..level {
510                out.push_str(unit);
511            }
512        }
513    };
514    match value {
515        Value::Null => out.push_str("null"),
516        Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
517        Value::Int(n) => {
518            use std::fmt::Write as _;
519            let _ = write!(out, "{n}");
520        }
521        Value::Float(f) => write_float(*f, out),
522        Value::List(items) => {
523            if items.is_empty() {
524                out.push_str("[]");
525                return;
526            }
527            out.push('[');
528            for (i, v) in items.iter().enumerate() {
529                if i > 0 {
530                    out.push(',');
531                }
532                newline(out, level + 1);
533                write(v, indent, level + 1, out);
534            }
535            newline(out, level);
536            out.push(']');
537        }
538        Value::Map(map) => {
539            if map.is_empty() {
540                out.push_str("{}");
541                return;
542            }
543            out.push('{');
544            for (i, (k, v)) in map.iter().enumerate() {
545                if i > 0 {
546                    out.push(',');
547                }
548                newline(out, level + 1);
549                write_string(k, out);
550                out.push_str(if indent.is_some() { ": " } else { ":" });
551                write(v, indent, level + 1, out);
552            }
553            newline(out, level);
554            out.push('}');
555        }
556        Value::Str(s) | Value::Text(s) => {
557            if s.starts_with('@') {
558                out.push('"');
559                out.push('@');
560                write_string_body(s, out);
561                out.push('"');
562            } else {
563                write_string(s, out);
564            }
565        }
566        typed => write_string(&crate::typed::typed_string(typed).unwrap_or_default(), out),
567    }
568}
569
570fn write_float(f: f64, out: &mut String) {
571    if !f.is_finite() {
572        out.push_str("null");
573    } else if f.fract() == 0.0 && f.abs() < 1e16 {
574        out.push_str(&format!("{f:.1}"));
575    } else {
576        out.push_str(&f.to_string());
577    }
578}
579
580/// `s` as a JSON string literal.
581pub fn write_string(s: &str, out: &mut String) {
582    out.push('"');
583    write_string_body(s, out);
584    out.push('"');
585}
586
587fn write_string_body(s: &str, out: &mut String) {
588    let bytes = s.as_bytes();
589    let mut run = 0;
590    while let Some(n) = bytes[run..]
591        .iter()
592        .position(|&b| b == b'"' || b == b'\\' || b < 0x20)
593    {
594        let i = run + n;
595        out.push_str(&s[run..i]);
596        match bytes[i] {
597            b'"' => out.push_str("\\\""),
598            b'\\' => out.push_str("\\\\"),
599            b'\n' => out.push_str("\\n"),
600            b'\r' => out.push_str("\\r"),
601            b'\t' => out.push_str("\\t"),
602            b => {
603                use std::fmt::Write as _;
604                let _ = write!(out, "\\u{b:04x}");
605            }
606        }
607        run = i + 1;
608    }
609    out.push_str(&s[run..]);
610}
611
612#[cfg(test)]
613mod tests {
614    use super::*;
615
616    #[test]
617    fn json5_subset() {
618        let doc = parse("\u{feff}{\n  // comment\n  name: \"Ruby\", /* inline */ $schema: 1,\n  \"list\": [1, 2.5, -3e2, true, null,],\n}").unwrap();
619        let m = doc.value.as_map().unwrap();
620        assert_eq!(m.keys().collect::<Vec<_>>(), ["name", "$schema", "list"]);
621        assert_eq!(
622            doc.value.get("list").unwrap(),
623            &Value::List(vec![
624                Value::Int(1),
625                Value::Float(2.5),
626                Value::Float(-300.0),
627                Value::Bool(true),
628                Value::Null
629            ])
630        );
631    }
632
633    #[test]
634    fn locates_values_and_keys() {
635        let text = "{\n  \"a\": [\n    1,\n    {\"b\": \"x\"}\n  ]\n}";
636        let doc = parse(text).unwrap();
637        let path = Path::root().key("a").index(1).key("b");
638        assert_eq!(doc.locate(text, &path), LineCol { line: 4, col: 11 });
639        assert_eq!(
640            LineCol::of(text, doc.spans_at(&path).key.unwrap().start as usize),
641            LineCol { line: 4, col: 6 }
642        );
643        // A path past the document: its deepest existing part.
644        assert_eq!(
645            doc.locate(text, &Path::root().key("a").index(9)),
646            LineCol { line: 2, col: 8 }
647        );
648    }
649
650    #[test]
651    fn errors_say_where() {
652        let cases = [
653            ("{\"a\": 1,, }", "1:9", "expected a key"),
654            ("{\"a\": 1 \"b\": 2}", "1:9", "expected `,` or `}`"),
655            ("{\"a\": 1, \"a\": 2}", "1:10", "already set"),
656            ("[1, 2", "1:6", "found the end of the file"),
657            ("\"ab\ncd\"", "1:4", "line break"),
658            ("'x'", "1:1", "double quotes"),
659            ("01", "1:1", "start with 0"),
660            ("{} x", "1:4", "unexpected text"),
661            ("/* never closed", "1:1", "unclosed"),
662            ("\"\\q\"", "1:2", "unknown escape"),
663            ("\"\\ud800\"", "1:2", "surrogate"),
664            ("{\n  \"é\": tru }", "2:8", "expected a value"),
665        ];
666        for (text, at, msg) in cases {
667            let e = parse(text).unwrap_err();
668            assert_eq!(e.at.to_string(), at, "{text:?}: {e}");
669            assert!(e.message.contains(msg), "{text:?}: {e}");
670        }
671        assert!(
672            parse(&"[".repeat(MAX_DEPTH + 1))
673                .unwrap_err()
674                .message
675                .contains("nested deeper")
676        );
677    }
678
679    #[test]
680    fn strings_and_numbers() {
681        let v = parse_value(
682            r#"["a\"b\\c\n\u00e9\ud83d\ude00", 9223372036854775807, 9223372036854775808, 0.1]"#,
683        )
684        .unwrap();
685        let l = v.as_list().unwrap();
686        assert_eq!(l[0], Value::Str("a\"b\\c\né😀".into()));
687        assert_eq!(l[1], Value::Int(i64::MAX));
688        assert_eq!(l[2], Value::Float(9223372036854775808.0));
689        assert_eq!(l[3], Value::Float(0.1));
690    }
691
692    #[test]
693    fn writes_what_it_reads() {
694        let text = "{\n  \"name\": \"Ruby \\\"red\\\"\\n\",\n  \"n\": [1, 2.0, 0.1, -5],\n  \"empty\": {},\n  \"none\": [],\n  \"tab\\u0001\": null\n}\n";
695        let v = parse_value(text).unwrap();
696        assert_eq!(parse_value(&to_string(&v)).unwrap(), v);
697        let pretty = to_string_pretty(&v);
698        assert_eq!(parse_value(&pretty).unwrap(), v);
699        assert!(
700            pretty.starts_with(
701                "{\n  \"name\": \"Ruby \\\"red\\\"\\n\",\n  \"n\": [\n    1,\n    2.0,"
702            ),
703            "{pretty}"
704        );
705        assert_eq!(
706            to_string(&v),
707            r#"{"name":"Ruby \"red\"\n","n":[1,2.0,0.1,-5],"empty":{},"none":[],"tab\u0001":null}"#
708        );
709    }
710
711    #[test]
712    fn typed_values_are_strings_in_text() {
713        let v = Value::List(vec![
714            Value::Ref(crate::Ref::new("item", "minecraft:stick")),
715            Value::Color(0xFF112233),
716            Value::Str("@not a ref".into()),
717        ]);
718        assert_eq!(
719            to_string(&v),
720            r##"["@item:minecraft:stick","#112233","@@not a ref"]"##
721        );
722    }
723}
yog_data/text.rs

yog_data/
text.rs
