yog_data/
text.rs

1//! The text syntax: JSON, plus `//` and `/* */` comments, trailing commas
2//! and unquoted keys (the part of JSON5 editors already understand).
3//!
4//! [`parse`] keeps where every value is ([`Spans`]), so a schema error at
5//! `entries[2].pages[0].item` can point at its line. Typed values stay
6//! strings here — the schema decides what a string means.
7
8use std::fmt;
9
10use crate::value::{Map, Path, Seg, Value};
11
12/// Deeper than this is an error, not a stack overflow.
13pub const MAX_DEPTH: usize = 256;
14
15/// A byte range of the source.
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
17pub struct Span {
18    pub start: u32,
19    pub end: u32,
20}
21
22/// Where each value of a parsed document is: the value's span, its key's
23/// (map entries), and its children's, in the same order as the value's.
24#[derive(Debug, Clone, Default, PartialEq)]
25pub struct Spans {
26    pub span: Span,
27    pub key: Option<Span>,
28    pub children: Vec<Spans>,
29}
30
31/// A parsed document: the value and where each of its parts is.
32#[derive(Debug, Clone, PartialEq)]
33pub struct Doc {
34    pub value: Value,
35    pub spans: Spans,
36}
37
38impl Doc {
39    /// The spans of the value at `path` — the closest existing parent's if
40    /// the path goes further than the document.
41    pub fn spans_at(&self, path: &Path) -> &Spans {
42        let mut v = &self.value;
43        let mut s = &self.spans;
44        for seg in path.segments() {
45            let i = match (seg, v) {
46                (Seg::Key(k), Value::Map(m)) => m.index_of(k),
47                (Seg::Index(i), Value::List(l)) if *i < l.len() => Some(*i),
48                _ => None,
49            };
50            let Some(i) = i else { break };
51            let Some(child) = s.children.get(i) else {
52                break;
53            };
54            v = match v {
55                Value::Map(m) => m.value_at(i).unwrap_or(v),
56                Value::List(l) => &l[i],
57                _ => v,
58            };
59            s = child;
60        }
61        s
62    }
63
64    /// The line and column of the value at `path` in `text` (the source this was parsed from).
65    pub fn locate(&self, text: &str, path: &Path) -> LineCol {
66        LineCol::of(text, self.spans_at(path).span.start as usize)
67    }
68}
69
70/// A 1-based line and column (in characters).
71#[derive(Debug, Clone, Copy, PartialEq, Eq)]
72pub struct LineCol {
73    pub line: u32,
74    pub col: u32,
75}
76
77impl LineCol {
78    /// Where byte `offset` of `text` is.
79    pub fn of(text: &str, offset: usize) -> LineCol {
80        let offset = offset.min(text.len());
81        let before = &text.as_bytes()[..offset];
82        let line = before.iter().filter(|&&b| b == b'\n').count() as u32 + 1;
83        let line_start = before
84            .iter()
85            .rposition(|&b| b == b'\n')
86            .map_or(0, |i| i + 1);
87        let col = text
88            .get(line_start..offset)
89            .map_or(offset - line_start, |s| s.chars().count()) as u32
90            + 1;
91        LineCol { line, col }
92    }
93}
94
95impl fmt::Display for LineCol {
96    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
97        write!(f, "{}:{}", self.line, self.col)
98    }
99}
100
101/// A syntax error.
102#[derive(Debug, Clone, PartialEq, Eq)]
103pub struct SyntaxError {
104    pub message: String,
105    pub at: LineCol,
106    pub offset: usize,
107}
108
109impl fmt::Display for SyntaxError {
110    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
111        write!(f, "{}: {}", self.at, self.message)
112    }
113}
114
115impl std::error::Error for SyntaxError {}
116
117/// Parses Yog Data text.
118pub fn parse(text: &str) -> Result<Doc, SyntaxError> {
119    let (value, spans) = run::<true>(text)?;
120    Ok(Doc { value, spans })
121}
122
123/// Parses Yog Data text, without the positions (faster).
124pub fn parse_value(text: &str) -> Result<Value, SyntaxError> {
125    run::<false>(text).map(|(v, _)| v)
126}
127
128fn run<const SPANS: bool>(text: &str) -> Result<(Value, Spans), SyntaxError> {
129    let mut p = Parser::<SPANS> {
130        src: text.as_bytes(),
131        text,
132        i: 0,
133        depth: 0,
134    };
135    if text.starts_with('\u{feff}') {
136        p.i = 3;
137    }
138    p.ws()?;
139    let (value, spans) = p.value()?;
140    p.ws()?;
141    if p.i < p.src.len() {
142        return Err(p.error("unexpected text after the value"));
143    }
144    Ok((value, spans))
145}
146
147/// `SPANS`: whether to record where each value is.
148struct Parser<'a, const SPANS: bool> {
149    src: &'a [u8],
150    text: &'a str,
151    i: usize,
152    depth: usize,
153}
154
155impl<const SPANS: bool> Parser<'_, SPANS> {
156    fn error(&self, message: impl Into<String>) -> SyntaxError {
157        self.error_at(self.i, message)
158    }
159
160    fn error_at(&self, offset: usize, message: impl Into<String>) -> SyntaxError {
161        let mut message = message.into();
162        if offset >= self.src.len() {
163            message.push_str(", found the end of the file");
164        }
165        SyntaxError {
166            message,
167            at: LineCol::of(self.text, offset),
168            offset,
169        }
170    }
171
172    fn peek(&self) -> Option<u8> {
173        self.src.get(self.i).copied()
174    }
175
176    /// Skips whitespace and comments.
177    fn ws(&mut self) -> Result<(), SyntaxError> {
178        loop {
179            let rest = &self.src[self.i..];
180            self.i += rest
181                .iter()
182                .position(|&b| !matches!(b, b' ' | b'\t' | b'\n' | b'\r'))
183                .unwrap_or(rest.len());
184            if self.peek() != Some(b'/') {
185                return Ok(());
186            }
187            match self.src.get(self.i + 1) {
188                Some(b'/') => {
189                    let rest = &self.src[self.i..];
190                    self.i += rest.iter().position(|&b| b == b'\n').unwrap_or(rest.len());
191                }
192                Some(b'*') => {
193                    let start = self.i;
194                    match self.text[self.i + 2..].find("*/") {
195                        Some(end) => self.i += 2 + end + 2,
196                        None => return Err(self.error_at(start, "unclosed `/*` comment")),
197                    }
198                }
199                _ => return Ok(()),
200            }
201        }
202    }
203
204    fn value(&mut self) -> Result<(Value, Spans), SyntaxError> {
205        let start = self.i;
206        let (value, children) = match self.peek() {
207            Some(b'{') => self.nested(|p| p.map())?,
208            Some(b'[') => self.nested(|p| p.list())?,
209            Some(b'"') => (Value::Str(self.string()?), Vec::new()),
210            Some(b'-' | b'0'..=b'9') => (self.number()?, Vec::new()),
211            Some(b't') if self.src[self.i..].starts_with(b"true") => {
212                self.i += 4;
213                (Value::Bool(true), Vec::new())
214            }
215            Some(b'f') if self.src[self.i..].starts_with(b"false") => {
216                self.i += 5;
217                (Value::Bool(false), Vec::new())
218            }
219            Some(b'n') if self.src[self.i..].starts_with(b"null") => {
220                self.i += 4;
221                (Value::Null, Vec::new())
222            }
223            Some(b'\'') => return Err(self.error("strings use double quotes")),
224            _ => return Err(self.error("expected a value")),
225        };
226        let span = Span {
227            start: start as u32,
228            end: self.i as u32,
229        };
230        Ok((
231            value,
232            Spans {
233                span,
234                key: None,
235                children,
236            },
237        ))
238    }
239
240    fn nested(
241        &mut self,
242        f: impl FnOnce(&mut Self) -> Result<(Value, Vec<Spans>), SyntaxError>,
243    ) -> Result<(Value, Vec<Spans>), SyntaxError> {
244        self.depth += 1;
245        if self.depth > MAX_DEPTH {
246            return Err(self.error(format!("nested deeper than {MAX_DEPTH} levels")));
247        }
248        let r = f(self);
249        self.depth -= 1;
250        r
251    }
252
253    fn list(&mut self) -> Result<(Value, Vec<Spans>), SyntaxError> {
254        self.i += 1;
255        let mut items = Vec::new();
256        let mut spans = Vec::new();
257        loop {
258            self.ws()?;
259            if self.peek() == Some(b']') {
260                self.i += 1;
261                break;
262            }
263            let (v, s) = self.value()?;
264            items.push(v);
265            if SPANS {
266                spans.push(s);
267            }
268            self.ws()?;
269            match self.peek() {
270                Some(b',') => self.i += 1,
271                Some(b']') => {
272                    self.i += 1;
273                    break;
274                }
275                _ => return Err(self.error("expected `,` or `]`")),
276            }
277        }
278        Ok((Value::List(items), spans))
279    }
280
281    fn map(&mut self) -> Result<(Value, Vec<Spans>), SyntaxError> {
282        self.i += 1;
283        let mut map = Map::new();
284        let mut spans = Vec::new();
285        // Duplicate keys: a scan for small objects, a set once they grow.
286        let mut seen: Option<std::collections::HashSet<String>> = None;
287        loop {
288            self.ws()?;
289            if self.peek() == Some(b'}') {
290                self.i += 1;
291                break;
292            }
293            let key_start = self.i;
294            let key = match self.peek() {
295                Some(b'"') => self.string()?,
296                Some(c) if c.is_ascii_alphabetic() || c == b'_' || c == b'$' => {
297                    while self
298                        .peek()
299                        .is_some_and(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'$')
300                    {
301                        self.i += 1;
302                    }
303                    self.text[key_start..self.i].to_string()
304                }
305                _ => return Err(self.error("expected a key")),
306            };
307            let key_span = Span {
308                start: key_start as u32,
309                end: self.i as u32,
310            };
311            let duplicate = match &mut seen {
312                Some(set) => !set.insert(key.clone()),
313                None if map.len() < 16 => map.get(&key).is_some(),
314                None => {
315                    let mut set: std::collections::HashSet<String> = map.keys().map(str::to_string).collect();
316                    let dup = !set.insert(key.clone());
317                    seen = Some(set);
318                    dup
319                }
320            };
321            if duplicate {
322                return Err(
323                    self.error_at(key_start, format!("`{key}` is already set in this object"))
324                );
325            }
326            self.ws()?;
327            if self.peek() != Some(b':') {
328                return Err(self.error("expected `:` after the key"));
329            }
330            self.i += 1;
331            self.ws()?;
332            let (v, mut s) = self.value()?;
333            s.key = Some(key_span);
334            map.push_unchecked(key, v);
335            if SPANS {
336                spans.push(s);
337            }
338            self.ws()?;
339            match self.peek() {
340                Some(b',') => self.i += 1,
341                Some(b'}') => {
342                    self.i += 1;
343                    break;
344                }
345                _ => return Err(self.error("expected `,` or `}`")),
346            }
347        }
348        Ok((Value::Map(map), spans))
349    }
350
351    fn string(&mut self) -> Result<String, SyntaxError> {
352        let open = self.i;
353        self.i += 1;
354        let mut out = String::new();
355        loop {
356            let rest = &self.src[self.i..];
357            let Some(n) = rest
358                .iter()
359                .position(|&b| b == b'"' || b == b'\\' || b < 0x20)
360            else {
361                return Err(self.error_at(open, "unclosed string"));
362            };
363            out.push_str(&self.text[self.i..self.i + n]);
364            self.i += n;
365            match self.src[self.i] {
366                b'"' => {
367                    self.i += 1;
368                    return Ok(out);
369                }
370                b'\\' => self.escape(&mut out)?,
371                b'\n' => return Err(self.error("line break in a string (write `\\n`)")),
372                _ => return Err(self.error("control character in a string")),
373            }
374        }
375    }
376
377    fn escape(&mut self, out: &mut String) -> Result<(), SyntaxError> {
378        let at = self.i;
379        self.i += 1;
380        let c = self.peek().ok_or_else(|| self.error("unclosed string"))?;
381        self.i += 1;
382        match c {
383            b'"' => out.push('"'),
384            b'\\' => out.push('\\'),
385            b'/' => out.push('/'),
386            b'b' => out.push('\u{8}'),
387            b'f' => out.push('\u{c}'),
388            b'n' => out.push('\n'),
389            b'r' => out.push('\r'),
390            b't' => out.push('\t'),
391            b'u' => {
392                let hi = self.hex4(at)?;
393                let ch = if (0xD800..0xDC00).contains(&hi) {
394                    if !self.src[self.i..].starts_with(b"\\u") {
395                        return Err(self.error_at(at, "unpaired surrogate in `\\u` escape"));
396                    }
397                    self.i += 2;
398                    let lo = self.hex4(at)?;
399                    if !(0xDC00..0xE000).contains(&lo) {
400                        return Err(self.error_at(at, "unpaired surrogate in `\\u` escape"));
401                    }
402                    char::from_u32(0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00))
403                } else {
404                    char::from_u32(hi)
405                };
406                out.push(
407                    ch.ok_or_else(|| self.error_at(at, "unpaired surrogate in `\\u` escape"))?,
408                );
409            }
410            _ => {
411                return Err(self.error_at(
412                    at,
413                    "unknown escape (`\\n`, `\\t`, `\\\"`, `\\\\`, `\\uXXXX`, ...)",
414                ));
415            }
416        }
417        Ok(())
418    }
419
420    fn hex4(&mut self, at: usize) -> Result<u32, SyntaxError> {
421        let digits = self
422            .text
423            .get(self.i..self.i + 4)
424            .filter(|d| d.bytes().all(|b| b.is_ascii_hexdigit()));
425        let d = digits.ok_or_else(|| self.error_at(at, "`\\u` needs four hex digits"))?;
426        self.i += 4;
427        Ok(u32::from_str_radix(d, 16).unwrap())
428    }
429
430    fn number(&mut self) -> Result<Value, SyntaxError> {
431        let start = self.i;
432        if self.peek() == Some(b'-') {
433            self.i += 1;
434        }
435        let digits = |p: &mut Self| {
436            let s = p.i;
437            while p.peek().is_some_and(|c| c.is_ascii_digit()) {
438                p.i += 1;
439            }
440            p.i - s
441        };
442        let int_start = self.i;
443        if digits(self) == 0 {
444            return Err(self.error("expected digits"));
445        }
446        if self.src[int_start] == b'0' && self.i - int_start > 1 {
447            return Err(self.error_at(int_start, "numbers don't start with 0"));
448        }
449        let mut float = false;
450        if self.peek() == Some(b'.') {
451            self.i += 1;
452            float = true;
453            if digits(self) == 0 {
454                return Err(self.error("expected digits after `.`"));
455            }
456        }
457        if matches!(self.peek(), Some(b'e' | b'E')) {
458            self.i += 1;
459            float = true;
460            if matches!(self.peek(), Some(b'+' | b'-')) {
461                self.i += 1;
462            }
463            if digits(self) == 0 {
464                return Err(self.error("expected digits in the exponent"));
465            }
466        }
467        let s = &self.text[start..self.i];
468        if !float && let Ok(n) = s.parse::<i64>() {
469            return Ok(Value::Int(n));
470        }
471        Ok(Value::Float(
472            s.parse::<f64>()
473                .map_err(|_| self.error_at(start, "invalid number"))?,
474        ))
475    }
476}
477
478// ── Writing ─────────────────────────────────────────────────────────────────
479
480/// `value` as compact text (typed values in their string forms).
481pub fn to_string(value: &Value) -> String {
482    let mut out = String::with_capacity(256);
483    write(value, None, 0, &mut out);
484    out
485}
486
487/// `value` as text indented by two spaces, one entry per line.
488pub fn to_string_pretty(value: &Value) -> String {
489    let mut out = String::new();
490    write(value, Some("  "), 0, &mut out);
491    out.push('\n');
492    out
493}
494
495fn write(value: &Value, indent: Option<&str>, level: usize, out: &mut String) {
496    let newline = |out: &mut String, level: usize| {
497        if let Some(unit) = indent {
498            out.push('\n');
499            for _ in 0..level {
500                out.push_str(unit);
501            }
502        }
503    };
504    match value {
505        Value::Null => out.push_str("null"),
506        Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
507        Value::Int(n) => {
508            use std::fmt::Write as _;
509            let _ = write!(out, "{n}");
510        }
511        Value::Float(f) => write_float(*f, out),
512        Value::List(items) => {
513            if items.is_empty() {
514                out.push_str("[]");
515                return;
516            }
517            out.push('[');
518            for (i, v) in items.iter().enumerate() {
519                if i > 0 {
520                    out.push(',');
521                }
522                newline(out, level + 1);
523                write(v, indent, level + 1, out);
524            }
525            newline(out, level);
526            out.push(']');
527        }
528        Value::Map(map) => {
529            if map.is_empty() {
530                out.push_str("{}");
531                return;
532            }
533            out.push('{');
534            for (i, (k, v)) in map.iter().enumerate() {
535                if i > 0 {
536                    out.push(',');
537                }
538                newline(out, level + 1);
539                write_string(k, out);
540                out.push_str(if indent.is_some() { ": " } else { ":" });
541                write(v, indent, level + 1, out);
542            }
543            newline(out, level);
544            out.push('}');
545        }
546        Value::Str(s) | Value::Text(s) => {
547            if s.starts_with('@') {
548                out.push('"');
549                out.push('@');
550                write_string_body(s, out);
551                out.push('"');
552            } else {
553                write_string(s, out);
554            }
555        }
556        typed => write_string(&crate::typed::typed_string(typed).unwrap_or_default(), out),
557    }
558}
559
560fn write_float(f: f64, out: &mut String) {
561    if !f.is_finite() {
562        out.push_str("null");
563    } else if f.fract() == 0.0 && f.abs() < 1e16 {
564        out.push_str(&format!("{f:.1}"));
565    } else {
566        out.push_str(&f.to_string());
567    }
568}
569
570/// `s` as a JSON string literal.
571pub fn write_string(s: &str, out: &mut String) {
572    out.push('"');
573    write_string_body(s, out);
574    out.push('"');
575}
576
577fn write_string_body(s: &str, out: &mut String) {
578    let bytes = s.as_bytes();
579    let mut run = 0;
580    while let Some(n) = bytes[run..]
581        .iter()
582        .position(|&b| b == b'"' || b == b'\\' || b < 0x20)
583    {
584        let i = run + n;
585        out.push_str(&s[run..i]);
586        match bytes[i] {
587            b'"' => out.push_str("\\\""),
588            b'\\' => out.push_str("\\\\"),
589            b'\n' => out.push_str("\\n"),
590            b'\r' => out.push_str("\\r"),
591            b'\t' => out.push_str("\\t"),
592            b => {
593                use std::fmt::Write as _;
594                let _ = write!(out, "\\u{b:04x}");
595            }
596        }
597        run = i + 1;
598    }
599    out.push_str(&s[run..]);
600}
601
602#[cfg(test)]
603mod tests {
604    use super::*;
605
606    #[test]
607    fn json5_subset() {
608        let doc = parse("\u{feff}{\n  // comment\n  name: \"Ruby\", /* inline */ $schema: 1,\n  \"list\": [1, 2.5, -3e2, true, null,],\n}").unwrap();
609        let m = doc.value.as_map().unwrap();
610        assert_eq!(m.keys().collect::<Vec<_>>(), ["name", "$schema", "list"]);
611        assert_eq!(
612            doc.value.get("list").unwrap(),
613            &Value::List(vec![
614                Value::Int(1),
615                Value::Float(2.5),
616                Value::Float(-300.0),
617                Value::Bool(true),
618                Value::Null
619            ])
620        );
621    }
622
623    #[test]
624    fn locates_values_and_keys() {
625        let text = "{\n  \"a\": [\n    1,\n    {\"b\": \"x\"}\n  ]\n}";
626        let doc = parse(text).unwrap();
627        let path = Path::root().key("a").index(1).key("b");
628        assert_eq!(doc.locate(text, &path), LineCol { line: 4, col: 11 });
629        assert_eq!(
630            LineCol::of(text, doc.spans_at(&path).key.unwrap().start as usize),
631            LineCol { line: 4, col: 6 }
632        );
633        // A path past the document: its deepest existing part.
634        assert_eq!(
635            doc.locate(text, &Path::root().key("a").index(9)),
636            LineCol { line: 2, col: 8 }
637        );
638    }
639
640    #[test]
641    fn errors_say_where() {
642        let cases = [
643            ("{\"a\": 1,, }", "1:9", "expected a key"),
644            ("{\"a\": 1 \"b\": 2}", "1:9", "expected `,` or `}`"),
645            ("{\"a\": 1, \"a\": 2}", "1:10", "already set"),
646            ("[1, 2", "1:6", "found the end of the file"),
647            ("\"ab\ncd\"", "1:4", "line break"),
648            ("'x'", "1:1", "double quotes"),
649            ("01", "1:1", "start with 0"),
650            ("{} x", "1:4", "unexpected text"),
651            ("/* never closed", "1:1", "unclosed"),
652            ("\"\\q\"", "1:2", "unknown escape"),
653            ("\"\\ud800\"", "1:2", "surrogate"),
654            ("{\n  \"é\": tru }", "2:8", "expected a value"),
655        ];
656        for (text, at, msg) in cases {
657            let e = parse(text).unwrap_err();
658            assert_eq!(e.at.to_string(), at, "{text:?}: {e}");
659            assert!(e.message.contains(msg), "{text:?}: {e}");
660        }
661        assert!(
662            parse(&"[".repeat(MAX_DEPTH + 1))
663                .unwrap_err()
664                .message
665                .contains("nested deeper")
666        );
667    }
668
669    #[test]
670    fn strings_and_numbers() {
671        let v = parse_value(
672            r#"["a\"b\\c\n\u00e9\ud83d\ude00", 9223372036854775807, 9223372036854775808, 0.1]"#,
673        )
674        .unwrap();
675        let l = v.as_list().unwrap();
676        assert_eq!(l[0], Value::Str("a\"b\\c\né😀".into()));
677        assert_eq!(l[1], Value::Int(i64::MAX));
678        assert_eq!(l[2], Value::Float(9223372036854775808.0));
679        assert_eq!(l[3], Value::Float(0.1));
680    }
681
682    #[test]
683    fn writes_what_it_reads() {
684        let text = "{\n  \"name\": \"Ruby \\\"red\\\"\\n\",\n  \"n\": [1, 2.0, 0.1, -5],\n  \"empty\": {},\n  \"none\": [],\n  \"tab\\u0001\": null\n}\n";
685        let v = parse_value(text).unwrap();
686        assert_eq!(parse_value(&to_string(&v)).unwrap(), v);
687        let pretty = to_string_pretty(&v);
688        assert_eq!(parse_value(&pretty).unwrap(), v);
689        assert!(
690            pretty.starts_with(
691                "{\n  \"name\": \"Ruby \\\"red\\\"\\n\",\n  \"n\": [\n    1,\n    2.0,"
692            ),
693            "{pretty}"
694        );
695        assert_eq!(
696            to_string(&v),
697            r#"{"name":"Ruby \"red\"\n","n":[1,2.0,0.1,-5],"empty":{},"none":[],"tab\u0001":null}"#
698        );
699    }
700
701    #[test]
702    fn typed_values_are_strings_in_text() {
703        let v = Value::List(vec![
704            Value::Ref(crate::Ref::new("item", "minecraft:stick")),
705            Value::Color(0xFF112233),
706            Value::Str("@not a ref".into()),
707        ]);
708        assert_eq!(
709            to_string(&v),
710            r##"["@item:minecraft:stick","#112233","@@not a ref"]"##
711        );
712    }
713}
yog_data/text.rs

yog_data/
text.rs
