yog_data/
text.rs

1//! The text syntax: JSON, plus `//` and `/* */` comments, trailing commas
2//! and unquoted keys (the part of JSON5 editors already understand).
3//!
4//! [`parse`] keeps where every value is ([`Spans`]), so a schema error at
5//! `entries[2].pages[0].item` can point at its line. Typed values stay
6//! strings here — the schema decides what a string means.
7
8use std::fmt;
9
10use crate::value::{Map, Path, Seg, Value};
11
12/// Deeper than this is an error, not a stack overflow.
13pub const MAX_DEPTH: usize = 256;
14
15/// A byte range of the source.
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
17pub struct Span {
18    pub start: u32,
19    pub end: u32,
20}
21
22/// Where each value of a parsed document is: the value's span, its key's
23/// (map entries), and its children's, in the same order as the value's.
24#[derive(Debug, Clone, Default, PartialEq)]
25pub struct Spans {
26    pub span: Span,
27    pub key: Option<Span>,
28    pub children: Vec<Spans>,
29}
30
31/// A parsed document: the value and where each of its parts is.
32#[derive(Debug, Clone, PartialEq)]
33pub struct Doc {
34    pub value: Value,
35    pub spans: Spans,
36}
37
38impl Doc {
39    /// The spans of the value at `path` — the closest existing parent's if
40    /// the path goes further than the document.
41    pub fn spans_at(&self, path: &Path) -> &Spans {
42        let mut v = &self.value;
43        let mut s = &self.spans;
44        for seg in path.segments() {
45            let i = match (seg, v) {
46                (Seg::Key(k), Value::Map(m)) => m.index_of(k),
47                (Seg::Index(i), Value::List(l)) if *i < l.len() => Some(*i),
48                _ => None,
49            };
50            let Some(i) = i else { break };
51            let Some(child) = s.children.get(i) else {
52                break;
53            };
54            v = match v {
55                Value::Map(m) => m.value_at(i).unwrap_or(v),
56                Value::List(l) => &l[i],
57                _ => v,
58            };
59            s = child;
60        }
61        s
62    }
63
64    /// The line and column of the value at `path` in `text` (the source this was parsed from).
65    pub fn locate(&self, text: &str, path: &Path) -> LineCol {
66        LineCol::of(text, self.spans_at(path).span.start as usize)
67    }
68}
69
70/// A 1-based line and column (in characters).
71#[derive(Debug, Clone, Copy, PartialEq, Eq)]
72pub struct LineCol {
73    pub line: u32,
74    pub col: u32,
75}
76
77impl LineCol {
78    /// Where byte `offset` of `text` is.
79    pub fn of(text: &str, offset: usize) -> LineCol {
80        let offset = offset.min(text.len());
81        let before = &text.as_bytes()[..offset];
82        let line = before.iter().filter(|&&b| b == b'\n').count() as u32 + 1;
83        let line_start = before
84            .iter()
85            .rposition(|&b| b == b'\n')
86            .map_or(0, |i| i + 1);
87        let col = text
88            .get(line_start..offset)
89            .map_or(offset - line_start, |s| s.chars().count()) as u32
90            + 1;
91        LineCol { line, col }
92    }
93}
94
95impl fmt::Display for LineCol {
96    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
97        write!(f, "{}:{}", self.line, self.col)
98    }
99}
100
101/// A syntax error.
102#[derive(Debug, Clone, PartialEq, Eq)]
103pub struct SyntaxError {
104    pub message: String,
105    pub at: LineCol,
106    pub offset: usize,
107}
108
109impl fmt::Display for SyntaxError {
110    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
111        write!(f, "{}: {}", self.at, self.message)
112    }
113}
114
115impl std::error::Error for SyntaxError {}
116
117/// Parses Yog Data text.
118pub fn parse(text: &str) -> Result<Doc, SyntaxError> {
119    let (value, spans) = run::<true>(text)?;
120    Ok(Doc { value, spans })
121}
122
123/// Parses Yog Data text, without the positions (faster).
124pub fn parse_value(text: &str) -> Result<Value, SyntaxError> {
125    run::<false>(text).map(|(v, _)| v)
126}
127
128fn run<const SPANS: bool>(text: &str) -> Result<(Value, Spans), SyntaxError> {
129    let mut p = Parser::<SPANS> {
130        src: text.as_bytes(),
131        text,
132        i: 0,
133        depth: 0,
134    };
135    if text.starts_with('\u{feff}') {
136        p.i = 3;
137    }
138    p.ws()?;
139    let (value, spans) = p.value()?;
140    p.ws()?;
141    if p.i < p.src.len() {
142        return Err(p.error("unexpected text after the value"));
143    }
144    Ok((value, spans))
145}
146
147/// `SPANS`: whether to record where each value is.
148struct Parser<'a, const SPANS: bool> {
149    src: &'a [u8],
150    text: &'a str,
151    i: usize,
152    depth: usize,
153}
154
155impl<const SPANS: bool> Parser<'_, SPANS> {
156    fn error(&self, message: impl Into<String>) -> SyntaxError {
157        self.error_at(self.i, message)
158    }
159
160    fn error_at(&self, offset: usize, message: impl Into<String>) -> SyntaxError {
161        let mut message = message.into();
162        if offset >= self.src.len() {
163            message.push_str(", found the end of the file");
164        }
165        SyntaxError {
166            message,
167            at: LineCol::of(self.text, offset),
168            offset,
169        }
170    }
171
172    fn peek(&self) -> Option<u8> {
173        self.src.get(self.i).copied()
174    }
175
176    /// Skips whitespace and comments.
177    fn ws(&mut self) -> Result<(), SyntaxError> {
178        loop {
179            let rest = &self.src[self.i..];
180            self.i += rest
181                .iter()
182                .position(|&b| !matches!(b, b' ' | b'\t' | b'\n' | b'\r'))
183                .unwrap_or(rest.len());
184            if self.peek() != Some(b'/') {
185                return Ok(());
186            }
187            match self.src.get(self.i + 1) {
188                Some(b'/') => {
189                    let rest = &self.src[self.i..];
190                    self.i += rest.iter().position(|&b| b == b'\n').unwrap_or(rest.len());
191                }
192                Some(b'*') => {
193                    let start = self.i;
194                    match self.text[self.i + 2..].find("*/") {
195                        Some(end) => self.i += 2 + end + 2,
196                        None => return Err(self.error_at(start, "unclosed `/*` comment")),
197                    }
198                }
199                _ => return Ok(()),
200            }
201        }
202    }
203
204    fn value(&mut self) -> Result<(Value, Spans), SyntaxError> {
205        let start = self.i;
206        let (value, children) = match self.peek() {
207            Some(b'{') => self.nested(|p| p.map())?,
208            Some(b'[') => self.nested(|p| p.list())?,
209            Some(b'"') => (Value::Str(self.string()?), Vec::new()),
210            Some(b'-' | b'0'..=b'9') => (self.number()?, Vec::new()),
211            Some(b't') if self.src[self.i..].starts_with(b"true") => {
212                self.i += 4;
213                (Value::Bool(true), Vec::new())
214            }
215            Some(b'f') if self.src[self.i..].starts_with(b"false") => {
216                self.i += 5;
217                (Value::Bool(false), Vec::new())
218            }
219            Some(b'n') if self.src[self.i..].starts_with(b"null") => {
220                self.i += 4;
221                (Value::Null, Vec::new())
222            }
223            Some(b'\'') => return Err(self.error("strings use double quotes")),
224            _ => return Err(self.error("expected a value")),
225        };
226        let span = Span {
227            start: start as u32,
228            end: self.i as u32,
229        };
230        Ok((
231            value,
232            Spans {
233                span,
234                key: None,
235                children,
236            },
237        ))
238    }
239
240    fn nested(
241        &mut self,
242        f: impl FnOnce(&mut Self) -> Result<(Value, Vec<Spans>), SyntaxError>,
243    ) -> Result<(Value, Vec<Spans>), SyntaxError> {
244        self.depth += 1;
245        if self.depth > MAX_DEPTH {
246            return Err(self.error(format!("nested deeper than {MAX_DEPTH} levels")));
247        }
248        let r = f(self);
249        self.depth -= 1;
250        r
251    }
252
253    fn list(&mut self) -> Result<(Value, Vec<Spans>), SyntaxError> {
254        self.i += 1;
255        let mut items = Vec::new();
256        let mut spans = Vec::new();
257        loop {
258            self.ws()?;
259            if self.peek() == Some(b']') {
260                self.i += 1;
261                break;
262            }
263            let (v, s) = self.value()?;
264            items.push(v);
265            if SPANS {
266                spans.push(s);
267            }
268            self.ws()?;
269            match self.peek() {
270                Some(b',') => self.i += 1,
271                Some(b']') => {
272                    self.i += 1;
273                    break;
274                }
275                _ => return Err(self.error("expected `,` or `]`")),
276            }
277        }
278        Ok((Value::List(items), spans))
279    }
280
281    fn map(&mut self) -> Result<(Value, Vec<Spans>), SyntaxError> {
282        self.i += 1;
283        let mut map = Map::new();
284        let mut spans = Vec::new();
285        // Duplicate keys: a scan for small objects, a set once they grow.
286        let mut seen: Option<std::collections::HashSet<String>> = None;
287        loop {
288            self.ws()?;
289            if self.peek() == Some(b'}') {
290                self.i += 1;
291                break;
292            }
293            let key_start = self.i;
294            let key = match self.peek() {
295                Some(b'"') => self.string()?,
296                Some(c) if c.is_ascii_alphabetic() || c == b'_' || c == b'$' => {
297                    while self
298                        .peek()
299                        .is_some_and(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'$')
300                    {
301                        self.i += 1;
302                    }
303                    self.text[key_start..self.i].to_string()
304                }
305                _ => return Err(self.error("expected a key")),
306            };
307            let key_span = Span {
308                start: key_start as u32,
309                end: self.i as u32,
310            };
311            let duplicate = match &mut seen {
312                Some(set) => !set.insert(key.clone()),
313                None if map.len() < 16 => map.get(&key).is_some(),
314                None => {
315                    let mut set: std::collections::HashSet<String> =
316                        map.keys().map(str::to_string).collect();
317                    let dup = !set.insert(key.clone());
318                    seen = Some(set);
319                    dup
320                }
321            };
322            if duplicate {
323                return Err(
324                    self.error_at(key_start, format!("`{key}` is already set in this object"))
325                );
326            }
327            self.ws()?;
328            if self.peek() != Some(b':') {
329                return Err(self.error("expected `:` after the key"));
330            }
331            self.i += 1;
332            self.ws()?;
333            let (v, mut s) = self.value()?;
334            s.key = Some(key_span);
335            map.push_unchecked(key.into(), v);
336            if SPANS {
337                spans.push(s);
338            }
339            self.ws()?;
340            match self.peek() {
341                Some(b',') => self.i += 1,
342                Some(b'}') => {
343                    self.i += 1;
344                    break;
345                }
346                _ => return Err(self.error("expected `,` or `}`")),
347            }
348        }
349        Ok((Value::Map(map), spans))
350    }
351
352    fn string(&mut self) -> Result<String, SyntaxError> {
353        let open = self.i;
354        self.i += 1;
355        let mut out = String::new();
356        loop {
357            let rest = &self.src[self.i..];
358            let Some(n) = rest
359                .iter()
360                .position(|&b| b == b'"' || b == b'\\' || b < 0x20)
361            else {
362                return Err(self.error_at(open, "unclosed string"));
363            };
364            out.push_str(&self.text[self.i..self.i + n]);
365            self.i += n;
366            match self.src[self.i] {
367                b'"' => {
368                    self.i += 1;
369                    return Ok(out);
370                }
371                b'\\' => self.escape(&mut out)?,
372                b'\n' => return Err(self.error("line break in a string (write `\\n`)")),
373                _ => return Err(self.error("control character in a string")),
374            }
375        }
376    }
377
378    fn escape(&mut self, out: &mut String) -> Result<(), SyntaxError> {
379        let at = self.i;
380        self.i += 1;
381        let c = self.peek().ok_or_else(|| self.error("unclosed string"))?;
382        self.i += 1;
383        match c {
384            b'"' => out.push('"'),
385            b'\\' => out.push('\\'),
386            b'/' => out.push('/'),
387            b'b' => out.push('\u{8}'),
388            b'f' => out.push('\u{c}'),
389            b'n' => out.push('\n'),
390            b'r' => out.push('\r'),
391            b't' => out.push('\t'),
392            b'u' => {
393                let hi = self.hex4(at)?;
394                let ch = if (0xD800..0xDC00).contains(&hi) {
395                    if !self.src[self.i..].starts_with(b"\\u") {
396                        return Err(self.error_at(at, "unpaired surrogate in `\\u` escape"));
397                    }
398                    self.i += 2;
399                    let lo = self.hex4(at)?;
400                    if !(0xDC00..0xE000).contains(&lo) {
401                        return Err(self.error_at(at, "unpaired surrogate in `\\u` escape"));
402                    }
403                    char::from_u32(0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00))
404                } else {
405                    char::from_u32(hi)
406                };
407                out.push(
408                    ch.ok_or_else(|| self.error_at(at, "unpaired surrogate in `\\u` escape"))?,
409                );
410            }
411            _ => {
412                return Err(self.error_at(
413                    at,
414                    "unknown escape (`\\n`, `\\t`, `\\\"`, `\\\\`, `\\uXXXX`, ...)",
415                ));
416            }
417        }
418        Ok(())
419    }
420
421    fn hex4(&mut self, at: usize) -> Result<u32, SyntaxError> {
422        let digits = self
423            .text
424            .get(self.i..self.i + 4)
425            .filter(|d| d.bytes().all(|b| b.is_ascii_hexdigit()));
426        let d = digits.ok_or_else(|| self.error_at(at, "`\\u` needs four hex digits"))?;
427        self.i += 4;
428        Ok(u32::from_str_radix(d, 16).unwrap())
429    }
430
431    fn number(&mut self) -> Result<Value, SyntaxError> {
432        let start = self.i;
433        if self.peek() == Some(b'-') {
434            self.i += 1;
435        }
436        let digits = |p: &mut Self| {
437            let s = p.i;
438            while p.peek().is_some_and(|c| c.is_ascii_digit()) {
439                p.i += 1;
440            }
441            p.i - s
442        };
443        let int_start = self.i;
444        if digits(self) == 0 {
445            return Err(self.error("expected digits"));
446        }
447        if self.src[int_start] == b'0' && self.i - int_start > 1 {
448            return Err(self.error_at(int_start, "numbers don't start with 0"));
449        }
450        let mut float = false;
451        if self.peek() == Some(b'.') {
452            self.i += 1;
453            float = true;
454            if digits(self) == 0 {
455                return Err(self.error("expected digits after `.`"));
456            }
457        }
458        if matches!(self.peek(), Some(b'e' | b'E')) {
459            self.i += 1;
460            float = true;
461            if matches!(self.peek(), Some(b'+' | b'-')) {
462                self.i += 1;
463            }
464            if digits(self) == 0 {
465                return Err(self.error("expected digits in the exponent"));
466            }
467        }
468        let s = &self.text[start..self.i];
469        if !float && let Ok(n) = s.parse::<i64>() {
470            return Ok(Value::Int(n));
471        }
472        Ok(Value::Float(
473            s.parse::<f64>()
474                .map_err(|_| self.error_at(start, "invalid number"))?,
475        ))
476    }
477}
478
479// ── Writing ─────────────────────────────────────────────────────────────────
480
481/// `value` as compact text (typed values in their string forms).
482pub fn to_string(value: &Value) -> String {
483    let mut out = String::with_capacity(256);
484    write(value, None, 0, &mut out);
485    out
486}
487
488/// `value` as text indented by two spaces, one entry per line.
489pub fn to_string_pretty(value: &Value) -> String {
490    let mut out = String::new();
491    write(value, Some("  "), 0, &mut out);
492    out.push('\n');
493    out
494}
495
496fn write(value: &Value, indent: Option<&str>, level: usize, out: &mut String) {
497    let newline = |out: &mut String, level: usize| {
498        if let Some(unit) = indent {
499            out.push('\n');
500            for _ in 0..level {
501                out.push_str(unit);
502            }
503        }
504    };
505    match value {
506        Value::Null => out.push_str("null"),
507        Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
508        Value::Int(n) => {
509            use std::fmt::Write as _;
510            let _ = write!(out, "{n}");
511        }
512        Value::Float(f) => write_float(*f, out),
513        Value::List(items) => {
514            if items.is_empty() {
515                out.push_str("[]");
516                return;
517            }
518            out.push('[');
519            for (i, v) in items.iter().enumerate() {
520                if i > 0 {
521                    out.push(',');
522                }
523                newline(out, level + 1);
524                write(v, indent, level + 1, out);
525            }
526            newline(out, level);
527            out.push(']');
528        }
529        Value::Map(map) => {
530            if map.is_empty() {
531                out.push_str("{}");
532                return;
533            }
534            out.push('{');
535            for (i, (k, v)) in map.iter().enumerate() {
536                if i > 0 {
537                    out.push(',');
538                }
539                newline(out, level + 1);
540                write_string(k, out);
541                out.push_str(if indent.is_some() { ": " } else { ":" });
542                write(v, indent, level + 1, out);
543            }
544            newline(out, level);
545            out.push('}');
546        }
547        Value::Str(s) | Value::Text(s) => {
548            if s.starts_with('@') {
549                out.push('"');
550                out.push('@');
551                write_string_body(s, out);
552                out.push('"');
553            } else {
554                write_string(s, out);
555            }
556        }
557        typed => write_string(&crate::typed::typed_string(typed).unwrap_or_default(), out),
558    }
559}
560
561fn write_float(f: f64, out: &mut String) {
562    if !f.is_finite() {
563        out.push_str("null");
564    } else if f.fract() == 0.0 && f.abs() < 1e16 {
565        out.push_str(&format!("{f:.1}"));
566    } else {
567        out.push_str(&f.to_string());
568    }
569}
570
571/// `s` as a JSON string literal.
572pub fn write_string(s: &str, out: &mut String) {
573    out.push('"');
574    write_string_body(s, out);
575    out.push('"');
576}
577
578fn write_string_body(s: &str, out: &mut String) {
579    let bytes = s.as_bytes();
580    let mut run = 0;
581    while let Some(n) = bytes[run..]
582        .iter()
583        .position(|&b| b == b'"' || b == b'\\' || b < 0x20)
584    {
585        let i = run + n;
586        out.push_str(&s[run..i]);
587        match bytes[i] {
588            b'"' => out.push_str("\\\""),
589            b'\\' => out.push_str("\\\\"),
590            b'\n' => out.push_str("\\n"),
591            b'\r' => out.push_str("\\r"),
592            b'\t' => out.push_str("\\t"),
593            b => {
594                use std::fmt::Write as _;
595                let _ = write!(out, "\\u{b:04x}");
596            }
597        }
598        run = i + 1;
599    }
600    out.push_str(&s[run..]);
601}
602
603#[cfg(test)]
604mod tests {
605    use super::*;
606
607    #[test]
608    fn json5_subset() {
609        let doc = parse("\u{feff}{\n  // comment\n  name: \"Ruby\", /* inline */ $schema: 1,\n  \"list\": [1, 2.5, -3e2, true, null,],\n}").unwrap();
610        let m = doc.value.as_map().unwrap();
611        assert_eq!(m.keys().collect::<Vec<_>>(), ["name", "$schema", "list"]);
612        assert_eq!(
613            doc.value.get("list").unwrap(),
614            &Value::List(vec![
615                Value::Int(1),
616                Value::Float(2.5),
617                Value::Float(-300.0),
618                Value::Bool(true),
619                Value::Null
620            ])
621        );
622    }
623
624    #[test]
625    fn locates_values_and_keys() {
626        let text = "{\n  \"a\": [\n    1,\n    {\"b\": \"x\"}\n  ]\n}";
627        let doc = parse(text).unwrap();
628        let path = Path::root().key("a").index(1).key("b");
629        assert_eq!(doc.locate(text, &path), LineCol { line: 4, col: 11 });
630        assert_eq!(
631            LineCol::of(text, doc.spans_at(&path).key.unwrap().start as usize),
632            LineCol { line: 4, col: 6 }
633        );
634        // A path past the document: its deepest existing part.
635        assert_eq!(
636            doc.locate(text, &Path::root().key("a").index(9)),
637            LineCol { line: 2, col: 8 }
638        );
639    }
640
641    #[test]
642    fn errors_say_where() {
643        let cases = [
644            ("{\"a\": 1,, }", "1:9", "expected a key"),
645            ("{\"a\": 1 \"b\": 2}", "1:9", "expected `,` or `}`"),
646            ("{\"a\": 1, \"a\": 2}", "1:10", "already set"),
647            ("[1, 2", "1:6", "found the end of the file"),
648            ("\"ab\ncd\"", "1:4", "line break"),
649            ("'x'", "1:1", "double quotes"),
650            ("01", "1:1", "start with 0"),
651            ("{} x", "1:4", "unexpected text"),
652            ("/* never closed", "1:1", "unclosed"),
653            ("\"\\q\"", "1:2", "unknown escape"),
654            ("\"\\ud800\"", "1:2", "surrogate"),
655            ("{\n  \"é\": tru }", "2:8", "expected a value"),
656        ];
657        for (text, at, msg) in cases {
658            let e = parse(text).unwrap_err();
659            assert_eq!(e.at.to_string(), at, "{text:?}: {e}");
660            assert!(e.message.contains(msg), "{text:?}: {e}");
661        }
662        assert!(
663            parse(&"[".repeat(MAX_DEPTH + 1))
664                .unwrap_err()
665                .message
666                .contains("nested deeper")
667        );
668    }
669
670    #[test]
671    fn strings_and_numbers() {
672        let v = parse_value(
673            r#"["a\"b\\c\n\u00e9\ud83d\ude00", 9223372036854775807, 9223372036854775808, 0.1]"#,
674        )
675        .unwrap();
676        let l = v.as_list().unwrap();
677        assert_eq!(l[0], Value::Str("a\"b\\c\né😀".into()));
678        assert_eq!(l[1], Value::Int(i64::MAX));
679        assert_eq!(l[2], Value::Float(9223372036854775808.0));
680        assert_eq!(l[3], Value::Float(0.1));
681    }
682
683    #[test]
684    fn writes_what_it_reads() {
685        let text = "{\n  \"name\": \"Ruby \\\"red\\\"\\n\",\n  \"n\": [1, 2.0, 0.1, -5],\n  \"empty\": {},\n  \"none\": [],\n  \"tab\\u0001\": null\n}\n";
686        let v = parse_value(text).unwrap();
687        assert_eq!(parse_value(&to_string(&v)).unwrap(), v);
688        let pretty = to_string_pretty(&v);
689        assert_eq!(parse_value(&pretty).unwrap(), v);
690        assert!(
691            pretty.starts_with(
692                "{\n  \"name\": \"Ruby \\\"red\\\"\\n\",\n  \"n\": [\n    1,\n    2.0,"
693            ),
694            "{pretty}"
695        );
696        assert_eq!(
697            to_string(&v),
698            r#"{"name":"Ruby \"red\"\n","n":[1,2.0,0.1,-5],"empty":{},"none":[],"tab\u0001":null}"#
699        );
700    }
701
702    #[test]
703    fn typed_values_are_strings_in_text() {
704        let v = Value::List(vec![
705            Value::Ref(crate::Ref::new("item", "minecraft:stick")),
706            Value::Color(0xFF112233),
707            Value::Str("@not a ref".into()),
708        ]);
709        assert_eq!(
710            to_string(&v),
711            r##"["@item:minecraft:stick","#112233","@@not a ref"]"##
712        );
713    }
714}
yog_data/text.rs

yog_data/
text.rs
