yog_data/
text.rs

1//! The text syntax: JSON, plus `//` and `/* */` comments, trailing commas
2//! and unquoted keys (the part of JSON5 editors already understand).
3//!
4//! [`parse`] keeps where every value is ([`Spans`]), so a schema error at
5//! `entries[2].pages[0].item` can point at its line. Typed values stay
6//! strings here — the schema decides what a string means.
7
8use std::fmt;
9
10use crate::value::{Map, Path, Seg, Value};
11
12/// Deeper than this is an error, not a stack overflow.
13pub const MAX_DEPTH: usize = 256;
14
15/// A byte range of the source.
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
17pub struct Span {
18    pub start: u32,
19    pub end: u32,
20}
21
22/// Where each value of a parsed document is: the value's span, its key's
23/// (map entries), and its children's, in the same order as the value's.
24#[derive(Debug, Clone, Default, PartialEq)]
25pub struct Spans {
26    pub span: Span,
27    pub key: Option<Span>,
28    pub children: Vec<Spans>,
29}
30
31/// A parsed document: the value and where each of its parts is.
32#[derive(Debug, Clone, PartialEq)]
33pub struct Doc {
34    pub value: Value,
35    pub spans: Spans,
36}
37
38impl Doc {
39    /// The spans of the value at `path` — the closest existing parent's if
40    /// the path goes further than the document.
41    pub fn spans_at(&self, path: &Path) -> &Spans {
42        let mut v = &self.value;
43        let mut s = &self.spans;
44        for seg in path.segments() {
45            let i = match (seg, v) {
46                (Seg::Key(k), Value::Map(m)) => m.index_of(k),
47                (Seg::Index(i), Value::List(l)) if *i < l.len() => Some(*i),
48                _ => None,
49            };
50            let Some(i) = i else { break };
51            let Some(child) = s.children.get(i) else {
52                break;
53            };
54            v = match v {
55                Value::Map(m) => m.iter().nth(i).map(|(_, v)| v).unwrap_or(v),
56                Value::List(l) => &l[i],
57                _ => v,
58            };
59            s = child;
60        }
61        s
62    }
63
64    /// The line and column of the value at `path` in `text` (the source this was parsed from).
65    pub fn locate(&self, text: &str, path: &Path) -> LineCol {
66        LineCol::of(text, self.spans_at(path).span.start as usize)
67    }
68}
69
70/// A 1-based line and column (in characters).
71#[derive(Debug, Clone, Copy, PartialEq, Eq)]
72pub struct LineCol {
73    pub line: u32,
74    pub col: u32,
75}
76
77impl LineCol {
78    /// Where byte `offset` of `text` is.
79    pub fn of(text: &str, offset: usize) -> LineCol {
80        let offset = offset.min(text.len());
81        let before = &text.as_bytes()[..offset];
82        let line = before.iter().filter(|&&b| b == b'\n').count() as u32 + 1;
83        let line_start = before
84            .iter()
85            .rposition(|&b| b == b'\n')
86            .map_or(0, |i| i + 1);
87        let col = text
88            .get(line_start..offset)
89            .map_or(offset - line_start, |s| s.chars().count()) as u32
90            + 1;
91        LineCol { line, col }
92    }
93}
94
95impl fmt::Display for LineCol {
96    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
97        write!(f, "{}:{}", self.line, self.col)
98    }
99}
100
101/// A syntax error.
102#[derive(Debug, Clone, PartialEq, Eq)]
103pub struct SyntaxError {
104    pub message: String,
105    pub at: LineCol,
106    pub offset: usize,
107}
108
109impl fmt::Display for SyntaxError {
110    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
111        write!(f, "{}: {}", self.at, self.message)
112    }
113}
114
115impl std::error::Error for SyntaxError {}
116
117/// Parses Yog Data text.
118pub fn parse(text: &str) -> Result<Doc, SyntaxError> {
119    let (value, spans) = run::<true>(text)?;
120    Ok(Doc { value, spans })
121}
122
123/// Parses Yog Data text, without the positions (faster).
124pub fn parse_value(text: &str) -> Result<Value, SyntaxError> {
125    run::<false>(text).map(|(v, _)| v)
126}
127
128fn run<const SPANS: bool>(text: &str) -> Result<(Value, Spans), SyntaxError> {
129    let mut p = Parser::<SPANS> {
130        src: text.as_bytes(),
131        text,
132        i: 0,
133        depth: 0,
134    };
135    if text.starts_with('\u{feff}') {
136        p.i = 3;
137    }
138    p.ws()?;
139    let (value, spans) = p.value()?;
140    p.ws()?;
141    if p.i < p.src.len() {
142        return Err(p.error("unexpected text after the value"));
143    }
144    Ok((value, spans))
145}
146
147/// `SPANS`: whether to record where each value is.
148struct Parser<'a, const SPANS: bool> {
149    src: &'a [u8],
150    text: &'a str,
151    i: usize,
152    depth: usize,
153}
154
155impl<const SPANS: bool> Parser<'_, SPANS> {
156    fn error(&self, message: impl Into<String>) -> SyntaxError {
157        self.error_at(self.i, message)
158    }
159
160    fn error_at(&self, offset: usize, message: impl Into<String>) -> SyntaxError {
161        let mut message = message.into();
162        if offset >= self.src.len() {
163            message.push_str(", found the end of the file");
164        }
165        SyntaxError {
166            message,
167            at: LineCol::of(self.text, offset),
168            offset,
169        }
170    }
171
172    fn peek(&self) -> Option<u8> {
173        self.src.get(self.i).copied()
174    }
175
176    /// Skips whitespace and comments.
177    fn ws(&mut self) -> Result<(), SyntaxError> {
178        loop {
179            let rest = &self.src[self.i..];
180            self.i += rest
181                .iter()
182                .position(|&b| !matches!(b, b' ' | b'\t' | b'\n' | b'\r'))
183                .unwrap_or(rest.len());
184            if self.peek() != Some(b'/') {
185                return Ok(());
186            }
187            match self.src.get(self.i + 1) {
188                Some(b'/') => {
189                    let rest = &self.src[self.i..];
190                    self.i += rest.iter().position(|&b| b == b'\n').unwrap_or(rest.len());
191                }
192                Some(b'*') => {
193                    let start = self.i;
194                    match self.text[self.i + 2..].find("*/") {
195                        Some(end) => self.i += 2 + end + 2,
196                        None => return Err(self.error_at(start, "unclosed `/*` comment")),
197                    }
198                }
199                _ => return Ok(()),
200            }
201        }
202    }
203
204    fn value(&mut self) -> Result<(Value, Spans), SyntaxError> {
205        let start = self.i;
206        let (value, children) = match self.peek() {
207            Some(b'{') => self.nested(|p| p.map())?,
208            Some(b'[') => self.nested(|p| p.list())?,
209            Some(b'"') => (Value::Str(self.string()?), Vec::new()),
210            Some(b'-' | b'0'..=b'9') => (self.number()?, Vec::new()),
211            Some(b't') if self.src[self.i..].starts_with(b"true") => {
212                self.i += 4;
213                (Value::Bool(true), Vec::new())
214            }
215            Some(b'f') if self.src[self.i..].starts_with(b"false") => {
216                self.i += 5;
217                (Value::Bool(false), Vec::new())
218            }
219            Some(b'n') if self.src[self.i..].starts_with(b"null") => {
220                self.i += 4;
221                (Value::Null, Vec::new())
222            }
223            Some(b'\'') => return Err(self.error("strings use double quotes")),
224            _ => return Err(self.error("expected a value")),
225        };
226        let span = Span {
227            start: start as u32,
228            end: self.i as u32,
229        };
230        Ok((
231            value,
232            Spans {
233                span,
234                key: None,
235                children,
236            },
237        ))
238    }
239
240    fn nested(
241        &mut self,
242        f: impl FnOnce(&mut Self) -> Result<(Value, Vec<Spans>), SyntaxError>,
243    ) -> Result<(Value, Vec<Spans>), SyntaxError> {
244        self.depth += 1;
245        if self.depth > MAX_DEPTH {
246            return Err(self.error(format!("nested deeper than {MAX_DEPTH} levels")));
247        }
248        let r = f(self);
249        self.depth -= 1;
250        r
251    }
252
253    fn list(&mut self) -> Result<(Value, Vec<Spans>), SyntaxError> {
254        self.i += 1;
255        let mut items = Vec::new();
256        let mut spans = Vec::new();
257        loop {
258            self.ws()?;
259            if self.peek() == Some(b']') {
260                self.i += 1;
261                break;
262            }
263            let (v, s) = self.value()?;
264            items.push(v);
265            if SPANS {
266                spans.push(s);
267            }
268            self.ws()?;
269            match self.peek() {
270                Some(b',') => self.i += 1,
271                Some(b']') => {
272                    self.i += 1;
273                    break;
274                }
275                _ => return Err(self.error("expected `,` or `]`")),
276            }
277        }
278        Ok((Value::List(items), spans))
279    }
280
281    fn map(&mut self) -> Result<(Value, Vec<Spans>), SyntaxError> {
282        self.i += 1;
283        let mut map = Map::new();
284        let mut spans = Vec::new();
285        loop {
286            self.ws()?;
287            if self.peek() == Some(b'}') {
288                self.i += 1;
289                break;
290            }
291            let key_start = self.i;
292            let key = match self.peek() {
293                Some(b'"') => self.string()?,
294                Some(c) if c.is_ascii_alphabetic() || c == b'_' || c == b'$' => {
295                    while self
296                        .peek()
297                        .is_some_and(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'$')
298                    {
299                        self.i += 1;
300                    }
301                    self.text[key_start..self.i].to_string()
302                }
303                _ => return Err(self.error("expected a key")),
304            };
305            let key_span = Span {
306                start: key_start as u32,
307                end: self.i as u32,
308            };
309            if map.get(&key).is_some() {
310                return Err(
311                    self.error_at(key_start, format!("`{key}` is already set in this object"))
312                );
313            }
314            self.ws()?;
315            if self.peek() != Some(b':') {
316                return Err(self.error("expected `:` after the key"));
317            }
318            self.i += 1;
319            self.ws()?;
320            let (v, mut s) = self.value()?;
321            s.key = Some(key_span);
322            map.push_unchecked(key, v);
323            if SPANS {
324                spans.push(s);
325            }
326            self.ws()?;
327            match self.peek() {
328                Some(b',') => self.i += 1,
329                Some(b'}') => {
330                    self.i += 1;
331                    break;
332                }
333                _ => return Err(self.error("expected `,` or `}`")),
334            }
335        }
336        Ok((Value::Map(map), spans))
337    }
338
339    fn string(&mut self) -> Result<String, SyntaxError> {
340        let open = self.i;
341        self.i += 1;
342        let mut out = String::new();
343        loop {
344            let rest = &self.src[self.i..];
345            let Some(n) = rest
346                .iter()
347                .position(|&b| b == b'"' || b == b'\\' || b < 0x20)
348            else {
349                return Err(self.error_at(open, "unclosed string"));
350            };
351            out.push_str(&self.text[self.i..self.i + n]);
352            self.i += n;
353            match self.src[self.i] {
354                b'"' => {
355                    self.i += 1;
356                    return Ok(out);
357                }
358                b'\\' => self.escape(&mut out)?,
359                b'\n' => return Err(self.error("line break in a string (write `\\n`)")),
360                _ => return Err(self.error("control character in a string")),
361            }
362        }
363    }
364
365    fn escape(&mut self, out: &mut String) -> Result<(), SyntaxError> {
366        let at = self.i;
367        self.i += 1;
368        let c = self.peek().ok_or_else(|| self.error("unclosed string"))?;
369        self.i += 1;
370        match c {
371            b'"' => out.push('"'),
372            b'\\' => out.push('\\'),
373            b'/' => out.push('/'),
374            b'b' => out.push('\u{8}'),
375            b'f' => out.push('\u{c}'),
376            b'n' => out.push('\n'),
377            b'r' => out.push('\r'),
378            b't' => out.push('\t'),
379            b'u' => {
380                let hi = self.hex4(at)?;
381                let ch = if (0xD800..0xDC00).contains(&hi) {
382                    if !self.src[self.i..].starts_with(b"\\u") {
383                        return Err(self.error_at(at, "unpaired surrogate in `\\u` escape"));
384                    }
385                    self.i += 2;
386                    let lo = self.hex4(at)?;
387                    if !(0xDC00..0xE000).contains(&lo) {
388                        return Err(self.error_at(at, "unpaired surrogate in `\\u` escape"));
389                    }
390                    char::from_u32(0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00))
391                } else {
392                    char::from_u32(hi)
393                };
394                out.push(
395                    ch.ok_or_else(|| self.error_at(at, "unpaired surrogate in `\\u` escape"))?,
396                );
397            }
398            _ => {
399                return Err(self.error_at(
400                    at,
401                    "unknown escape (`\\n`, `\\t`, `\\\"`, `\\\\`, `\\uXXXX`, ...)",
402                ));
403            }
404        }
405        Ok(())
406    }
407
408    fn hex4(&mut self, at: usize) -> Result<u32, SyntaxError> {
409        let digits = self
410            .text
411            .get(self.i..self.i + 4)
412            .filter(|d| d.bytes().all(|b| b.is_ascii_hexdigit()));
413        let d = digits.ok_or_else(|| self.error_at(at, "`\\u` needs four hex digits"))?;
414        self.i += 4;
415        Ok(u32::from_str_radix(d, 16).unwrap())
416    }
417
418    fn number(&mut self) -> Result<Value, SyntaxError> {
419        let start = self.i;
420        if self.peek() == Some(b'-') {
421            self.i += 1;
422        }
423        let digits = |p: &mut Self| {
424            let s = p.i;
425            while p.peek().is_some_and(|c| c.is_ascii_digit()) {
426                p.i += 1;
427            }
428            p.i - s
429        };
430        let int_start = self.i;
431        if digits(self) == 0 {
432            return Err(self.error("expected digits"));
433        }
434        if self.src[int_start] == b'0' && self.i - int_start > 1 {
435            return Err(self.error_at(int_start, "numbers don't start with 0"));
436        }
437        let mut float = false;
438        if self.peek() == Some(b'.') {
439            self.i += 1;
440            float = true;
441            if digits(self) == 0 {
442                return Err(self.error("expected digits after `.`"));
443            }
444        }
445        if matches!(self.peek(), Some(b'e' | b'E')) {
446            self.i += 1;
447            float = true;
448            if matches!(self.peek(), Some(b'+' | b'-')) {
449                self.i += 1;
450            }
451            if digits(self) == 0 {
452                return Err(self.error("expected digits in the exponent"));
453            }
454        }
455        let s = &self.text[start..self.i];
456        if !float && let Ok(n) = s.parse::<i64>() {
457            return Ok(Value::Int(n));
458        }
459        Ok(Value::Float(
460            s.parse::<f64>()
461                .map_err(|_| self.error_at(start, "invalid number"))?,
462        ))
463    }
464}
465
466// ── Writing ─────────────────────────────────────────────────────────────────
467
468/// `value` as compact text (typed values in their string forms).
469pub fn to_string(value: &Value) -> String {
470    let mut out = String::with_capacity(256);
471    write(value, None, 0, &mut out);
472    out
473}
474
475/// `value` as text indented by two spaces, one entry per line.
476pub fn to_string_pretty(value: &Value) -> String {
477    let mut out = String::new();
478    write(value, Some("  "), 0, &mut out);
479    out.push('\n');
480    out
481}
482
483fn write(value: &Value, indent: Option<&str>, level: usize, out: &mut String) {
484    let newline = |out: &mut String, level: usize| {
485        if let Some(unit) = indent {
486            out.push('\n');
487            for _ in 0..level {
488                out.push_str(unit);
489            }
490        }
491    };
492    match value {
493        Value::Null => out.push_str("null"),
494        Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
495        Value::Int(n) => {
496            use std::fmt::Write as _;
497            let _ = write!(out, "{n}");
498        }
499        Value::Float(f) => write_float(*f, out),
500        Value::List(items) => {
501            if items.is_empty() {
502                out.push_str("[]");
503                return;
504            }
505            out.push('[');
506            for (i, v) in items.iter().enumerate() {
507                if i > 0 {
508                    out.push(',');
509                }
510                newline(out, level + 1);
511                write(v, indent, level + 1, out);
512            }
513            newline(out, level);
514            out.push(']');
515        }
516        Value::Map(map) => {
517            if map.is_empty() {
518                out.push_str("{}");
519                return;
520            }
521            out.push('{');
522            for (i, (k, v)) in map.iter().enumerate() {
523                if i > 0 {
524                    out.push(',');
525                }
526                newline(out, level + 1);
527                write_string(k, out);
528                out.push_str(if indent.is_some() { ": " } else { ":" });
529                write(v, indent, level + 1, out);
530            }
531            newline(out, level);
532            out.push('}');
533        }
534        Value::Str(s) | Value::Text(s) => {
535            if s.starts_with('@') {
536                out.push('"');
537                out.push('@');
538                write_string_body(s, out);
539                out.push('"');
540            } else {
541                write_string(s, out);
542            }
543        }
544        typed => write_string(&crate::typed::typed_string(typed).unwrap_or_default(), out),
545    }
546}
547
548fn write_float(f: f64, out: &mut String) {
549    if !f.is_finite() {
550        out.push_str("null");
551    } else if f.fract() == 0.0 && f.abs() < 1e16 {
552        out.push_str(&format!("{f:.1}"));
553    } else {
554        out.push_str(&f.to_string());
555    }
556}
557
558/// `s` as a JSON string literal.
559pub fn write_string(s: &str, out: &mut String) {
560    out.push('"');
561    write_string_body(s, out);
562    out.push('"');
563}
564
565fn write_string_body(s: &str, out: &mut String) {
566    let bytes = s.as_bytes();
567    let mut run = 0;
568    while let Some(n) = bytes[run..]
569        .iter()
570        .position(|&b| b == b'"' || b == b'\\' || b < 0x20)
571    {
572        let i = run + n;
573        out.push_str(&s[run..i]);
574        match bytes[i] {
575            b'"' => out.push_str("\\\""),
576            b'\\' => out.push_str("\\\\"),
577            b'\n' => out.push_str("\\n"),
578            b'\r' => out.push_str("\\r"),
579            b'\t' => out.push_str("\\t"),
580            b => {
581                use std::fmt::Write as _;
582                let _ = write!(out, "\\u{b:04x}");
583            }
584        }
585        run = i + 1;
586    }
587    out.push_str(&s[run..]);
588}
589
590#[cfg(test)]
591mod tests {
592    use super::*;
593
594    #[test]
595    fn json5_subset() {
596        let doc = parse("\u{feff}{\n  // comment\n  name: \"Ruby\", /* inline */ $schema: 1,\n  \"list\": [1, 2.5, -3e2, true, null,],\n}").unwrap();
597        let m = doc.value.as_map().unwrap();
598        assert_eq!(m.keys().collect::<Vec<_>>(), ["name", "$schema", "list"]);
599        assert_eq!(
600            doc.value.get("list").unwrap(),
601            &Value::List(vec![
602                Value::Int(1),
603                Value::Float(2.5),
604                Value::Float(-300.0),
605                Value::Bool(true),
606                Value::Null
607            ])
608        );
609    }
610
611    #[test]
612    fn locates_values_and_keys() {
613        let text = "{\n  \"a\": [\n    1,\n    {\"b\": \"x\"}\n  ]\n}";
614        let doc = parse(text).unwrap();
615        let path = Path::root().key("a").index(1).key("b");
616        assert_eq!(doc.locate(text, &path), LineCol { line: 4, col: 11 });
617        assert_eq!(
618            LineCol::of(text, doc.spans_at(&path).key.unwrap().start as usize),
619            LineCol { line: 4, col: 6 }
620        );
621        // A path past the document: its deepest existing part.
622        assert_eq!(
623            doc.locate(text, &Path::root().key("a").index(9)),
624            LineCol { line: 2, col: 8 }
625        );
626    }
627
628    #[test]
629    fn errors_say_where() {
630        let cases = [
631            ("{\"a\": 1,, }", "1:9", "expected a key"),
632            ("{\"a\": 1 \"b\": 2}", "1:9", "expected `,` or `}`"),
633            ("{\"a\": 1, \"a\": 2}", "1:10", "already set"),
634            ("[1, 2", "1:6", "found the end of the file"),
635            ("\"ab\ncd\"", "1:4", "line break"),
636            ("'x'", "1:1", "double quotes"),
637            ("01", "1:1", "start with 0"),
638            ("{} x", "1:4", "unexpected text"),
639            ("/* never closed", "1:1", "unclosed"),
640            ("\"\\q\"", "1:2", "unknown escape"),
641            ("\"\\ud800\"", "1:2", "surrogate"),
642            ("{\n  \"é\": tru }", "2:8", "expected a value"),
643        ];
644        for (text, at, msg) in cases {
645            let e = parse(text).unwrap_err();
646            assert_eq!(e.at.to_string(), at, "{text:?}: {e}");
647            assert!(e.message.contains(msg), "{text:?}: {e}");
648        }
649        assert!(
650            parse(&"[".repeat(MAX_DEPTH + 1))
651                .unwrap_err()
652                .message
653                .contains("nested deeper")
654        );
655    }
656
657    #[test]
658    fn strings_and_numbers() {
659        let v = parse_value(
660            r#"["a\"b\\c\n\u00e9\ud83d\ude00", 9223372036854775807, 9223372036854775808, 0.1]"#,
661        )
662        .unwrap();
663        let l = v.as_list().unwrap();
664        assert_eq!(l[0], Value::Str("a\"b\\c\né😀".into()));
665        assert_eq!(l[1], Value::Int(i64::MAX));
666        assert_eq!(l[2], Value::Float(9223372036854775808.0));
667        assert_eq!(l[3], Value::Float(0.1));
668    }
669
670    #[test]
671    fn writes_what_it_reads() {
672        let text = "{\n  \"name\": \"Ruby \\\"red\\\"\\n\",\n  \"n\": [1, 2.0, 0.1, -5],\n  \"empty\": {},\n  \"none\": [],\n  \"tab\\u0001\": null\n}\n";
673        let v = parse_value(text).unwrap();
674        assert_eq!(parse_value(&to_string(&v)).unwrap(), v);
675        let pretty = to_string_pretty(&v);
676        assert_eq!(parse_value(&pretty).unwrap(), v);
677        assert!(
678            pretty.starts_with(
679                "{\n  \"name\": \"Ruby \\\"red\\\"\\n\",\n  \"n\": [\n    1,\n    2.0,"
680            ),
681            "{pretty}"
682        );
683        assert_eq!(
684            to_string(&v),
685            r#"{"name":"Ruby \"red\"\n","n":[1,2.0,0.1,-5],"empty":{},"none":[],"tab\u0001":null}"#
686        );
687    }
688
689    #[test]
690    fn typed_values_are_strings_in_text() {
691        let v = Value::List(vec![
692            Value::Ref(crate::Ref::new("item", "minecraft:stick")),
693            Value::Color(0xFF112233),
694            Value::Str("@not a ref".into()),
695        ]);
696        assert_eq!(
697            to_string(&v),
698            r##"["@item:minecraft:stick","#112233","@@not a ref"]"##
699        );
700    }
701}
yog_data/text.rs

yog_data/
text.rs
