tabnas/
token.rs

1// Copyright (c) 2013-2026 Richard Rodger, MIT License
2
3use crate::value::unwrap_arc;
4use crate::value::Value;
5use crate::{Context, Rule};
6use std::collections::HashMap;
7use std::fmt;
8use std::sync::Arc;
9
10type TokenValCallback = dyn Fn(&mut Rule, &mut Context) -> Value + Send + Sync;
11
12/// Tin is a token identification number.
13pub type Tin = i32;
14
15pub const TIN_BD: Tin = 1; // #BD - BAD
16pub const TIN_ZZ: Tin = 2; // #ZZ - END
17pub const TIN_UK: Tin = 3; // #UK - UNKNOWN
18pub const TIN_AA: Tin = 4; // #AA - ANY
19pub const TIN_SP: Tin = 5; // #SP - SPACE
20pub const TIN_LN: Tin = 6; // #LN - LINE
21pub const TIN_CM: Tin = 7; // #CM - COMMENT
22pub const TIN_NR: Tin = 8; // #NR - NUMBER
23pub const TIN_ST: Tin = 9; // #ST - STRING
24pub const TIN_TX: Tin = 10; // #TX - TEXT
25pub const TIN_VL: Tin = 11; // #VL - VALUE (true, false, null)
26pub const TIN_OB: Tin = 12; // #OB - Open Brace {
27pub const TIN_CB: Tin = 13; // #CB - Close Brace }
28pub const TIN_OS: Tin = 14; // #OS - Open Square [
29pub const TIN_CS: Tin = 15; // #CS - Close Square ]
30pub const TIN_CL: Tin = 16; // #CL - Colon :
31pub const TIN_CA: Tin = 17; // #CA - Comma ,
32pub const TIN_MAX: Tin = 18;
33
34pub fn tin_name(tin: Tin) -> &'static str {
35    match tin {
36        TIN_BD => "#BD",
37        TIN_ZZ => "#ZZ",
38        TIN_UK => "#UK",
39        TIN_AA => "#AA",
40        TIN_SP => "#SP",
41        TIN_LN => "#LN",
42        TIN_CM => "#CM",
43        TIN_NR => "#NR",
44        TIN_ST => "#ST",
45        TIN_TX => "#TX",
46        TIN_VL => "#VL",
47        TIN_OB => "#OB",
48        TIN_CB => "#CB",
49        TIN_OS => "#OS",
50        TIN_CS => "#CS",
51        TIN_CL => "#CL",
52        TIN_CA => "#CA",
53        _ => "#UNKNOWN",
54    }
55}
56
57pub fn name_to_tin(name: &str) -> Option<Tin> {
58    match name {
59        "#BD" | "BD" => Some(TIN_BD),
60        "#ZZ" | "ZZ" => Some(TIN_ZZ),
61        "#UK" | "UK" => Some(TIN_UK),
62        "#AA" | "AA" => Some(TIN_AA),
63        "#SP" | "SP" => Some(TIN_SP),
64        "#LN" | "LN" => Some(TIN_LN),
65        "#CM" | "CM" => Some(TIN_CM),
66        "#NR" | "NR" => Some(TIN_NR),
67        "#ST" | "ST" => Some(TIN_ST),
68        "#TX" | "TX" => Some(TIN_TX),
69        "#VL" | "VL" => Some(TIN_VL),
70        "#OB" | "OB" => Some(TIN_OB),
71        "#CB" | "CB" => Some(TIN_CB),
72        "#OS" | "OS" => Some(TIN_OS),
73        "#CS" | "CS" => Some(TIN_CS),
74        "#CL" | "CL" => Some(TIN_CL),
75        "#CA" | "CA" => Some(TIN_CA),
76        _ => None,
77    }
78}
79
80/// One location in the source text: the index reached, and the row and
81/// column that index sits at. It is the same data TypeScript carries as
82/// the loose `sI`/`rI`/`cI` fields on both `Point` and `Token`
83/// (ts/src/lexer.ts), and as the `ScanOut` record its scan driver writes
84/// back — naming it once is what keeps those uses in step.
85///
86/// The extra field is `pos`, and it is not extra data: TypeScript indexes
87/// source by UTF-16 code unit and reports that same number in a
88/// diagnostic, while Rust slices by UTF-8 byte. So `si` is the offset
89/// this port slices with and `pos` is the offset it REPORTS, and the two
90/// together say what TypeScript's single `sI` says. See
91/// `go/doc/differences.md` and DIVERGENCE.md "Column positions for astral
92/// characters" for what that costs at the edges.
93///
94/// Deliberately NOT carrying a length: `Point::len` is the length of the
95/// whole source and `Token::len` is the length of that token's matched
96/// text, so the two mean different things and only the position is shared.
97#[derive(Debug, Clone, Copy, PartialEq, Eq)]
98pub struct Site {
99    pub si: usize,  // 0-based UTF-8 byte position used for source slicing
100    pub pos: usize, // 0-based Unicode-scalar position used by diagnostics
101    pub ri: usize,  // 1-based row
102    pub ci: usize,  // 1-based column
103}
104
105impl Default for Site {
106    fn default() -> Self {
107        Site {
108            si: 0,
109            pos: 0,
110            ri: 1,
111            ci: 1,
112        }
113    }
114}
115
116impl fmt::Display for Site {
117    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
118        write!(formatter, "{},{},{}", self.si, self.ri, self.ci)
119    }
120}
121
122/// Cursor position within the source text.
123#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
124pub struct Point {
125    pub len: usize, // Total UTF-8 byte length of the source.
126    pub site: Site, // Where the cursor currently sits.
127}
128
129impl fmt::Display for Point {
130    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
131        write!(
132            formatter,
133            "Point[{}/{},{},{}]",
134            self.site.si, self.len, self.site.ri, self.site.ci
135        )
136    }
137}
138
139/// Lazy token value callback, evaluated only when a parser action asks for
140/// the token's semantic value. This is the Rust counterpart of the canonical
141/// `TokenValFunc` `(rule, context) => value` extension point.
142#[derive(Clone)]
143pub struct TokenValFunc {
144    callback: Arc<TokenValCallback>,
145}
146
147impl TokenValFunc {
148    pub fn new(
149        callback: impl Fn(&mut Rule, &mut Context) -> Value + Send + Sync + 'static,
150    ) -> Self {
151        Self {
152            callback: Arc::new(callback),
153        }
154    }
155
156    pub fn call(&self, rule: &mut Rule, context: &mut Context) -> Value {
157        (self.callback)(rule, context)
158    }
159
160    pub(crate) fn same_callback(&self, other: &Self) -> bool {
161        Arc::ptr_eq(&self.callback, &other.callback)
162    }
163}
164
165impl fmt::Debug for TokenValFunc {
166    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
167        formatter.write_str("TokenValFunc(<function>)")
168    }
169}
170
171impl PartialEq for TokenValFunc {
172    fn eq(&self, other: &Self) -> bool {
173        self.same_callback(other)
174    }
175}
176
177impl Eq for TokenValFunc {}
178
179/// A token's name or matched source text.
180///
181/// Tokens are cloned about six times per input construct, and each
182/// clone used to copy both of these strings. Sharing them makes a
183/// clone a pointer copy; names go further and are interned per token
184/// identity, so lexing one costs nothing at all. It behaves like the
185/// `String` it replaced: compare it with a literal, print it, index
186/// it, or take a `&str` from it.
187#[derive(Clone, Default)]
188pub struct TokenText(crate::text::InlineText);
189
190impl TokenText {
191    pub fn as_str(&self) -> &str {
192        self.0.as_str()
193    }
194
195    pub fn is_empty(&self) -> bool {
196        self.as_str().is_empty()
197    }
198
199    fn build(text: &str) -> Self {
200        TokenText(crate::text::InlineText::new(text))
201    }
202}
203
204impl Eq for TokenText {}
205
206impl PartialEq for TokenText {
207    fn eq(&self, other: &Self) -> bool {
208        self.as_str() == other.as_str()
209    }
210}
211
212impl std::hash::Hash for TokenText {
213    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
214        self.as_str().hash(state)
215    }
216}
217
218impl PartialOrd for TokenText {
219    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
220        Some(self.cmp(other))
221    }
222}
223
224impl Ord for TokenText {
225    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
226        self.as_str().cmp(other.as_str())
227    }
228}
229
230impl std::ops::Deref for TokenText {
231    type Target = str;
232
233    fn deref(&self) -> &str {
234        self.as_str()
235    }
236}
237
238impl AsRef<str> for TokenText {
239    fn as_ref(&self) -> &str {
240        self.as_str()
241    }
242}
243
244impl std::borrow::Borrow<str> for TokenText {
245    fn borrow(&self) -> &str {
246        self.as_str()
247    }
248}
249
250impl fmt::Display for TokenText {
251    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
252        f.write_str(self.as_str())
253    }
254}
255
256/// Printed as the bare text, so a `{:?}` of a token reads the way it
257/// did when these were `String`s.
258impl fmt::Debug for TokenText {
259    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
260        fmt::Debug::fmt(self.as_str(), f)
261    }
262}
263
264impl PartialEq<str> for TokenText {
265    fn eq(&self, other: &str) -> bool {
266        self.as_str() == other
267    }
268}
269
270impl PartialEq<&str> for TokenText {
271    fn eq(&self, other: &&str) -> bool {
272        self.as_str() == *other
273    }
274}
275
276impl PartialEq<String> for TokenText {
277    fn eq(&self, other: &String) -> bool {
278        self.as_str() == other.as_str()
279    }
280}
281
282impl PartialEq<TokenText> for str {
283    fn eq(&self, other: &TokenText) -> bool {
284        self == other.as_str()
285    }
286}
287
288impl PartialEq<TokenText> for &str {
289    fn eq(&self, other: &TokenText) -> bool {
290        *self == other.as_str()
291    }
292}
293
294impl PartialEq<TokenText> for String {
295    fn eq(&self, other: &TokenText) -> bool {
296        self.as_str() == other.as_str()
297    }
298}
299
300impl From<&str> for TokenText {
301    fn from(text: &str) -> Self {
302        TokenText::build(text)
303    }
304}
305
306impl From<String> for TokenText {
307    fn from(text: String) -> Self {
308        TokenText::build(text.as_str())
309    }
310}
311
312impl From<&String> for TokenText {
313    fn from(text: &String) -> Self {
314        TokenText::build(text.as_str())
315    }
316}
317
318impl From<Arc<str>> for TokenText {
319    fn from(text: Arc<str>) -> Self {
320        TokenText(crate::text::InlineText::shared(text))
321    }
322}
323
324impl From<TokenText> for String {
325    fn from(text: TokenText) -> Self {
326        text.as_str().to_string()
327    }
328}
329
330/// A token's diagnostic code, held behind a pointer.
331///
332/// `err` and `why` are empty on virtually every token of every parse, and a
333/// `Token` is moved and cloned several times per input construct. Carried
334/// inline as `TokenText` the pair took 48 of the token's 248 bytes. Padding
335/// `Token` by those 32 bytes measured 1.2% to 3.8% across the benchmark
336/// rows, which is what carrying them inline was costing; behind a pointer
337/// the pair costs 16 bytes and one allocation on the error path, which is
338/// already the expensive one.
339///
340/// The surface matches `TokenText`, so reading code does not change: it
341/// derefs, compares and prints as `str`, and an absent code reads as the
342/// empty string. Setting one to `""` stores nothing, so `is_empty` answers
343/// the same question either way.
344#[derive(Clone, Default)]
345pub struct TokenCode(Option<Box<TokenText>>);
346
347impl TokenCode {
348    pub fn as_str(&self) -> &str {
349        match &self.0 {
350            Some(text) => text.as_str(),
351            None => "",
352        }
353    }
354
355    pub fn is_empty(&self) -> bool {
356        self.0.is_none()
357    }
358
359    fn build(text: &str) -> Self {
360        TokenCode((!text.is_empty()).then(|| Box::new(TokenText::from(text))))
361    }
362}
363
364impl Eq for TokenCode {}
365
366impl PartialEq for TokenCode {
367    fn eq(&self, other: &Self) -> bool {
368        self.as_str() == other.as_str()
369    }
370}
371
372impl std::hash::Hash for TokenCode {
373    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
374        self.as_str().hash(state)
375    }
376}
377
378impl PartialOrd for TokenCode {
379    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
380        Some(self.cmp(other))
381    }
382}
383
384impl Ord for TokenCode {
385    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
386        self.as_str().cmp(other.as_str())
387    }
388}
389
390impl std::ops::Deref for TokenCode {
391    type Target = str;
392
393    fn deref(&self) -> &str {
394        self.as_str()
395    }
396}
397
398impl AsRef<str> for TokenCode {
399    fn as_ref(&self) -> &str {
400        self.as_str()
401    }
402}
403
404impl std::borrow::Borrow<str> for TokenCode {
405    fn borrow(&self) -> &str {
406        self.as_str()
407    }
408}
409
410impl fmt::Display for TokenCode {
411    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
412        f.write_str(self.as_str())
413    }
414}
415
416/// Printed as the bare text, like `TokenText`.
417impl fmt::Debug for TokenCode {
418    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
419        fmt::Debug::fmt(self.as_str(), f)
420    }
421}
422
423impl PartialEq<str> for TokenCode {
424    fn eq(&self, other: &str) -> bool {
425        self.as_str() == other
426    }
427}
428
429impl PartialEq<&str> for TokenCode {
430    fn eq(&self, other: &&str) -> bool {
431        self.as_str() == *other
432    }
433}
434
435impl PartialEq<String> for TokenCode {
436    fn eq(&self, other: &String) -> bool {
437        self.as_str() == other.as_str()
438    }
439}
440
441impl PartialEq<TokenCode> for str {
442    fn eq(&self, other: &TokenCode) -> bool {
443        self == other.as_str()
444    }
445}
446
447impl PartialEq<TokenCode> for &str {
448    fn eq(&self, other: &TokenCode) -> bool {
449        *self == other.as_str()
450    }
451}
452
453impl PartialEq<TokenCode> for String {
454    fn eq(&self, other: &TokenCode) -> bool {
455        self.as_str() == other.as_str()
456    }
457}
458
459impl From<&str> for TokenCode {
460    fn from(text: &str) -> Self {
461        TokenCode::build(text)
462    }
463}
464
465impl From<String> for TokenCode {
466    fn from(text: String) -> Self {
467        TokenCode::build(text.as_str())
468    }
469}
470
471impl From<&String> for TokenCode {
472    fn from(text: &String) -> Self {
473        TokenCode::build(text.as_str())
474    }
475}
476
477impl From<TokenText> for TokenCode {
478    fn from(text: TokenText) -> Self {
479        TokenCode((!text.is_empty()).then(|| Box::new(text)))
480    }
481}
482
483impl From<TokenCode> for String {
484    fn from(text: TokenCode) -> Self {
485        text.as_str().to_string()
486    }
487}
488
489/// A single lexical token produced by the lexer.
490#[derive(Debug, Clone, PartialEq)]
491pub struct Token {
492    pub name: TokenText,
493    pub tin: Tin,
494    pub val: Value,
495    pub src: TokenText,
496    /// UTF-8 byte length of `src`, paired with the byte offset `site.si`.
497    pub len: usize,
498    /// Where in the source this token starts.
499    pub site: Site,
500    pub err: TokenCode,
501    pub why: TokenCode,
502    /// Plugin diagnostic details, boxed and absent until something
503    /// writes one. Prefer `use_data()` and `use_data_mut()` to reaching
504    /// through the `Option`; the shape is public only so that a `Token`
505    /// can still be built with a struct literal. It is boxed because a `HashMap` is 48 bytes inline, a token is
506    /// cloned about six times per input construct, and almost no token
507    /// ever carries a detail. Measured by padding `Token`, 96 bytes of
508    /// it is worth 3% to 12% depending on the grammar.
509    pub use_data: Option<Box<HashMap<String, Value>>>,
510    /// Optional ignored trivia associated with this token. Negotiated
511    /// re-lexing carries it to the replacement token.
512    pub ignored: Option<Box<Token>>,
513    /// Optional semantic value callback. `val` remains the eagerly produced
514    /// fallback and the value shown by raw token inspection.
515    pub val_fn: Option<TokenValFunc>,
516}
517
518impl Default for Token {
519    fn default() -> Self {
520        Token {
521            name: TokenText::default(),
522            tin: -1,
523            val: Value::Undefined,
524            src: TokenText::default(),
525            len: 0,
526            site: Site::default(),
527            err: TokenCode::default(),
528            why: TokenCode::default(),
529            use_data: None,
530            ignored: None,
531            val_fn: None,
532        }
533    }
534}
535
536impl Token {
537    pub fn new(
538        name: impl AsRef<str>,
539        tin: Tin,
540        val: Value,
541        src: impl Into<TokenText>,
542        pnt: Point,
543    ) -> Self {
544        let src: TokenText = src.into();
545        let len = src.len();
546        Token {
547            name: TokenText::from(name.as_ref()),
548            tin,
549            val,
550            src,
551            len,
552            site: pnt.site,
553            err: TokenCode::default(),
554            why: TokenCode::default(),
555            use_data: None,
556            ignored: None,
557            val_fn: None,
558        }
559    }
560
561    /// Plugin diagnostic details. Empty unless something wrote one.
562    pub fn use_data(&self) -> &HashMap<String, Value> {
563        static EMPTY: std::sync::OnceLock<HashMap<String, Value>> = std::sync::OnceLock::new();
564        match &self.use_data {
565            Some(details) => details,
566            None => EMPTY.get_or_init(HashMap::new),
567        }
568    }
569
570    /// Mutable access, allocating the bag on first write.
571    pub fn use_data_mut(&mut self) -> &mut HashMap<String, Value> {
572        self.use_data.get_or_insert_with(Box::default)
573    }
574
575    pub fn no_token() -> Self {
576        Token {
577            // The canonical sentinel has no public token name; identity is
578            // carried by tin -1 rather than a synthetic grammar token.
579            name: TokenText::default(),
580            tin: -1,
581            val: Value::Undefined,
582            src: TokenText::default(),
583            len: 0,
584            site: Site::default(),
585            err: TokenCode::default(),
586            why: TokenCode::default(),
587            use_data: None,
588            ignored: None,
589            val_fn: None,
590        }
591    }
592
593    pub fn is_no_token(&self) -> bool {
594        self.tin == -1
595    }
596
597    pub fn bad(&mut self, err: &str) -> &mut Self {
598        self.err = TokenCode::from(err);
599        self
600    }
601
602    /// Mark this token bad and deep-merge plugin diagnostic details into its
603    /// existing `use_data` bag. This is the typed Rust form of
604    /// `token.bad(code, details)`.
605    pub fn bad_with_details(
606        &mut self,
607        err: &str,
608        details: impl IntoIterator<Item = (String, Value)>,
609    ) -> &mut Self {
610        self.err = TokenCode::from(err);
611        for (key, value) in details {
612            let details = self.use_data_mut();
613            let previous = details.remove(&key).unwrap_or(Value::Undefined);
614            details.insert(key, merge_detail(previous, value));
615        }
616        self
617    }
618
619    /// Attach an already shared lazy value callback.
620    pub fn with_val_func(mut self, callback: TokenValFunc) -> Self {
621        self.val_fn = Some(callback);
622        self
623    }
624
625    /// Attach a lazy value callback without constructing a wrapper first.
626    pub fn with_lazy_value(
627        self,
628        callback: impl Fn(&mut Rule, &mut Context) -> Value + Send + Sync + 'static,
629    ) -> Self {
630        self.with_val_func(TokenValFunc::new(callback))
631    }
632
633    /// Resolve the semantic token value against the live parse state.
634    pub fn resolve_val(&self, rule: &mut Rule, context: &mut Context) -> Value {
635        self.val_fn
636            .as_ref()
637            .map_or_else(|| self.val.clone(), |callback| callback.call(rule, context))
638    }
639}
640
641impl fmt::Display for Token {
642    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
643        write!(
644            formatter,
645            "Token[{}={} {}",
646            self.name,
647            self.tin,
648            snip(&self.src, 5)
649        )?;
650        if !self.val.is_undefined() && !matches!(self.name.as_str(), "#ST" | "#TX") {
651            write!(formatter, "={}", snip(&value_text(&self.val), 5))?;
652        }
653        write!(formatter, " {}", self.site)?;
654        if !self.use_data().is_empty() {
655            let mut entries = self.use_data().iter().collect::<Vec<_>>();
656            entries.sort_by_key(|(key, _)| *key);
657            let details = entries
658                .into_iter()
659                .map(|(key, value)| format!("{key}:{}", detail_json(value)))
660                .collect::<Vec<_>>()
661                .join(",");
662            write!(
663                formatter,
664                " {}",
665                snip(&format!("{{{details}}}").replace('"', ""), 22)
666            )?;
667        }
668        if !self.err.is_empty() {
669            write!(formatter, " {}", self.err)?;
670        }
671        if !self.why.is_empty() {
672            write!(formatter, " {}", snip(&self.why, 22))?;
673        }
674        formatter.write_str("]")
675    }
676}
677
678fn merge_detail(base: Value, overlay: Value) -> Value {
679    match (base, overlay) {
680        (base, Value::Undefined) => base,
681        (Value::Object(base), Value::Object(overlay)) => {
682            let mut base = unwrap_arc(base);
683            for (key, value) in unwrap_arc(overlay) {
684                let previous = base.shift_remove(&key).unwrap_or(Value::Undefined);
685                base.insert(key, merge_detail(previous, value));
686            }
687            Value::object(base)
688        }
689        (Value::Array(base), Value::Array(overlay)) => {
690            let mut base = unwrap_arc(base);
691            let overlay = unwrap_arc(overlay);
692            if base.len() < overlay.len() {
693                base.resize(overlay.len(), Value::Undefined);
694            }
695            for (index, value) in overlay.into_iter().enumerate() {
696                let previous = std::mem::replace(&mut base[index], Value::Undefined);
697                base[index] = merge_detail(previous, value);
698            }
699            Value::array(base)
700        }
701        (_, overlay) => overlay,
702    }
703}
704
705fn value_text(value: &Value) -> String {
706    match value {
707        Value::Undefined => String::new(),
708        Value::Null => "null".into(),
709        Value::Bool(value) => value.to_string(),
710        Value::Number(value) => Value::Number(*value).to_string(),
711        Value::String(value) | Value::Text(crate::Text { string: value, .. }) => value.clone(),
712        Value::Array(values) => values.iter().map(value_text).collect::<Vec<_>>().join(","),
713        Value::ListRef(list) => list
714            .value
715            .iter()
716            .map(value_text)
717            .collect::<Vec<_>>()
718            .join(","),
719        Value::Object(_) | Value::MapRef(_) => "[object Object]".into(),
720    }
721}
722
723fn detail_json(value: &Value) -> String {
724    match value {
725        Value::Undefined | Value::Null => "null".into(),
726        Value::Bool(value) => value.to_string(),
727        Value::Number(value) => Value::Number(*value).to_string(),
728        Value::String(value) | Value::Text(crate::Text { string: value, .. }) => {
729            serde_json::to_string(value).unwrap_or_default()
730        }
731        Value::Array(values) => format!(
732            "[{}]",
733            values.iter().map(detail_json).collect::<Vec<_>>().join(",")
734        ),
735        Value::Object(values) => format!(
736            "{{{}}}",
737            values
738                .iter()
739                // JSON.stringify omits undefined-valued object properties
740                // (while array slots below render as null).
741                .filter(|(_, value)| !value.is_undefined())
742                .map(|(key, value)| format!(
743                    "{}:{}",
744                    serde_json::to_string(key).unwrap_or_default(),
745                    detail_json(value)
746                ))
747                .collect::<Vec<_>>()
748                .join(",")
749        ),
750        Value::ListRef(list) => detail_json(&Value::array(list.value.clone())),
751        Value::MapRef(map) => detail_json(&Value::object(map.value.clone())),
752    }
753}
754
755fn snip(value: &str, max_len: usize) -> String {
756    value
757        .chars()
758        .take(max_len)
759        .map(|character| match character {
760            '\r' | '\n' | '\t' => '.',
761            character => character,
762        })
763        .collect()
764}
tabnas/token.rs

tabnas/
token.rs
