Skip to main content

tabnas/
options.rs

1// Copyright (c) 2013-2026 Richard Rodger, MIT License
2
3use crate::token::{Tin, TIN_CM, TIN_LN, TIN_MAX, TIN_NR, TIN_SP, TIN_ST, TIN_TX, TIN_VL};
4use indexmap::IndexMap;
5use regex::Regex;
6use std::collections::HashMap;
7use std::fmt;
8use std::sync::Arc;
9
10use crate::context::Context;
11use crate::lexer::Lexer;
12use crate::rule::Rule;
13use crate::token::Token;
14
15type ConfigModifierCallback = dyn Fn(&mut Options, &Options) + Send + Sync;
16
17#[derive(Clone)]
18pub struct ConfigModifier {
19    callback: Arc<ConfigModifierCallback>,
20}
21
22impl ConfigModifier {
23    pub(crate) fn new(callback: impl Fn(&mut Options) + Send + Sync + 'static) -> Self {
24        Self {
25            callback: Arc::new(move |config, _options| callback(config)),
26        }
27    }
28
29    pub(crate) fn with_options(
30        callback: impl Fn(&mut Options, &Options) + Send + Sync + 'static,
31    ) -> Self {
32        Self {
33            callback: Arc::new(callback),
34        }
35    }
36
37    pub(crate) fn run(&self, config: &mut Options, options: &Options) {
38        (self.callback)(config, options);
39    }
40
41    pub(crate) fn same_callback(&self, other: &Self) -> bool {
42        Arc::ptr_eq(&self.callback, &other.callback)
43    }
44}
45
46impl fmt::Debug for ConfigModifier {
47    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
48        formatter.write_str("ConfigModifier(<function>)")
49    }
50}
51
52#[derive(Debug, Clone, PartialEq, Eq)]
53pub struct ColorOptions {
54    pub active: bool,
55    pub reset: String,
56    pub hi: String,
57    pub lo: String,
58    pub line: String,
59}
60
61impl ColorOptions {
62    pub(crate) fn codes(&self) -> (&str, &str, &str, &str) {
63        if self.active {
64            (&self.hi, &self.lo, &self.line, &self.reset)
65        } else {
66            ("", "", "", "")
67        }
68    }
69}
70
71impl Default for ColorOptions {
72    fn default() -> Self {
73        Self {
74            active: true,
75            reset: "\x1b[0m".into(),
76            hi: "\x1b[91m".into(),
77            lo: "\x1b[2m".into(),
78            line: "\x1b[34m".into(),
79        }
80    }
81}
82
83#[derive(Debug, Clone, PartialEq, Eq)]
84pub struct ErrorSuffixContext {
85    pub code: String,
86    pub source: String,
87    pub message: String,
88    pub hint: String,
89    pub pos: usize,
90    pub row: usize,
91    pub col: usize,
92    pub name: String,
93    pub tag: String,
94    pub rule: String,
95    pub rule_state: String,
96    pub token: String,
97    pub why: String,
98    pub plugins: Vec<String>,
99    pub color: ColorOptions,
100}
101
102pub type ErrorSuffixCallback = Arc<dyn Fn(&ErrorSuffixContext) -> String + Send + Sync>;
103
104#[derive(Clone)]
105pub enum ErrorSuffix {
106    Standard,
107    Disabled,
108    Text(String),
109    Callback(ErrorSuffixCallback),
110}
111
112impl fmt::Debug for ErrorSuffix {
113    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
114        match self {
115            Self::Standard => formatter.write_str("Standard"),
116            Self::Disabled => formatter.write_str("Disabled"),
117            Self::Text(text) => formatter.debug_tuple("Text").field(text).finish(),
118            Self::Callback(_) => formatter.write_str("Callback(<function>)"),
119        }
120    }
121}
122
123impl PartialEq for ErrorSuffix {
124    fn eq(&self, other: &Self) -> bool {
125        match (self, other) {
126            (Self::Standard, Self::Standard) | (Self::Disabled, Self::Disabled) => true,
127            (Self::Text(left), Self::Text(right)) => left == right,
128            (Self::Callback(left), Self::Callback(right)) => Arc::ptr_eq(left, right),
129            _ => false,
130        }
131    }
132}
133
134impl Eq for ErrorSuffix {}
135
136#[derive(Debug, Clone, PartialEq, Eq)]
137pub struct ErrMsgOptions {
138    pub name: String,
139    pub suffix: ErrorSuffix,
140    pub link: String,
141}
142
143impl Default for ErrMsgOptions {
144    fn default() -> Self {
145        Self {
146            name: "tabnas".into(),
147            suffix: ErrorSuffix::Standard,
148            link: String::new(),
149        }
150    }
151}
152
153#[derive(Debug, Clone, PartialEq)]
154pub struct LexCheckToken {
155    pub name: String,
156    pub tin: Tin,
157    pub source: String,
158    pub value: crate::Value,
159}
160
161impl LexCheckToken {
162    pub fn new(
163        name: impl Into<String>,
164        tin: Tin,
165        source: impl Into<String>,
166        value: crate::Value,
167    ) -> Self {
168        Self {
169            name: name.into(),
170            tin,
171            source: source.into(),
172            value,
173        }
174    }
175
176    /// Build a token effect whose numeric identity is resolved from its name
177    /// after the serialized grammar has declared that token.
178    pub fn named(name: impl Into<String>, source: impl Into<String>, value: crate::Value) -> Self {
179        let name = name.into();
180        Self {
181            name: if name.starts_with('#') {
182                name
183            } else {
184                format!("#{name}")
185            },
186            tin: -1,
187            source: source.into(),
188            value,
189        }
190    }
191}
192
193/// Effect returned by a matcher-family preflight hook.
194#[derive(Debug, Clone, PartialEq)]
195pub enum LexCheckResult {
196    /// Run the matcher's normal implementation.
197    Continue,
198    /// Skip this matcher and try the next matcher family.
199    Skip,
200    /// Emit an owned token while consuming its non-empty source prefix.
201    Token(Box<LexCheckToken>),
202    /// Return a token built by a live lexer callback. The callback owns all
203    /// cursor movement, matching the canonical `LexCheck(lex)` contract.
204    NativeToken(Box<Token>),
205}
206
207impl LexCheckResult {
208    pub fn token(token: LexCheckToken) -> Self {
209        Self::Token(Box::new(token))
210    }
211
212    pub fn native_token(token: Token) -> Self {
213        Self::NativeToken(Box::new(token))
214    }
215}
216
217pub type ImperativeLexCheck =
218    Arc<dyn for<'source> Fn(&mut Lexer<'source>) -> LexCheckResult + Send + Sync>;
219
220#[derive(Clone)]
221pub struct LexCheck {
222    callback: Option<Arc<LexCheckEffect>>,
223    imperative: Option<ImperativeLexCheck>,
224}
225
226type LexCheckEffect = dyn Fn(&str) -> LexCheckResult + Send + Sync;
227
228impl LexCheck {
229    pub(crate) fn new(callback: impl Fn(&str) -> LexCheckResult + Send + Sync + 'static) -> Self {
230        Self {
231            callback: Some(Arc::new(callback)),
232            imperative: None,
233        }
234    }
235
236    pub(crate) fn new_imperative(
237        callback: impl for<'source> Fn(&mut Lexer<'source>) -> LexCheckResult + Send + Sync + 'static,
238    ) -> Self {
239        Self {
240            callback: None,
241            imperative: Some(Arc::new(callback)),
242        }
243    }
244
245    pub(crate) fn run(&self, source: &str) -> Option<LexCheckResult> {
246        self.callback.as_ref().map(|callback| callback(source))
247    }
248
249    pub(crate) fn run_imperative(&self, lexer: &mut Lexer<'_>) -> Option<LexCheckResult> {
250        self.imperative.as_ref().map(|callback| callback(lexer))
251    }
252
253    pub(crate) fn same_callback(&self, other: &Self) -> bool {
254        match (
255            &self.callback,
256            &self.imperative,
257            &other.callback,
258            &other.imperative,
259        ) {
260            (Some(left), None, Some(right), None) => Arc::ptr_eq(left, right),
261            (None, Some(left), None, Some(right)) => Arc::ptr_eq(left, right),
262            _ => false,
263        }
264    }
265}
266
267impl fmt::Debug for LexCheck {
268    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
269        formatter.write_str(if self.imperative.is_some() {
270            "LexCheck(<live lexer function>)"
271        } else {
272            "LexCheck(<effect function>)"
273        })
274    }
275}
276
277#[derive(Debug, Clone)]
278pub struct FixedToken {
279    pub name: String,
280    pub tin: Tin,
281    pub source: String,
282}
283
284#[derive(Debug, Clone)]
285pub struct FixedOptions {
286    pub lex: bool,
287    pub tokens: IndexMap<String, FixedToken>,
288    pub check: Option<LexCheck>,
289}
290
291impl Default for FixedOptions {
292    fn default() -> Self {
293        let mut tokens = IndexMap::new();
294        for (name, tin, source) in [
295            ("#OB", crate::token::TIN_OB, "{"),
296            ("#CB", crate::token::TIN_CB, "}"),
297            ("#OS", crate::token::TIN_OS, "["),
298            ("#CS", crate::token::TIN_CS, "]"),
299            ("#CL", crate::token::TIN_CL, ":"),
300            ("#CA", crate::token::TIN_CA, ","),
301        ] {
302            tokens.insert(
303                name.to_string(),
304                FixedToken {
305                    name: name.to_string(),
306                    tin,
307                    source: source.to_string(),
308                },
309            );
310        }
311        Self {
312            lex: true,
313            tokens,
314            check: None,
315        }
316    }
317}
318
319pub type ValueTextModifier = Arc<dyn Fn(crate::Value) -> crate::Value + Send + Sync>;
320pub type ImperativeTextModifier = Arc<
321    dyn for<'source> Fn(
322            crate::Value,
323            &mut Lexer<'source>,
324            &mut Rule,
325            &mut Context,
326            &Options,
327        ) -> crate::Value
328        + Send
329        + Sync,
330>;
331
332/// One unquoted-text modifier in declaration order. The value-only form is
333/// convenient for pure serialized hooks; the imperative form exposes the
334/// same live lexer/config state as the canonical `ValModifier` and also hands
335/// Rust callers explicit rule/context references instead of hiding them on
336/// the lexer object.
337#[derive(Clone)]
338pub enum TextModifier {
339    Value(ValueTextModifier),
340    Imperative(ImperativeTextModifier),
341}
342
343impl TextModifier {
344    pub(crate) fn new(
345        modifier: impl Fn(crate::Value) -> crate::Value + Send + Sync + 'static,
346    ) -> Self {
347        Self::Value(Arc::new(modifier))
348    }
349
350    pub(crate) fn new_imperative(
351        modifier: impl for<'source> Fn(
352                crate::Value,
353                &mut Lexer<'source>,
354                &mut Rule,
355                &mut Context,
356                &Options,
357            ) -> crate::Value
358            + Send
359            + Sync
360            + 'static,
361    ) -> Self {
362        Self::Imperative(Arc::new(modifier))
363    }
364
365    pub(crate) fn run(
366        &self,
367        value: crate::Value,
368        lexer: &mut Lexer<'_>,
369        rule: &mut Rule,
370        context: &mut Context,
371        options: &Options,
372    ) -> crate::Value {
373        match self {
374            Self::Value(modifier) => modifier(value),
375            Self::Imperative(modifier) => modifier(value, lexer, rule, context, options),
376        }
377    }
378
379    pub(crate) fn same_callback(&self, other: &Self) -> bool {
380        match (self, other) {
381            (Self::Value(left), Self::Value(right)) => Arc::ptr_eq(left, right),
382            (Self::Imperative(left), Self::Imperative(right)) => Arc::ptr_eq(left, right),
383            _ => false,
384        }
385    }
386}
387
388impl fmt::Debug for TextModifier {
389    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
390        formatter.write_str(match self {
391            Self::Value(_) => "TextModifier::Value(<function>)",
392            Self::Imperative(_) => "TextModifier::Imperative(<function>)",
393        })
394    }
395}
396
397#[derive(Clone)]
398pub struct TextOptions {
399    pub lex: bool,
400    pub modify: Vec<TextModifier>,
401    pub check: Option<LexCheck>,
402}
403
404impl fmt::Debug for TextOptions {
405    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
406        formatter
407            .debug_struct("TextOptions")
408            .field("lex", &self.lex)
409            .field("modify", &self.modify.len())
410            .field("check", &self.check)
411            .finish()
412    }
413}
414
415impl Default for TextOptions {
416    fn default() -> Self {
417        TextOptions {
418            lex: true,
419            modify: Vec::new(),
420            check: None,
421        }
422    }
423}
424
425#[derive(Debug, Clone)]
426pub struct SpaceOptions {
427    pub lex: bool,
428    pub chars: String,
429    pub check: Option<LexCheck>,
430}
431
432impl Default for SpaceOptions {
433    fn default() -> Self {
434        Self {
435            lex: true,
436            chars: " \t".into(),
437            check: None,
438        }
439    }
440}
441
442#[derive(Debug, Clone)]
443pub struct NumberOptions {
444    pub lex: bool,
445    pub hex: bool,
446    pub oct: bool,
447    pub bin: bool,
448    pub sep: Option<String>,
449    pub exclude: Option<String>, // regex string e.g. "^00+"
450    pub check: Option<LexCheck>,
451}
452
453impl Default for NumberOptions {
454    fn default() -> Self {
455        NumberOptions {
456            lex: true,
457            hex: true,
458            oct: true,
459            bin: true,
460            sep: Some("_".to_string()),
461            exclude: None,
462            check: None,
463        }
464    }
465}
466
467#[derive(Debug, Clone)]
468pub struct StringOptions {
469    pub lex: bool,
470    pub chars: String,
471    pub multi_chars: String,
472    pub escape_char: char,
473    pub escape: HashMap<char, String>,
474    pub replace: HashMap<char, String>,
475    pub allow_unknown: bool,
476    pub escape_strict: bool,
477    pub allow_control: bool,
478    pub abandon: bool,
479    pub check: Option<LexCheck>,
480}
481
482impl Default for StringOptions {
483    fn default() -> Self {
484        let escape = [
485            ('b', "\u{0008}"),
486            ('f', "\u{000c}"),
487            ('n', "\n"),
488            ('r', "\r"),
489            ('t', "\t"),
490            ('v', "\u{000b}"),
491            ('"', "\""),
492            ('\'', "'"),
493            ('`', "`"),
494            ('\\', "\\"),
495            ('/', "/"),
496        ]
497        .into_iter()
498        .map(|(key, value)| (key, value.into()))
499        .collect();
500        StringOptions {
501            lex: true,
502            chars: "\"'`".to_string(),
503            multi_chars: "`".to_string(),
504            escape_char: '\\',
505            escape,
506            replace: HashMap::new(),
507            allow_unknown: true,
508            escape_strict: false,
509            allow_control: false,
510            abandon: false,
511            check: None,
512        }
513    }
514}
515
516#[derive(Debug, Clone)]
517pub struct LineOptions {
518    pub lex: bool,
519    pub chars: String,
520    pub row_chars: String,
521    pub single: bool,
522    /// Extra line terminators retained for compatibility with the first
523    /// Rust slice. Serialized grammars should prefer `line.chars`.
524    pub fixed: Vec<char>,
525    pub check: Option<LexCheck>,
526}
527
528impl Default for LineOptions {
529    fn default() -> Self {
530        Self {
531            lex: true,
532            chars: "\r\n".into(),
533            row_chars: "\n".into(),
534            single: false,
535            fixed: Vec::new(),
536            check: None,
537        }
538    }
539}
540
541#[derive(Debug, Clone)]
542pub struct CommentDef {
543    pub line: bool,
544    pub start: String,
545    pub end: String,
546    pub lex: bool,
547    pub suffixes: Vec<String>,
548    pub suffix_matcher: Option<CommentSuffixMatcher>,
549    pub eat_line: bool,
550}
551
552#[derive(Clone)]
553pub struct CommentSuffixMatcher {
554    callback: Option<Arc<CommentSuffixCallback>>,
555    imperative: Option<ImperativeCommentSuffixMatcher>,
556}
557
558type CommentSuffixCallback = dyn Fn(&str) -> Option<String> + Send + Sync;
559pub type ImperativeCommentSuffixMatcher =
560    Arc<dyn for<'source> Fn(&mut Lexer<'source>) -> Option<Token> + Send + Sync>;
561
562impl CommentSuffixMatcher {
563    pub(crate) fn new(callback: impl Fn(&str) -> Option<String> + Send + Sync + 'static) -> Self {
564        Self {
565            callback: Some(Arc::new(callback)),
566            imperative: None,
567        }
568    }
569
570    pub(crate) fn new_imperative(
571        callback: impl for<'source> Fn(&mut Lexer<'source>) -> Option<Token> + Send + Sync + 'static,
572    ) -> Self {
573        Self {
574            callback: None,
575            imperative: Some(Arc::new(callback)),
576        }
577    }
578
579    pub(crate) fn run(&self, source: &str) -> Option<String> {
580        self.callback.as_ref().and_then(|callback| callback(source))
581    }
582
583    pub(crate) fn run_imperative(&self, lexer: &mut Lexer<'_>) -> Option<Token> {
584        self.imperative
585            .as_ref()
586            .and_then(|callback| callback(lexer))
587    }
588
589    pub(crate) fn same_callback(&self, other: &Self) -> bool {
590        match (
591            &self.callback,
592            &self.imperative,
593            &other.callback,
594            &other.imperative,
595        ) {
596            (Some(left), None, Some(right), None) => Arc::ptr_eq(left, right),
597            (None, Some(left), None, Some(right)) => Arc::ptr_eq(left, right),
598            _ => false,
599        }
600    }
601}
602
603impl fmt::Debug for CommentSuffixMatcher {
604    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
605        formatter.write_str(if self.imperative.is_some() {
606            "CommentSuffixMatcher(<live lexer function>)"
607        } else {
608            "CommentSuffixMatcher(<effect function>)"
609        })
610    }
611}
612
613#[derive(Debug, Clone)]
614pub struct CommentOptions {
615    pub lex: bool,
616    pub definitions: IndexMap<String, CommentDef>,
617    pub check: Option<LexCheck>,
618}
619
620#[derive(Clone)]
621pub struct ValueDef {
622    pub val: Option<crate::Value>,
623    pub matcher: Option<Regex>,
624    pub transform: Option<ValueTransform>,
625    pub consume: bool,
626}
627
628pub type ValueTransform = Arc<dyn Fn(&[String]) -> crate::Value + Send + Sync>;
629
630impl fmt::Debug for ValueDef {
631    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
632        formatter
633            .debug_struct("ValueDef")
634            .field("val", &self.val)
635            .field("matcher", &self.matcher)
636            .field("transform", &self.transform.as_ref().map(|_| "<function>"))
637            .field("consume", &self.consume)
638            .finish()
639    }
640}
641
642#[derive(Debug, Clone)]
643pub struct ValueOptions {
644    pub lex: bool,
645    pub definitions: IndexMap<String, ValueDef>,
646}
647
648impl Default for ValueOptions {
649    fn default() -> Self {
650        let mut definitions = IndexMap::new();
651        for (source, value) in [
652            ("true", crate::Value::Bool(true)),
653            ("false", crate::Value::Bool(false)),
654            ("null", crate::Value::Null),
655        ] {
656            definitions.insert(
657                source.into(),
658                ValueDef {
659                    val: Some(value),
660                    matcher: None,
661                    transform: None,
662                    consume: false,
663                },
664            );
665        }
666        Self {
667            lex: true,
668            definitions,
669        }
670    }
671}
672
673impl Default for CommentOptions {
674    fn default() -> Self {
675        let mut definitions = IndexMap::new();
676        for (name, line, start, end) in [
677            ("hash", true, "#", ""),
678            ("slash", true, "//", ""),
679            ("multi", false, "/*", "*/"),
680        ] {
681            definitions.insert(
682                name.into(),
683                CommentDef {
684                    line,
685                    start: start.into(),
686                    end: end.into(),
687                    lex: true,
688                    suffixes: Vec::new(),
689                    suffix_matcher: None,
690                    eat_line: false,
691                },
692            );
693        }
694        CommentOptions {
695            lex: true,
696            definitions,
697            check: None,
698        }
699    }
700}
701
702#[derive(Debug, Clone)]
703pub struct SafeOptions {
704    pub key: bool,
705}
706
707impl Default for SafeOptions {
708    fn default() -> Self {
709        Self { key: true }
710    }
711}
712
713pub type MapMerge =
714    Arc<dyn Fn(crate::Value, crate::Value, &mut Rule, &mut Context) -> crate::Value + Send + Sync>;
715
716#[derive(Clone)]
717pub struct MapOptions {
718    pub extend: bool,
719    pub merge: Option<MapMerge>,
720    pub child: bool,
721    /// Rust's `IndexMap` always retains insertion order. This flag is kept so
722    /// serialized option overlays and plugin code observe the canonical
723    /// option surface even though enabling it requires no representation
724    /// change here.
725    pub ordered: bool,
726}
727
728impl fmt::Debug for MapOptions {
729    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
730        formatter
731            .debug_struct("MapOptions")
732            .field("extend", &self.extend)
733            .field("merge", &self.merge.as_ref().map(|_| "<function>"))
734            .field("child", &self.child)
735            .field("ordered", &self.ordered)
736            .finish()
737    }
738}
739
740impl Default for MapOptions {
741    fn default() -> Self {
742        MapOptions {
743            extend: true,
744            merge: None,
745            child: false,
746            ordered: false,
747        }
748    }
749}
750
751#[derive(Debug, Clone)]
752pub struct ListOptions {
753    pub property: bool,
754    pub pair: bool,
755    pub child: bool,
756}
757
758impl Default for ListOptions {
759    fn default() -> Self {
760        Self {
761            property: true,
762            pair: false,
763            child: false,
764        }
765    }
766}
767
768/// Controls typed metadata carriers on native parse results.
769///
770/// The wrappers serialize as their underlying JSON values, matching the
771/// TypeScript implementation's non-enumerable metadata marker.
772#[derive(Debug, Clone)]
773pub struct InfoOptions {
774    pub map: bool,
775    pub list: bool,
776    pub text: bool,
777    pub marker: String,
778}
779
780impl Default for InfoOptions {
781    fn default() -> Self {
782        Self {
783            map: false,
784            list: false,
785            text: false,
786            marker: "__info__".into(),
787        }
788    }
789}
790
791#[derive(Debug, Clone)]
792pub struct LexOptions {
793    pub empty: bool,
794    pub empty_result: crate::Value,
795    pub relex: bool,
796    pub matchers: IndexMap<String, LexMatcher>,
797    /// The grammar's matchers and actions read the source only through
798    /// the lexer's window ([`crate::Lexer::window`], [`crate::Lexer::fill`]),
799    /// so a parse from a reader may hold a window of it rather than all of
800    /// it ([`crate::Tabnas::parse_reader`]). Without it, a reader is read
801    /// whole before the parse begins. Off by default (admin ADR-30).
802    pub window: bool,
803}
804
805#[derive(Debug, Clone)]
806pub struct RewindOptions {
807    /// Maximum retained consumed-token history. `None` is unbounded,
808    /// the serialized `false`; `Some(0)` retains nothing; the default is
809    /// `Some(64)`, which a serialized `null` also means. `Some(0)` used
810    /// to be read as unbounded, the opposite of the canonical runtime
811    /// (#142), and `null` as unbounded rather than the default (#144).
812    pub history: Option<usize>,
813}
814
815impl Default for RewindOptions {
816    fn default() -> Self {
817        Self { history: Some(64) }
818    }
819}
820
821impl Default for LexOptions {
822    fn default() -> Self {
823        LexOptions {
824            empty: true,
825            empty_result: crate::Value::Undefined,
826            relex: false,
827            matchers: IndexMap::new(),
828            window: false,
829        }
830    }
831}
832
833#[derive(Debug, Clone, Default)]
834pub struct ResultOptions {
835    pub fail: Vec<crate::Value>,
836}
837
838/// The largest `options.rule.history` a rule's links honour. Every
839/// replace and push copies up to `history` snapshots, so the bound is
840/// the cost of each link: capped, a parse stays linear in its length
841/// whatever the bound, where a bound as long as the sequence copied
842/// `1 + 2 + … + N` snapshots. Sixteen is five times the deepest walk the
843/// fleet reads (three hops), and the largest bound measured faster than
844/// no bound at all: over 300,000 array items a release build took 2.2 s
845/// at 16 and 2.8 s unbounded, 3.3 s at 32 and 6.2 s at 64
846/// (`doc/rule-history-bound.md`). A grammar that asks for more is
847/// refused; a larger value set on [`RuleOptions::history`] directly is
848/// read as this, and `Some(0)` as 1.
849pub const MAX_RULE_HISTORY: usize = 16;
850
851/// The bound [`RuleOptions::history`] takes when nothing sets it: three
852/// predecessor snapshots, every walk the fleet reads.
853pub const DEFAULT_RULE_HISTORY: usize = 3;
854
855/// The bound a rule's links honour for `history`: 1 to
856/// [`MAX_RULE_HISTORY`], or `None` for every link. Read wherever the
857/// option decides something, the links themselves and a merge, so two
858/// values that parse alike are alike everywhere.
859pub(crate) fn effective_rule_history(history: Option<usize>) -> Option<usize> {
860    history.map(|history| history.clamp(1, MAX_RULE_HISTORY))
861}
862
863#[derive(Debug, Clone)]
864pub struct RuleOptions {
865    /// The grammar's end-of-source switch (default true). jsonic's
866    /// `@finish` reads it: an unterminated structure is closed at the end
867    /// of the source when it is true and refused with `end_of_source`
868    /// when it is false. The engine stores, merges and serializes it for
869    /// the grammar and does not read it: trailing content is an error
870    /// either way.
871    pub finish: bool,
872    pub maxmul: usize,
873    pub include: String,
874    pub exclude: String,
875    pub start: String,
876    /// How many predecessor snapshots a rule can reach through `prev`:
877    /// the `prev_rule` chain a replace or a push links is cut past this
878    /// many links, so a sequence's iterations are not all kept until its
879    /// container closes. The default is [`DEFAULT_RULE_HISTORY`]; `None`
880    /// keeps every link, which costs memory in proportion to the length
881    /// of the document (`doc/rule-history-bound.md`). Read as 1 to
882    /// [`MAX_RULE_HISTORY`]: `Some(0)` as 1, a larger value as the cap.
883    pub history: Option<usize>,
884}
885
886#[derive(Debug, Clone)]
887pub struct MatchToken {
888    pub name: String,
889    pub tin: Tin,
890    pub matcher: MatchTokenMatcher,
891    pub eager: bool,
892}
893
894#[derive(Clone)]
895pub struct MatchValue {
896    pub name: String,
897    pub matcher: MatchTokenMatcher,
898    pub val: Option<crate::Value>,
899    pub transform: Option<ValueTransform>,
900}
901
902impl fmt::Debug for MatchValue {
903    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
904        formatter
905            .debug_struct("MatchValue")
906            .field("name", &self.name)
907            .field("matcher", &self.matcher)
908            .field("val", &self.val)
909            .field("transform", &self.transform.as_ref().map(|_| "<function>"))
910            .finish()
911    }
912}
913
914#[derive(Debug, Clone, PartialEq)]
915pub struct MatchTokenResult {
916    /// Non-empty source prefix consumed by the match.
917    pub source: String,
918    /// Token value exposed to rule actions.
919    pub value: crate::Value,
920}
921
922impl MatchTokenResult {
923    pub fn new(source: impl Into<String>, value: crate::Value) -> Self {
924        Self {
925            source: source.into(),
926            value,
927        }
928    }
929}
930
931pub type MatchTokenCallback = Arc<dyn Fn(&str) -> Option<MatchTokenResult> + Send + Sync>;
932
933/// Effect-based custom lexer matcher used by serialized `options.lex.match`
934/// entries. The returned token must consume a non-empty prefix of the
935/// remaining source.
936pub type LexMatcherCallback = Arc<dyn Fn(&str) -> Option<LexCheckToken> + Send + Sync>;
937
938/// Full native custom matcher. Unlike the serialized effect callback, this
939/// form can inspect and mutate the live rule/context and advance the lexer.
940pub type ImperativeLexMatcher = Arc<
941    dyn for<'source> Fn(&mut Lexer<'source>, &mut Rule, &mut Context) -> Option<Token>
942        + Send
943        + Sync,
944>;
945
946/// Setup-time matcher constructor. Rust has one resolved typed option tree,
947/// so it serves the roles of both canonical `cfg` and raw `opts` arguments.
948pub type LexMatcherFactory = Arc<dyn Fn(&Options) -> Option<ImperativeLexMatcher> + Send + Sync>;
949
950#[derive(Clone)]
951pub struct LexMatcher {
952    pub name: String,
953    pub order: f64,
954    pub matcher: Option<LexMatcherCallback>,
955    pub imperative: Option<ImperativeLexMatcher>,
956    pub factory: Option<LexMatcherFactory>,
957}
958
959impl fmt::Debug for LexMatcher {
960    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
961        formatter
962            .debug_struct("LexMatcher")
963            .field("name", &self.name)
964            .field("order", &self.order)
965            .field("matcher", &self.matcher.as_ref().map(|_| "<function>"))
966            .field(
967                "imperative",
968                &self.imperative.as_ref().map(|_| "<function>"),
969            )
970            .field("factory", &self.factory.as_ref().map(|_| "<function>"))
971            .finish()
972    }
973}
974
975#[derive(Clone)]
976pub enum MatchTokenMatcher {
977    Regex(Regex),
978    Callback(MatchTokenCallback),
979}
980
981impl fmt::Debug for MatchTokenMatcher {
982    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
983        match self {
984            Self::Regex(regex) => formatter.debug_tuple("Regex").field(regex).finish(),
985            Self::Callback(_) => formatter.write_str("Callback(<function>)"),
986        }
987    }
988}
989
990impl Default for RuleOptions {
991    fn default() -> Self {
992        RuleOptions {
993            finish: true,
994            maxmul: 3,
995            include: String::new(),
996            exclude: String::new(),
997            start: "val".to_string(),
998            history: Some(DEFAULT_RULE_HISTORY),
999        }
1000    }
1001}
1002
1003pub type BudgetCheck = Arc<dyn Fn(&Context) -> bool + Send + Sync>;
1004
1005#[derive(Clone, Default)]
1006pub struct BudgetOptions {
1007    pub check_every_n: usize,
1008    pub on_check: Option<BudgetCheck>,
1009}
1010
1011impl fmt::Debug for BudgetOptions {
1012    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1013        formatter
1014            .debug_struct("BudgetOptions")
1015            .field("check_every_n", &self.check_every_n)
1016            .field("on_check", &self.on_check.as_ref().map(|_| "<callback>"))
1017            .finish()
1018    }
1019}
1020
1021#[derive(Debug, Clone)]
1022pub struct RecoverOptions {
1023    pub enabled: bool,
1024    pub sync_groups: Vec<String>,
1025    pub sync_tokens: Vec<String>,
1026    pub pop_until_valid: bool,
1027    pub max_skip: usize,
1028    pub max_recoveries: usize,
1029    pub suppress: usize,
1030}
1031
1032impl Default for RecoverOptions {
1033    fn default() -> Self {
1034        Self {
1035            enabled: false,
1036            sync_groups: vec!["close".into(), "comma".into(), "end".into()],
1037            sync_tokens: Vec::new(),
1038            pop_until_valid: true,
1039            max_skip: 64,
1040            max_recoveries: 32,
1041            suppress: 4,
1042        }
1043    }
1044}
1045
1046pub type ContextParsePrepare = Arc<dyn Fn(&mut Context) + Send + Sync>;
1047pub type ParsePrepareWithInstance =
1048    Arc<dyn Fn(&crate::Tabnas, &mut Context, &crate::Value) + Send + Sync>;
1049
1050/// Pre-parse hook. The context-only form preserves the original Rust API;
1051/// `WithInstance` exposes the owning parser and caller metadata carried by
1052/// the canonical callback contract.
1053#[derive(Clone)]
1054pub enum ParsePrepare {
1055    Context(ContextParsePrepare),
1056    WithInstance(ParsePrepareWithInstance),
1057}
1058
1059impl ParsePrepare {
1060    pub(crate) fn same_callback(&self, other: &Self) -> bool {
1061        match (self, other) {
1062            (Self::Context(left), Self::Context(right)) => Arc::ptr_eq(left, right),
1063            (Self::WithInstance(left), Self::WithInstance(right)) => Arc::ptr_eq(left, right),
1064            _ => false,
1065        }
1066    }
1067
1068    pub(crate) fn run(
1069        &self,
1070        owner: Option<&crate::Tabnas>,
1071        context: &mut Context,
1072        meta: &crate::Value,
1073    ) -> Result<(), &'static str> {
1074        match self {
1075            Self::Context(callback) => {
1076                callback(context);
1077                Ok(())
1078            }
1079            Self::WithInstance(callback) => {
1080                let owner = owner.ok_or(
1081                    "parse.prepare requires an owning Tabnas instance; call Tabnas::parse",
1082                )?;
1083                callback(owner, context, meta);
1084                Ok(())
1085            }
1086        }
1087    }
1088}
1089
1090impl fmt::Debug for ParsePrepare {
1091    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1092        match self {
1093            Self::Context(_) => formatter.write_str("ParsePrepare::Context(<function>)"),
1094            Self::WithInstance(_) => formatter.write_str("ParsePrepare::WithInstance(<function>)"),
1095        }
1096    }
1097}
1098
1099pub type ParserStart =
1100    Arc<dyn Fn(&str) -> Result<crate::Value, Box<crate::TabnasError>> + Send + Sync>;
1101pub type ParserStartWithInstance = Arc<
1102    dyn Fn(&str, &crate::Tabnas, &crate::Value) -> Result<crate::Value, Box<crate::TabnasError>>
1103        + Send
1104        + Sync,
1105>;
1106pub type ParserStartWithContext = Arc<
1107    dyn Fn(
1108            &str,
1109            &crate::Tabnas,
1110            &crate::Value,
1111            Option<&crate::ContextSeed>,
1112        ) -> Result<crate::Value, Box<crate::TabnasError>>
1113        + Send
1114        + Sync,
1115>;
1116
1117pub type DebugOutput = Arc<dyn Fn(&str) + Send + Sync>;
1118pub type DebugSourceFormatter = Arc<dyn Fn(&crate::Value) -> String + Send + Sync>;
1119
1120#[derive(Clone, Default)]
1121pub struct DebugPrintOptions {
1122    pub config: bool,
1123    pub source: Option<DebugSourceFormatter>,
1124}
1125
1126impl fmt::Debug for DebugPrintOptions {
1127    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1128        formatter
1129            .debug_struct("DebugPrintOptions")
1130            .field("config", &self.config)
1131            .field("source", &self.source.as_ref().map(|_| "<callback>"))
1132            .finish()
1133    }
1134}
1135
1136#[derive(Clone)]
1137pub struct DebugOptions {
1138    /// Maximum Unicode-scalar length of values rendered by the default
1139    /// source formatter.
1140    pub maxlen: usize,
1141    pub print: DebugPrintOptions,
1142    /// Rust equivalent of TypeScript's console provider. `None` writes trace
1143    /// and configuration output to stderr.
1144    pub output: Option<DebugOutput>,
1145}
1146
1147impl Default for DebugOptions {
1148    fn default() -> Self {
1149        Self {
1150            maxlen: 99,
1151            print: DebugPrintOptions::default(),
1152            output: None,
1153        }
1154    }
1155}
1156
1157impl fmt::Debug for DebugOptions {
1158    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1159        formatter
1160            .debug_struct("DebugOptions")
1161            .field("maxlen", &self.maxlen)
1162            .field("print", &self.print)
1163            .field("output", &self.output.as_ref().map(|_| "<callback>"))
1164            .finish()
1165    }
1166}
1167
1168impl DebugOptions {
1169    pub fn format_source(&self, value: &crate::Value) -> String {
1170        if let Some(formatter) = &self.print.source {
1171            return formatter(value);
1172        }
1173        let rendered = match value {
1174            crate::Value::Undefined | crate::Value::Null => String::new(),
1175            value => serde_json::to_string(&value.to_json()).unwrap_or_else(|_| value.to_string()),
1176        };
1177        let mut chars = rendered.chars();
1178        let prefix = chars.by_ref().take(self.maxlen).collect::<String>();
1179        if chars.next().is_some() {
1180            format!("{prefix}...")
1181        } else {
1182            prefix
1183        }
1184    }
1185
1186    pub fn write(&self, message: &str) {
1187        if let Some(output) = &self.output {
1188            // Debugging must not make construction or parsing less reliable.
1189            // Parse-time trace sinks are also protected by the subscriber
1190            // boundary, but configuration printing runs during setup.
1191            let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| output(message)));
1192        } else {
1193            eprintln!("{message}");
1194        }
1195    }
1196}
1197
1198#[derive(Clone, Default)]
1199pub struct ParserOptions {
1200    pub start: Option<ParserStart>,
1201    pub start_with_instance: Option<ParserStartWithInstance>,
1202    pub start_with_context: Option<ParserStartWithContext>,
1203}
1204
1205impl fmt::Debug for ParserOptions {
1206    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1207        formatter
1208            .debug_struct("ParserOptions")
1209            .field("start", &self.start.as_ref().map(|_| "<callback>"))
1210            .field(
1211                "start_with_instance",
1212                &self.start_with_instance.as_ref().map(|_| "<callback>"),
1213            )
1214            .field(
1215                "start_with_context",
1216                &self.start_with_context.as_ref().map(|_| "<callback>"),
1217            )
1218            .finish()
1219    }
1220}
1221
1222#[derive(Clone, Default)]
1223pub struct ParseOptions {
1224    pub prepare: Vec<ParsePrepare>,
1225    pub named_prepare: IndexMap<String, ParsePrepare>,
1226    pub budget: BudgetOptions,
1227    pub recover: RecoverOptions,
1228}
1229
1230impl fmt::Debug for ParseOptions {
1231    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1232        formatter
1233            .debug_struct("ParseOptions")
1234            .field("prepare", &self.prepare.len())
1235            .field("named_prepare", &self.named_prepare.len())
1236            .field("budget", &self.budget)
1237            .field("recover", &self.recover)
1238            .finish()
1239    }
1240}
1241
1242#[derive(Debug, Clone)]
1243pub struct Options {
1244    pub safe: SafeOptions,
1245    pub fixed: FixedOptions,
1246    pub space: SpaceOptions,
1247    pub text: TextOptions,
1248    pub number: NumberOptions,
1249    pub string: StringOptions,
1250    pub line: LineOptions,
1251    pub comment: CommentOptions,
1252    pub value: ValueOptions,
1253    pub ender: Vec<String>,
1254    pub map: MapOptions,
1255    pub list: ListOptions,
1256    pub info: InfoOptions,
1257    pub lex: LexOptions,
1258    pub rewind: RewindOptions,
1259    pub rule: RuleOptions,
1260    pub result: ResultOptions,
1261    pub parse: ParseOptions,
1262    pub parser: ParserOptions,
1263    pub debug: DebugOptions,
1264    /// Per-plugin accumulated option bags (`options.plugin`).
1265    pub plugin: IndexMap<String, crate::Value>,
1266    pub token_set: HashMap<String, Vec<Tin>>,
1267    /// Named token identities without an attached built-in, fixed, or match
1268    /// producer. Serialized rule slots allocate these on first reference.
1269    pub tokens: IndexMap<String, Tin>,
1270    /// Enable serialized/custom regexp token matching (`options.match.lex`).
1271    pub match_lex: bool,
1272    pub match_check: Option<LexCheck>,
1273    pub match_tokens: IndexMap<String, MatchToken>,
1274    pub match_values: IndexMap<String, MatchValue>,
1275    pub error: HashMap<String, String>,
1276    pub hint: HashMap<String, String>,
1277    pub errmsg: ErrMsgOptions,
1278    pub color: ColorOptions,
1279    pub config_modify: IndexMap<String, ConfigModifier>,
1280    pub tag: String,
1281}
1282
1283impl Default for Options {
1284    fn default() -> Self {
1285        let mut token_set = HashMap::new();
1286        token_set.insert("IGNORE".to_string(), vec![TIN_SP, TIN_LN, TIN_CM]);
1287        token_set.insert("VAL".to_string(), vec![TIN_TX, TIN_NR, TIN_ST, TIN_VL]);
1288        token_set.insert("KEY".to_string(), vec![TIN_TX, TIN_NR, TIN_ST, TIN_VL]);
1289
1290        Options {
1291            safe: SafeOptions::default(),
1292            fixed: FixedOptions::default(),
1293            space: SpaceOptions::default(),
1294            text: TextOptions::default(),
1295            number: NumberOptions::default(),
1296            string: StringOptions::default(),
1297            line: LineOptions::default(),
1298            comment: CommentOptions::default(),
1299            value: ValueOptions::default(),
1300            ender: Vec::new(),
1301            map: MapOptions::default(),
1302            list: ListOptions::default(),
1303            info: InfoOptions::default(),
1304            lex: LexOptions::default(),
1305            rewind: RewindOptions::default(),
1306            rule: RuleOptions::default(),
1307            result: ResultOptions::default(),
1308            parse: ParseOptions::default(),
1309            parser: ParserOptions::default(),
1310            debug: DebugOptions::default(),
1311            plugin: IndexMap::new(),
1312            token_set,
1313            tokens: IndexMap::new(),
1314            match_lex: true,
1315            match_check: None,
1316            match_tokens: IndexMap::new(),
1317            match_values: IndexMap::new(),
1318            error: crate::error::default_error_messages(),
1319            hint: crate::error::default_error_hints(),
1320            errmsg: ErrMsgOptions::default(),
1321            color: ColorOptions::default(),
1322            config_modify: IndexMap::new(),
1323            tag: "-".to_string(),
1324        }
1325    }
1326}
1327
1328impl Options {
1329    /// Refuse block-comment definitions that cannot close.
1330    ///
1331    /// The three engines historically disagreed on an empty terminator;
1332    /// making it a configuration error keeps that malformed state out of
1333    /// every lexer and names the exact definition that must be repaired.
1334    pub(crate) fn validate_comment_definitions(&self) -> Result<(), String> {
1335        for (name, definition) in &self.comment.definitions {
1336            if !definition.line && definition.end.is_empty() {
1337                return Err(format!(
1338                    "options.comment.def.{name}.end: block comments require a non-empty end marker"
1339                ));
1340            }
1341        }
1342        Ok(())
1343    }
1344
1345    /// Rebuild the resolved configuration callbacks from this option tree.
1346    /// Config modifiers run before matcher factories, matching canonical
1347    /// `configure`: factories must observe the modifier's final values.
1348    pub fn refresh_configuration(&mut self) -> Result<(), String> {
1349        let raw_options = self.clone();
1350        let modifiers: Vec<_> = self
1351            .config_modify
1352            .iter()
1353            .map(|(name, modifier)| (name.clone(), modifier.clone()))
1354            .collect();
1355        for (name, modifier) in modifiers {
1356            std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1357                modifier.run(self, &raw_options)
1358            }))
1359            .map_err(|_| format!("config modifier {name} panicked"))?;
1360        }
1361        self.validate_comment_definitions()?;
1362        self.refresh_lex_matchers()
1363    }
1364
1365    /// Rebuild setup-time lexer matchers from the fully resolved option tree.
1366    /// Factories run once per configuration/derivation, not once per parse.
1367    pub fn refresh_lex_matchers(&mut self) -> Result<(), String> {
1368        let factories: Vec<_> = self
1369            .lex
1370            .matchers
1371            .iter()
1372            .filter_map(|(name, matcher)| {
1373                matcher
1374                    .factory
1375                    .as_ref()
1376                    .map(|factory| (name.clone(), factory.clone()))
1377            })
1378            .collect();
1379        for (name, factory) in factories {
1380            let matcher = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| factory(self)))
1381                .map_err(|_| format!("lexer matcher factory {name} panicked"))?;
1382            if let Some(entry) = self.lex.matchers.get_mut(&name) {
1383                entry.imperative = matcher;
1384            }
1385        }
1386        Ok(())
1387    }
1388
1389    /// Baseline for `Tabnas::empty`: no standard token producers or token
1390    /// sets. Structural and diagnostic fields retain typed values so plugins
1391    /// can opt individual facilities back in without handling missing data.
1392    pub fn empty() -> Self {
1393        let mut options = Self::default();
1394        options.fixed.lex = false;
1395        options.fixed.tokens.clear();
1396        options.space.lex = false;
1397        options.text.lex = false;
1398        options.number.lex = false;
1399        options.string.lex = false;
1400        options.line.lex = false;
1401        options.comment.lex = false;
1402        options.comment.definitions.clear();
1403        options.value.lex = false;
1404        options.value.definitions.clear();
1405        options.match_lex = false;
1406        options.match_tokens.clear();
1407        options.match_values.clear();
1408        options.lex.matchers.clear();
1409        options.token_set.clear();
1410        options.tokens.clear();
1411        options
1412    }
1413
1414    /// Put the serialized matchers in the order the lexer evaluates
1415    /// them. TypeScript evaluates token matchers in tin order; keep
1416    /// that deterministic even when callers assembled `Options` by
1417    /// mutating the public maps directly. Done once per prepared
1418    /// configuration rather than once per lexer.
1419    pub(crate) fn sort_for_lexing(&mut self) {
1420        self.match_tokens
1421            .sort_by(|_, left, _, right| left.tin.cmp(&right.tin));
1422        self.match_values.sort_keys();
1423        self.lex.matchers.sort_by(|name_a, left, name_b, right| {
1424            left.order
1425                .total_cmp(&right.order)
1426                .then_with(|| name_a.cmp(name_b))
1427        });
1428    }
1429
1430    pub fn is_ignored(&self, tin: Tin) -> bool {
1431        self.token_set
1432            .get("IGNORE")
1433            .is_some_and(|ignored| ignored.contains(&tin))
1434    }
1435
1436    /// The ignore set, resolved once. `is_ignored` is asked for every
1437    /// lookahead token, and hashing the literal `"IGNORE"` each time
1438    /// cost more than the answer; the lexer and the parser each take a
1439    /// copy when they are built.
1440    pub(crate) fn ignore_tins(&self) -> Vec<Tin> {
1441        self.token_set.get("IGNORE").cloned().unwrap_or_default()
1442    }
1443
1444    /// The lexer's character classes, tabulated. Built alongside
1445    /// `ignore_tins` and carrying the same caveat: these answer from a
1446    /// snapshot of the options, so whatever holds them has to be rebuilt
1447    /// when the options change. The lexer is, once per parse.
1448    ///
1449    /// `line_ends` merges `line.chars` with `line.fixed`, which is how most
1450    /// sites ask. `line` is `line.chars` ALONE: string lexing decides what
1451    /// is unprintable without consulting `line.fixed`, and folding the two
1452    /// together there would change which characters a string rejects.
1453    pub(crate) fn char_sets(&self) -> crate::text::CharSets {
1454        crate::text::CharSets {
1455            space: crate::text::CharSet::new(&self.space.chars),
1456            line_ends: crate::text::CharSet::with_extra(&self.line.chars, &self.line.fixed),
1457            line: crate::text::CharSet::new(&self.line.chars),
1458            row: crate::text::CharSet::new(&self.line.row_chars),
1459            string: crate::text::CharSet::new(&self.string.chars),
1460        }
1461    }
1462
1463    pub fn token(&self, name: &str) -> Option<Tin> {
1464        crate::token::name_to_tin(name).or_else(|| {
1465            let name = if name.starts_with('#') {
1466                name.to_string()
1467            } else {
1468                format!("#{name}")
1469            };
1470            self.tokens
1471                .get(&name)
1472                .copied()
1473                .or_else(|| self.match_tokens.get(&name).map(|matcher| matcher.tin))
1474                .or_else(|| self.fixed.tokens.get(&name).map(|token| token.tin))
1475        })
1476    }
1477
1478    pub fn register_token(&mut self, name: impl Into<String>) -> Tin {
1479        let name = name.into();
1480        let name = if name.starts_with('#') {
1481            name
1482        } else {
1483            format!("#{name}")
1484        };
1485        if let Some(tin) = self.token(&name) {
1486            return tin;
1487        }
1488        let tin = self.next_tin();
1489        self.tokens.insert(name, tin);
1490        tin
1491    }
1492
1493    pub fn next_tin(&self) -> Tin {
1494        self.match_tokens
1495            .values()
1496            .map(|matcher| matcher.tin)
1497            .chain(self.fixed.tokens.values().map(|token| token.tin))
1498            .chain(self.tokens.values().copied())
1499            .max()
1500            .unwrap_or(TIN_MAX - 1)
1501            + 1
1502    }
1503
1504    pub fn token_name(&self, tin: Tin) -> String {
1505        self.match_tokens
1506            .values()
1507            .find(|matcher| matcher.tin == tin)
1508            .map(|matcher| matcher.name.clone())
1509            .or_else(|| {
1510                self.fixed
1511                    .tokens
1512                    .values()
1513                    .find(|token| token.tin == tin)
1514                    .map(|token| token.name.clone())
1515            })
1516            .or_else(|| {
1517                self.tokens
1518                    .iter()
1519                    .find(|(_, token_tin)| **token_tin == tin)
1520                    .map(|(name, _)| name.clone())
1521            })
1522            .unwrap_or_else(|| crate::token::tin_name(tin).to_string())
1523    }
1524}