Skip to main content

tabnas/
lexer.rs

1// Copyright (c) 2013-2026 Richard Rodger, MIT License
2
3use crate::error::TabnasError;
4use crate::options::{LexCheck, LexCheckResult, MatchTokenMatcher, Options};
5use crate::token::{
6    Point, Token, TIN_BD, TIN_CM, TIN_LN, TIN_NR, TIN_SP, TIN_ST, TIN_TX, TIN_VL, TIN_ZZ,
7};
8use crate::value::Value;
9use regex::Regex;
10use std::panic::{catch_unwind, AssertUnwindSafe};
11use std::sync::Arc;
12
13pub struct Lexer<'a> {
14    /// The text: the whole source, or the window of one read from a
15    /// reader ([`Text`]).
16    src: Text<'a>,
17    /// The characters from `low` on: the whole source's, or the window's.
18    chars: Vec<char>,
19    /// The byte offset of each character, or `None` for an ASCII source,
20    /// where a character's index is its byte offset (`byte_at`). Offsets
21    /// and indexes are the source's, from its start, in a window too.
22    byte_indices: Option<Vec<usize>>,
23    /// The end of the characters read: the source's length, or the
24    /// window's end.
25    char_len: usize,
26    /// The index and the byte offset of the first character `chars`
27    /// holds: zero for the whole source, the window's start otherwise.
28    low: usize,
29    low_byte: usize,
30    /// A read reached the window's end while the source had more: the
31    /// token being cut may be cut short, and is cut again over a larger
32    /// window ([`Lexer::next_raw_inner`]).
33    touched_end: std::cell::Cell<bool>,
34    idx: usize,
35    ri: usize,
36    ci: usize,
37    options: Arc<Options>,
38    ignore_tins: Vec<crate::Tin>,
39    char_sets: crate::text::CharSets,
40    err: Option<TabnasError>,
41    end_reached: bool,
42    /// `number.exclude`, compiled once per grammar and shared by the
43    /// parser that owns it; see [`compile_number_exclude`].
44    exclude_regex: Option<Arc<Regex>>,
45    want: Option<Vec<crate::Tin>>,
46    standalone: Option<(crate::Rule, crate::Context)>,
47}
48
49#[derive(Clone)]
50pub(crate) struct LexerState {
51    idx: usize,
52    ri: usize,
53    ci: usize,
54    err: Option<TabnasError>,
55    end_reached: bool,
56}
57
58/// Opaque snapshot returned by [`Lexer::relex_for_rule`]. Pass it to
59/// [`Lexer::unrelex`] if the caller later rejects the committed recut.
60#[derive(Clone)]
61pub struct RelexCheckpoint {
62    state: LexerState,
63    replay: std::collections::VecDeque<Token>,
64}
65
66enum CheckFlow {
67    Continue,
68    Skip,
69    Token(Box<Token>),
70}
71
72/// The result the lexer passes through its own internals.
73///
74/// `TabnasError` is 664 bytes, so `LexResult<Token>` is 664 bytes
75/// too: three times the token it carries, and moved on the SUCCESS path
76/// through every frame of the lexer chain. Boxing the error inside the
77/// engine makes those results the size of a token again, and costs an
78/// allocation only when there is an error to report, which is the path that
79/// is already building a 664-byte diagnostic.
80///
81/// The public entry points still hand back an unboxed `TabnasError`, so no
82/// caller of this crate sees the box.
83type LexResult<T> = Result<T, Box<TabnasError>>;
84
85/// The compiled form of `number.exclude`, or `None` when there is no
86/// pattern or it does not compile (an invalid pattern excludes nothing,
87/// as it always has).
88///
89/// Compiling a regex costs about 700K instructions, which is more than
90/// a small document costs to parse, so the parser compiles it once per
91/// grammar and hands every lexer the same automaton through the `Arc`.
92/// Cloning a `Regex` would not do: it shares the automaton but builds a
93/// fresh scratch-cache pool, and the first search from each clone fills
94/// it. Go and TypeScript compile the pattern once, at configuration time.
95pub(crate) fn compile_number_exclude(options: &Options) -> Option<Arc<Regex>> {
96    let pattern = options.number.exclude.as_deref()?;
97    Regex::new(pattern).ok().map(Arc::new)
98}
99
100/// What a lexer reads: the whole source, borrowed, or a window of a
101/// source it reads from a reader a chunk at a time, filling the window as
102/// it needs more and dropping it behind the oldest token the parser may
103/// still re-cut ([`Lexer::release_before`]).
104enum Text<'a> {
105    Whole(&'a str),
106    Window(Box<Window<'a>>),
107}
108
109/// The text an error shows its line from, and the row and column it
110/// starts at ([`Lexer::excerpt`]).
111pub(crate) type Excerpt = (String, usize, usize);
112
113/// A source read from a reader, as the parser hands it to the lexer: the
114/// reader, the window's sizes, and a cell the window leaves its text in
115/// when the lexer is done, so that an error the parser raised after the
116/// lexer is gone still shows its line from it.
117pub(crate) struct ReaderSource<'a> {
118    pub(crate) reader: Box<dyn std::io::Read + 'a>,
119    pub(crate) sizes: WindowSizes,
120    pub(crate) excerpt: std::rc::Rc<std::cell::RefCell<Option<Excerpt>>>,
121}
122
123impl<'a> ReaderSource<'a> {
124    pub(crate) fn new(reader: Box<dyn std::io::Read + 'a>, sizes: WindowSizes) -> Self {
125        ReaderSource {
126            reader,
127            sizes,
128            excerpt: std::rc::Rc::default(),
129        }
130    }
131}
132
133/// A source read from a reader ([`Text::Window`]).
134struct Window<'a> {
135    /// The text read and kept, from the lexer's `low_byte` on.
136    text: String,
137    reader: Box<dyn std::io::Read + 'a>,
138    /// Bytes read and not yet decoded: the start of a character a read
139    /// split.
140    pending: Vec<u8>,
141    /// The buffer a read fills, kept from one read to the next.
142    buffer: Vec<u8>,
143    exhausted: bool,
144    /// Why the source ended early, and where: a read that failed, or bytes
145    /// that are not UTF-8. The next token raises it ([`Lexer::unreadable`]).
146    failure: Option<Unreadable>,
147    /// The oldest character the parser may still ask for again.
148    keep: usize,
149    /// The row and column of the window's first character, counted as an
150    /// error's excerpt counts them: a line feed starts a row.
151    low_row: usize,
152    low_col: usize,
153    /// The row and column after the last character read, counted the same
154    /// way, which place a failure.
155    read_row: usize,
156    read_col: usize,
157    /// How far ahead of the cursor the window reaches before each token.
158    margin: usize,
159    /// How much dead text gathers before it is dropped, and how much a
160    /// read asks for.
161    block: usize,
162    chunk: usize,
163    /// Where the window leaves its text when the lexer is dropped
164    /// ([`ReaderSource::excerpt`]).
165    excerpt: std::rc::Rc<std::cell::RefCell<Option<Excerpt>>>,
166}
167
168impl Window<'_> {
169    /// Read on to the end of the window's last line, no further than a
170    /// margin, so that an excerpt shows the line an error is on as a parse
171    /// of the whole text shows it, but for a line longer than the window.
172    /// The lexer is done with the window, so this only adds to its text.
173    fn finish_line(&mut self) {
174        let mut read = 0;
175        while !self.exhausted && !self.text.ends_with('\n') && read < self.margin {
176            if self.buffer.len() != self.chunk {
177                self.buffer = vec![0u8; self.chunk];
178            }
179            match self.reader.read(&mut self.buffer) {
180                Ok(0) | Err(_) => self.exhausted = true,
181                Ok(n) => self.pending.extend_from_slice(&self.buffer[..n]),
182            }
183            let valid = match std::str::from_utf8(&self.pending) {
184                Ok(_) => self.pending.len(),
185                Err(e) => {
186                    if e.error_len().is_some() {
187                        self.exhausted = true;
188                    }
189                    e.valid_up_to()
190                }
191            };
192            let decoded = std::str::from_utf8(&self.pending[..valid]).expect("the prefix is UTF-8");
193            for c in decoded.chars() {
194                if self.text.ends_with('\n') || read >= self.margin {
195                    break;
196                }
197                self.text.push(c);
198                read += 1;
199            }
200            self.pending.drain(..valid);
201        }
202    }
203}
204
205impl Drop for Window<'_> {
206    fn drop(&mut self) {
207        // A parse that ended before its source did (an error) shows its
208        // line read to its end. The text is moved, not copied: the lexer
209        // is done with it.
210        self.finish_line();
211        *self.excerpt.borrow_mut() =
212            Some((std::mem::take(&mut self.text), self.low_row, self.low_col));
213    }
214}
215
216/// A source that could not be read to its end: what went wrong, and the
217/// character, row and column it went wrong at.
218#[derive(Clone, Debug)]
219pub(crate) struct Unreadable {
220    pub(crate) reason: String,
221    pub(crate) pos: usize,
222    pub(crate) row: usize,
223    pub(crate) col: usize,
224}
225
226impl Unreadable {
227    /// The error a parse fails with: `unreadable`, the reason as its source
228    /// text, at the character the source failed at.
229    pub(crate) fn error(&self) -> TabnasError {
230        TabnasError::new(
231            "unreadable",
232            self.reason.as_str(),
233            "",
234            self.pos,
235            self.row,
236            self.col,
237        )
238    }
239}
240
241#[cfg(test)]
242thread_local! {
243    /// The most characters a window has held on this thread, for the test
244    /// that holds a parse from a reader to a bounded window.
245    static WINDOW_PEAK: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
246}
247
248/// The reason a source is not UTF-8, as a reader's own `read_to_string`
249/// gives it.
250pub(crate) const NOT_UTF8: &str = "stream did not contain valid UTF-8";
251
252/// Where `text` ends, as a failure's position counts it: its characters,
253/// and the row and column after its last one, a line feed starting a row.
254pub(crate) fn position_after(text: &str) -> (usize, usize, usize) {
255    let (mut pos, mut row, mut col) = (0, 1, 1);
256    for c in text.chars() {
257        pos += 1;
258        if c == '\n' {
259            row += 1;
260            col = 1;
261        } else {
262            col += 1;
263        }
264    }
265    (pos, row, col)
266}
267
268/// A window's sizes: the margin ahead of the cursor, the dead text
269/// gathered before it is dropped, and a read's length. Each is 64 KiB
270/// (admin ADR-30), and none is an option: only the differential test makes
271/// them a few bytes, so that every matcher meets the window's edge
272/// ([`crate::Tabnas::parse_reader_with_sizes`]).
273#[doc(hidden)]
274#[derive(Clone, Copy, Debug)]
275pub struct WindowSizes {
276    pub margin: usize,
277    pub block: usize,
278    pub chunk: usize,
279}
280
281impl Default for WindowSizes {
282    fn default() -> Self {
283        WindowSizes {
284            margin: 64 * 1024,
285            block: 64 * 1024,
286            chunk: 64 * 1024,
287        }
288    }
289}
290
291impl<'a> Lexer<'a> {
292    pub fn new(src: &'a str, mut options: Options) -> Self {
293        if let Err(error) = options.validate_comment_definitions() {
294            panic!("invalid options: {error}");
295        }
296        // A lexer built directly may be handed options nobody has
297        // ordered yet. The parser's own lexer comes through
298        // `with_shared`, whose options were ordered when they were
299        // prepared, and whose exclude pattern was compiled then too.
300        options.sort_for_lexing();
301        let exclude_regex = compile_number_exclude(&options);
302        Self::with_shared(src, Arc::new(options), exclude_regex)
303    }
304
305    /// Lex against options the parser already owns and has ordered, with
306    /// the `number.exclude` pattern it compiled from them.
307    pub(crate) fn with_shared(
308        src: &'a str,
309        options: Arc<Options>,
310        exclude_regex: Option<Arc<Regex>>,
311    ) -> Self {
312        // Both tables are sized before they are filled: grown by doubling,
313        // a long source's peak was twice its tables.
314        let (chars, byte_indices) = if src.is_ascii() {
315            (src.bytes().map(char::from).collect::<Vec<char>>(), None)
316        } else {
317            let count = src.chars().count();
318            let mut chars = Vec::with_capacity(count);
319            let mut byte_indices = Vec::with_capacity(count);
320            for (b_idx, c) in src.char_indices() {
321                chars.push(c);
322                byte_indices.push(b_idx);
323            }
324            (chars, Some(byte_indices))
325        };
326        let char_len = chars.len();
327
328        Lexer {
329            src: Text::Whole(src),
330            chars,
331            byte_indices,
332            char_len,
333            low: 0,
334            low_byte: 0,
335            touched_end: std::cell::Cell::new(false),
336            idx: 0,
337            ri: 1,
338            ci: 1,
339            ignore_tins: options.ignore_tins(),
340            char_sets: options.char_sets(),
341            options,
342            err: None,
343            end_reached: false,
344            exclude_regex,
345            want: None,
346            standalone: None,
347        }
348    }
349
350    /// Lex a source read from `reader`, a window of it at a time
351    /// ([`Text::Window`]).
352    pub(crate) fn with_reader(
353        source: ReaderSource<'a>,
354        options: Arc<Options>,
355        exclude_regex: Option<Arc<Regex>>,
356    ) -> Self {
357        let ReaderSource {
358            reader,
359            sizes,
360            excerpt,
361        } = source;
362        Lexer {
363            src: Text::Window(Box::new(Window {
364                text: String::new(),
365                reader,
366                pending: Vec::new(),
367                buffer: Vec::new(),
368                exhausted: false,
369                failure: None,
370                keep: 0,
371                low_row: 1,
372                low_col: 1,
373                read_row: 1,
374                read_col: 1,
375                margin: sizes.margin.max(1),
376                block: sizes.block.max(1),
377                chunk: sizes.chunk.max(1),
378                excerpt,
379            })),
380            chars: Vec::new(),
381            byte_indices: None,
382            char_len: 0,
383            low: 0,
384            low_byte: 0,
385            touched_end: std::cell::Cell::new(false),
386            idx: 0,
387            ri: 1,
388            ci: 1,
389            ignore_tins: options.ignore_tins(),
390            char_sets: options.char_sets(),
391            options,
392            err: None,
393            end_reached: false,
394            exclude_regex,
395            want: None,
396            standalone: None,
397        }
398    }
399
400    /// The text the lexer holds: the whole source, or the window.
401    fn text(&self) -> &str {
402        match &self.src {
403            Text::Whole(src) => src,
404            Text::Window(window) => &window.text,
405        }
406    }
407
408    /// The source's length in bytes, or, in a window, the length read.
409    fn src_len(&self) -> usize {
410        match &self.src {
411            Text::Whole(src) => src.len(),
412            Text::Window(window) => self.low_byte + window.text.len(),
413        }
414    }
415
416    /// The text from the source's byte offset `byte` on, as far as the
417    /// lexer holds it.
418    fn tail_at_byte(&self, byte: usize) -> &str {
419        &self.text()[byte - self.low_byte..]
420    }
421
422    /// Whether the lexer reads a window of a source that has more to give.
423    #[inline]
424    fn window_open(&self) -> bool {
425        matches!(&self.src, Text::Window(window) if !window.exhausted)
426    }
427
428    /// Note a read at the window's end, where the source has more.
429    #[inline]
430    fn touch_end(&self) {
431        if self.window_open() {
432            self.touched_end.set(true);
433        }
434    }
435
436    /// The character at the source's index `index`, which the lexer holds.
437    #[inline]
438    fn ch(&self, index: usize) -> char {
439        self.chars[index - self.low]
440    }
441
442    /// The character at `index`, or `None` past what the lexer holds.
443    #[inline]
444    fn ch_get(&self, index: usize) -> Option<char> {
445        let found = self.chars.get(index - self.low).copied();
446        if found.is_none() {
447            self.touch_end();
448        }
449        found
450    }
451
452    /// Whether the lexer reads a window of its source.
453    pub fn windowed(&self) -> bool {
454        matches!(self.src, Text::Window(_))
455    }
456
457    /// The characters read so far, counted as UTF-16 units: in a window,
458    /// what the parser's step budget is measured against as it grows.
459    pub(crate) fn utf16_read(&self) -> usize {
460        match &self.src {
461            Text::Whole(src) => src.encode_utf16().count(),
462            Text::Window(_) => self.char_len,
463        }
464    }
465
466    /// Whether a source read from a reader turns out empty: the window is
467    /// filled until it holds a character or the reader is done. A source
468    /// that fails before its first character is not empty: its failure is
469    /// the parse's error.
470    pub(crate) fn is_empty_source(&mut self) -> bool {
471        self.fill(1);
472        self.char_len == 0 && !matches!(&self.src, Text::Window(window) if window.failure.is_some())
473    }
474
475    /// The failure that ended a source read from a reader early, taken
476    /// once: the parse fails with it at the next token.
477    pub(crate) fn unreadable(&mut self) -> Option<Unreadable> {
478        match &mut self.src {
479            Text::Window(window) => window.failure.take(),
480            Text::Whole(_) => None,
481        }
482    }
483
484    /// Whether the window has gathered a block of text behind the cursor,
485    /// so that dropping what the parser no longer holds may pay. Until it
486    /// has, the parser need not find the oldest token it holds.
487    #[inline]
488    pub(crate) fn may_release(&self) -> bool {
489        matches!(&self.src, Text::Window(window) if self.idx >= self.low + window.block)
490    }
491
492    /// The least position a window drops its text up to: it drops nothing
493    /// unless the oldest token the parser holds starts at or past it.
494    #[inline]
495    pub(crate) fn release_floor(&self) -> usize {
496        match &self.src {
497            Text::Window(window) => self.low + window.block,
498            Text::Whole(_) => usize::MAX,
499        }
500    }
501
502    /// The parser holds no token that starts before `pos`, so the window
503    /// may drop the text before it (or before the cursor, if that is
504    /// earlier).
505    pub(crate) fn release_before(&mut self, pos: usize) {
506        let keep = pos.min(self.idx);
507        if let Text::Window(window) = &mut self.src {
508            window.keep = keep;
509        }
510    }
511
512    /// The text an error shows its line from, and the row and column that
513    /// text starts at: the whole source from row 1 and column 1, or in a
514    /// parse from a reader the window, whose first line is cut where the
515    /// window begins.
516    pub(crate) fn excerpt(&self) -> (String, usize, usize) {
517        match &self.src {
518            Text::Whole(src) => ((*src).to_string(), 1, 1),
519            Text::Window(window) => (window.text.clone(), window.low_row, window.low_col),
520        }
521    }
522
523    /// The text from the cursor to the end of what the lexer holds: in a
524    /// parse from a reader, the end of the window. A matcher in a grammar
525    /// that declares `lex.window` reads the source through this and
526    /// [`Lexer::fill`], never past them.
527    pub fn window(&self) -> &str {
528        self.tail_at_byte(self.byte_position())
529    }
530
531    /// Read the source until `ahead` characters lie past the cursor, or it
532    /// ends, and say whether they do. A whole source is all read, so this
533    /// only answers. In a parse from a reader the window may drop text
534    /// behind the oldest token the parser holds, never the token being
535    /// cut, and a source that fails to read ends where it failed, the parse
536    /// failing with it at the next token.
537    pub fn fill(&mut self, ahead: usize) -> bool {
538        self.fill_window(ahead);
539        self.char_len - self.idx >= ahead
540    }
541
542    /// Fill the window until `ahead` characters lie past the cursor or the
543    /// source is exhausted, dropping first the text before what the parser
544    /// may still ask for, a block at a time.
545    fn fill_window(&mut self, ahead: usize) {
546        let Text::Window(window) = &mut self.src else {
547            return;
548        };
549        let window: &mut Window<'a> = window;
550        let keep = window.keep.min(self.idx);
551        // Most tokens find the window long enough and nothing to drop.
552        if (window.exhausted || self.char_len - self.idx >= ahead) && keep < self.low + window.block
553        {
554            return;
555        }
556        if keep >= self.low + window.block {
557            let chars = keep - self.low;
558            // A drop to the end of what has been read, where the parser
559            // holds no token and the cursor stands at the end, drops the
560            // whole text: there is no character there to take the offset
561            // of (#322).
562            let bytes = match &self.byte_indices {
563                Some(offsets) => offsets
564                    .get(chars)
565                    .map_or(window.text.len(), |offset| offset - self.low_byte),
566                None => chars,
567            };
568            let dropped = &window.text[..bytes];
569            match dropped.rfind('\n') {
570                Some(last) => {
571                    window.low_row += dropped.bytes().filter(|&b| b == b'\n').count();
572                    window.low_col = 1 + dropped[last + 1..].chars().count();
573                }
574                None => window.low_col += chars,
575            }
576            self.chars.drain(..chars);
577            if let Some(offsets) = &mut self.byte_indices {
578                offsets.drain(..chars);
579            }
580            window.text.drain(..bytes);
581            self.low = keep;
582            self.low_byte += bytes;
583        }
584        if window.buffer.len() != window.chunk {
585            window.buffer = vec![0u8; window.chunk];
586        }
587        while !window.exhausted && self.char_len - self.idx < ahead {
588            let read = loop {
589                match window.reader.read(&mut window.buffer) {
590                    Ok(read) => break Ok(read),
591                    Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
592                    Err(e) => break Err(e.to_string()),
593                }
594            };
595            let read = match read {
596                Ok(read) => read,
597                Err(reason) => {
598                    window.exhausted = true;
599                    window.failure = Some(Unreadable {
600                        reason,
601                        pos: self.char_len,
602                        row: window.read_row,
603                        col: window.read_col,
604                    });
605                    break;
606                }
607            };
608            if read == 0 {
609                window.exhausted = true;
610            }
611            window.pending.extend_from_slice(&window.buffer[..read]);
612            // The longest prefix that decodes. A character a read split
613            // waits for its last byte; bytes that are no UTF-8, or a
614            // character the source ends inside, end the source there, as
615            // `read_to_string` refuses them.
616            let valid = match std::str::from_utf8(&window.pending) {
617                Ok(_) => window.pending.len(),
618                Err(e) => {
619                    if e.error_len().is_some() || window.exhausted {
620                        window.exhausted = true;
621                        window.failure = Some(Unreadable {
622                            reason: NOT_UTF8.to_string(),
623                            pos: 0,
624                            row: 0,
625                            col: 0,
626                        });
627                    }
628                    e.valid_up_to()
629                }
630            };
631            let decoded =
632                std::str::from_utf8(&window.pending[..valid]).expect("the prefix is UTF-8");
633            for c in decoded.chars() {
634                if self.byte_indices.is_none() && !c.is_ascii() {
635                    // Every character so far was ASCII, its offset its
636                    // index.
637                    let mut offsets = Vec::with_capacity(self.chars.capacity());
638                    offsets.extend((0..self.chars.len()).map(|k| self.low + k));
639                    self.byte_indices = Some(offsets);
640                }
641                if let Some(offsets) = &mut self.byte_indices {
642                    offsets.push(self.low_byte + window.text.len());
643                }
644                if c == '\n' {
645                    window.read_row += 1;
646                    window.read_col = 1;
647                } else {
648                    window.read_col += 1;
649                }
650                window.text.push(c);
651                self.chars.push(c);
652            }
653            window.pending.drain(..valid);
654            self.char_len = self.low + self.chars.len();
655            #[cfg(test)]
656            WINDOW_PEAK.with(|peak| peak.set(peak.get().max(self.chars.len())));
657            if let Some(failure) = &mut window.failure {
658                if failure.row == 0 {
659                    // The bytes that are no UTF-8 start after the last
660                    // character decoded.
661                    failure.pos = self.char_len;
662                    failure.row = window.read_row;
663                    failure.col = window.read_col;
664                }
665            }
666        }
667    }
668
669    fn current_point(&self) -> Point {
670        Point {
671            len: self.src_len(),
672            site: crate::Site {
673                si: self.byte_position(),
674                pos: self.idx,
675                ri: self.ri,
676                ci: self.ci,
677            },
678        }
679    }
680
681    /// The source from char index `start` up to `end`, clipped to the end
682    /// of the source: the span of a bad token, cut as TypeScript's
683    /// `lex.bad(why, pstart, pend)` cuts it (`ts/src/lexer.ts`).
684    ///
685    /// Only the END is clipped. The caller must supply `start <= end`
686    /// and `start <= self.char_len`, or the indexing panics. Every call
687    /// site below passes `esc_point.site.pos - 1`, which additionally
688    /// needs `1 <= pos`; that holds because `esc_point` is captured
689    /// after the escape character has been consumed. These three are
690    /// the preconditions `rs/verus/lexer_span.rs` ASSUMES: it proves the
691    /// span arithmetic is in bounds given them, and does not model the
692    /// call sites, so nothing machine-checks that they hold here. A
693    /// refactor that captured the point before the advance would still
694    /// pass `rs/verus/run.sh`.
695    fn source_span(&self, start: usize, end: usize) -> String {
696        if end > self.char_len {
697            // A span cut at a window's end may be cut short.
698            self.touch_end();
699        }
700        self.chars[start - self.low..end.min(self.char_len) - self.low]
701            .iter()
702            .collect()
703    }
704
705    fn state(&self) -> LexerState {
706        LexerState {
707            idx: self.idx,
708            ri: self.ri,
709            ci: self.ci,
710            err: self.err.clone(),
711            end_reached: self.end_reached,
712        }
713    }
714
715    /// Full immutable source supplied to this lexer; in a parse from a
716    /// reader, the window the lexer holds.
717    pub fn source(&self) -> &str {
718        self.text()
719    }
720
721    /// Source remaining at the live cursor; in a parse from a reader, to
722    /// the window's end.
723    pub fn remaining(&self) -> &str {
724        self.tail_at_byte(self.byte_position())
725    }
726
727    /// Return at most `max_chars` Unicode scalar values from the live cursor.
728    /// This is the Rust counterpart of the public `lex.fwd`/`Lex.Fwd` helper.
729    pub fn forward(&self, max_chars: usize) -> &str {
730        let remaining = self.remaining();
731        let end = remaining
732            .char_indices()
733            .nth(max_chars)
734            .map_or(remaining.len(), |(index, _)| index);
735        &remaining[..end]
736    }
737
738    /// Snapshot the live cursor for token construction.
739    pub fn point(&self) -> Point {
740        self.current_point()
741    }
742
743    /// Advance by Unicode scalar values. Returns false without moving when
744    /// the requested count extends beyond end-of-source.
745    pub fn advance_chars(&mut self, count: usize) -> bool {
746        if self.idx.saturating_add(count) > self.char_len {
747            // In a window, past its end may yet be in the source: the cut
748            // is made again over a larger window.
749            self.touch_end();
750            return false;
751        }
752        for _ in 0..count {
753            self.advance();
754        }
755        true
756    }
757
758    /// Construct a token from a point captured before cursor advancement.
759    pub fn token(
760        &self,
761        name: impl AsRef<str>,
762        tin: crate::Tin,
763        value: Value,
764        source: impl Into<crate::TokenText>,
765        point: Point,
766    ) -> Token {
767        Token::new(name, tin, value, source, point)
768    }
769
770    /// Resolve or allocate a token identity in this lexer's configuration.
771    pub fn token_tin(&mut self, name: impl Into<String>) -> crate::Tin {
772        Arc::make_mut(&mut self.options).register_token(name)
773    }
774
775    /// Resolve a token identity back to its configured name.
776    pub fn token_name(&self, tin: crate::Tin) -> String {
777        self.options.token_name(tin)
778    }
779
780    /// Whether the options this lexer runs under enable `alt`: not when
781    /// `rule.exclude` names one of its groups, nor when `rule.include`
782    /// lists groups and it declares none of them. The parser skips such an
783    /// alternate, and TypeScript removes it from the rule spec
784    /// (`filterRules`) before anything reads the spec, so a custom matcher
785    /// that reads a rule's alternates, to tell a key position from a value
786    /// one, asks this to see the same alternates TypeScript does.
787    pub fn alt_enabled(&self, alt: &crate::AltSpec) -> bool {
788        crate::parser::groups_enabled(alt, &self.options)
789    }
790
791    /// Construct a bad token at the current cursor.
792    ///
793    /// A custom matcher that returns it reports a lexer fault, which the
794    /// parser handles as it handles its own lexer's: a rule's fetch raises
795    /// it at once with `why` as the code, at this position, or, under
796    /// recovery, records it, coalescing a run of them into one error, and
797    /// moves the cursor past its source; relexing leaves it for an
798    /// alternate to re-cut. The cursor need not be advanced here.
799    pub fn bad(&self, why: impl Into<String>) -> Token {
800        let point = self.current_point();
801        let source = self
802            .peek()
803            .map_or_else(String::new, |character| character.to_string());
804        let mut token = Token::new("#BD", TIN_BD, Value::Undefined, source, point);
805        token.err = crate::TokenCode::from(why.into());
806        token.why = token.err.clone();
807        token
808    }
809
810    /// Construct a bad token whose displayed source is a scalar-indexed span.
811    /// As in TypeScript, the diagnostic point remains the live cursor.
812    /// Returned from a matcher it is handled as [`Lexer::bad`] describes;
813    /// recovery steps past the span, counted from that cursor.
814    pub fn bad_span(&self, why: impl Into<String>, start: usize, end: usize) -> Token {
815        let point = self.current_point();
816        let source = if start <= end && end <= self.char_len {
817            self.text()[self.byte_at(start) - self.low_byte..self.byte_at(end) - self.low_byte]
818                .to_string()
819        } else {
820            self.peek()
821                .map_or_else(String::new, |character| character.to_string())
822        };
823        let mut token = Token::new("#BD", TIN_BD, Value::Undefined, source, point);
824        token.err = crate::TokenCode::from(why.into());
825        token.why = token.err.clone();
826        token
827    }
828
829    fn byte_position(&self) -> usize {
830        self.byte_at(self.idx)
831    }
832
833    /// The byte offset of the character at `index`, or the source's length
834    /// from its end on.
835    fn byte_at(&self, index: usize) -> usize {
836        match &self.byte_indices {
837            Some(offsets) => offsets
838                .get(index - self.low)
839                .copied()
840                .unwrap_or(self.src_len()),
841            None => index.min(self.src_len()),
842        }
843    }
844
845    fn advance(&mut self) -> Option<char> {
846        if self.idx < self.char_len {
847            let c = self.chars[self.idx - self.low];
848            self.idx += 1;
849            if self.char_sets.row.contains(c) {
850                self.ri += 1;
851                self.ci = 1;
852            } else {
853                self.ci += 1;
854            }
855            Some(c)
856        } else {
857            self.touch_end();
858            None
859        }
860    }
861
862    /// Step over one character of a string body, counting a column and
863    /// never a row. `advance` counts a row for any character in
864    /// `line.rowChars`, which is right between tokens and wrong inside a
865    /// string: TypeScript's `buildStringBodySpec` (ts/src/lexer.ts) makes
866    /// a row character plain body there unless the string is multi-line
867    /// and the character is in `line.chars` as well, which is
868    /// [`Lexer::advance_string_line`].
869    fn advance_body(&mut self) -> Option<char> {
870        let c = self.peek()?;
871        self.idx += 1;
872        self.ci += 1;
873        Some(c)
874    }
875
876    /// Step over a line character inside a multi-line string: the column
877    /// resets, and a character in `line.rowChars` counts a row, as
878    /// TypeScript's string-body classes LINE and LINE+ROW do
879    /// (`STRING_BODY_TABLE` in ts/src/lexer.ts) and as Go's
880    /// `BuildStringBodySpec` ports them.
881    fn advance_string_line(&mut self) -> Option<char> {
882        let c = self.peek()?;
883        self.idx += 1;
884        if self.char_sets.row.contains(c) {
885            self.ri += 1;
886        }
887        self.ci = 1;
888        Some(c)
889    }
890
891    fn peek(&self) -> Option<char> {
892        if self.idx < self.char_len {
893            Some(self.chars[self.idx - self.low])
894        } else {
895            self.touch_end();
896            None
897        }
898    }
899
900    fn peek_at(&self, offset: usize) -> Option<char> {
901        let i = self.idx + offset;
902        if i < self.char_len {
903            Some(self.chars[i - self.low])
904        } else {
905            self.touch_end();
906            None
907        }
908    }
909
910    fn wants(&self, tin: crate::Tin) -> bool {
911        self.want
912            .as_ref()
913            .is_none_or(|wanted| wanted.contains(&tin))
914    }
915
916    /// Bring a fetch's point and current character up to the live cursor.
917    ///
918    /// A custom matcher, or an imperative check, may move the cursor and
919    /// then decline: xml's steps over a byte-order mark and goes on to
920    /// look for a tag. Every matcher after it starts where it left the
921    /// cursor, as TypeScript's do (each reads `lex.pnt`) and Go's do (each
922    /// reads `l.pnt`), and the current character is `None` once the cursor
923    /// is at the end of the source. Reading the character the fetch began
924    /// on instead put the built-in matchers' tokens at the wrong place,
925    /// sent them looking for a character that was no longer there, and,
926    /// when the matcher had stepped over the LAST character, panicked on
927    /// the missing one.
928    #[inline]
929    fn resync(&self, point: &mut Point, current: &mut Option<char>) {
930        if self.idx != point.site.pos {
931            *point = self.current_point();
932            *current = self.peek();
933        }
934    }
935
936    /// Whether TypeScript would try a built-in matcher at all in a fetch
937    /// that began on `first` and whose cursor has since been moved by a
938    /// matcher that declined.
939    ///
940    /// TypeScript chooses the matchers a fetch tries from a table indexed
941    /// by the character the fetch begins on (`buildLexDispatch` in
942    /// ts/src/utility.ts, read in `Lex.next`): a built-in is listed for a
943    /// Latin-1 character only when a token it makes could start with it,
944    /// unless it carries a check, and every built-in is listed for a
945    /// character from U+0100 up. The table is not consulted again when a
946    /// custom matcher moves the cursor, so this asks it about `first`, and
947    /// the matcher then tests the character under the cursor as usual.
948    /// Before the cursor moves the two are the same character and the
949    /// matcher's own test is the whole answer, so this is `true`.
950    #[inline]
951    fn listed(
952        &self,
953        entry: usize,
954        first: char,
955        has_check: bool,
956        could_start: impl FnOnce(char) -> bool,
957    ) -> bool {
958        self.idx == entry || u32::from(first) >= 256 || has_check || could_start(first)
959    }
960
961    fn run_check(&mut self, check: Option<LexCheck>, point: Point) -> CheckFlow {
962        let Some(check) = check else {
963            return CheckFlow::Continue;
964        };
965        let at = self.byte_position();
966        let result = match check.run_imperative(self) {
967            Some(result) => result,
968            None => check
969                .run(self.tail_at_byte(at))
970                .unwrap_or(LexCheckResult::Continue),
971        };
972        match result {
973            LexCheckResult::Continue => CheckFlow::Continue,
974            LexCheckResult::Skip => CheckFlow::Skip,
975            LexCheckResult::NativeToken(token) => CheckFlow::Token(token),
976            LexCheckResult::Token(token)
977                if !token.source.is_empty() && self.tail_at_byte(at).starts_with(&token.source) =>
978            {
979                let tin = if token.tin < 0 {
980                    self.options.token(&token.name).unwrap_or(token.tin)
981                } else {
982                    token.tin
983                };
984                if tin < 0 {
985                    return CheckFlow::Skip;
986                }
987                for _ in token.source.chars() {
988                    self.advance();
989                }
990                CheckFlow::Token(Box::new(Token::new(
991                    token.name,
992                    tin,
993                    token.value,
994                    token.source,
995                    point,
996                )))
997            }
998            LexCheckResult::Token(_) => CheckFlow::Skip,
999        }
1000    }
1001
1002    /// Give any plugin matcher whose order is below `before` its turn.
1003    ///
1004    /// Nine sites in the lexer call this per token, once at each stage a
1005    /// matcher is allowed to intervene. A grammar with no custom matcher,
1006    /// which is most of them, was paying nine index lookups per token to be
1007    /// told nine times that there is nothing to run. The guard is inline so
1008    /// those sites skip the call itself; the walk stays out of line.
1009    #[inline]
1010    fn run_custom_matchers(
1011        &mut self,
1012        index: &mut usize,
1013        before: f64,
1014        plugin: &mut Option<(&mut crate::Rule, &mut crate::Context)>,
1015    ) -> Option<Token> {
1016        if *index >= self.options.lex.matchers.len() {
1017            return None;
1018        }
1019        self.run_remaining_custom_matchers(index, before, plugin)
1020    }
1021
1022    #[inline(never)]
1023    fn run_remaining_custom_matchers(
1024        &mut self,
1025        index: &mut usize,
1026        before: f64,
1027        plugin: &mut Option<(&mut crate::Rule, &mut crate::Context)>,
1028    ) -> Option<Token> {
1029        while let Some(matcher) = self
1030            .options
1031            .lex
1032            .matchers
1033            .get_index(*index)
1034            .map(|(_, matcher)| matcher)
1035            .filter(|matcher| matcher.order < before)
1036            .cloned()
1037        {
1038            *index += 1;
1039            // Each matcher starts where the one before it left the cursor:
1040            // one may step over a character and decline, as xml's does over
1041            // a byte-order mark, and TypeScript's matchers all read the
1042            // live `lex.pnt`.
1043            let point = self.current_point();
1044            let at = self.byte_position();
1045            let saved = self.state();
1046            let token = if let Some(callback) = matcher.imperative.as_ref() {
1047                let Some((rule, context)) = plugin.as_mut() else {
1048                    continue;
1049                };
1050                callback(self, rule, context)
1051            } else {
1052                let remaining = self.tail_at_byte(at);
1053                matcher
1054                    .matcher
1055                    .as_ref()
1056                    .and_then(|callback| callback(remaining))
1057                    .filter(|token| {
1058                        !token.source.is_empty() && remaining.starts_with(&token.source)
1059                    })
1060                    .map(|token| {
1061                        Token::new(token.name, token.tin, token.value, token.source, point)
1062                    })
1063            };
1064            let Some(mut token) = token else {
1065                if self.want.is_some() {
1066                    self.rewind(saved);
1067                }
1068                continue;
1069            };
1070
1071            // TypeScript and Go run opaque custom matchers speculatively for
1072            // a negotiated cut. An unwanted result rolls back locally so a
1073            // later matcher can still satisfy the request.
1074            let tin = if token.tin < 0 {
1075                self.options.token(&token.name).unwrap_or(token.tin)
1076            } else {
1077                token.tin
1078            };
1079            if tin < 0 || !self.wants(tin) {
1080                self.rewind(saved);
1081                continue;
1082            }
1083            if matcher.imperative.is_none() {
1084                for _ in token.src.chars() {
1085                    self.advance();
1086                }
1087            }
1088            token.tin = tin;
1089            return Some(token);
1090        }
1091        None
1092    }
1093
1094    fn is_text_delimiter_here(&self) -> bool {
1095        self.is_text_delimiter_at(self.idx)
1096    }
1097
1098    fn is_text_delimiter_at(&self, index: usize) -> bool {
1099        let Some(ch) = self.ch_get(index) else {
1100            return true;
1101        };
1102        let remaining = self.tail_at_byte(self.byte_at(index));
1103        (self.options.space.lex && self.char_sets.space.contains(ch))
1104            || (self.options.fixed.lex
1105                && self
1106                    .options
1107                    .fixed
1108                    .tokens
1109                    .values()
1110                    .any(|token| !token.source.is_empty() && remaining.starts_with(&token.source)))
1111            || (self.options.line.lex
1112                && (self.char_sets.line_ends.contains(ch) || matches!(ch, '\u{2028}' | '\u{2029}')))
1113            || (self.options.comment.lex
1114                && self.options.comment.definitions.values().any(|definition| {
1115                    definition.lex
1116                        && !definition.start.is_empty()
1117                        && remaining.starts_with(&definition.start)
1118                }))
1119            || self
1120                .options
1121                .ender
1122                .iter()
1123                .any(|ender| !ender.is_empty() && remaining.starts_with(ender))
1124    }
1125
1126    /// Fetches the next non-IGNORE token (skipping spaces, lines, comments).
1127    pub fn next_token(&mut self) -> Result<Token, TabnasError> {
1128        let point = self.current_point();
1129        let result = match catch_unwind(AssertUnwindSafe(|| {
1130            if let Some(ref error) = self.err {
1131                return Err(Box::new(error.clone()));
1132            }
1133
1134            loop {
1135                let token = self.next_raw(None)?;
1136                if !self.ignore_tins.contains(&token.tin) {
1137                    return Ok(token);
1138                }
1139            }
1140        })) {
1141            Ok(result) => result,
1142            Err(payload) => self.record_panic(payload, "Lexer::next_token", point),
1143        };
1144        // The box is internal to the engine; a caller gets the error itself.
1145        result.map_err(|error| *error)
1146    }
1147
1148    /// Fetch the next token without discarding whitespace, line, or comment tokens.
1149    pub fn next_raw_token(&mut self) -> Result<Token, TabnasError> {
1150        let point = self.current_point();
1151        let result = match catch_unwind(AssertUnwindSafe(|| self.next_raw(None))) {
1152            Ok(result) => result,
1153            Err(payload) => self.record_panic(payload, "Lexer::next_raw_token", point),
1154        };
1155        result.map_err(|error| *error)
1156    }
1157
1158    /// Fetch one token for an imperative parser callback, preserving ignored
1159    /// space/line/comment tokens just like TypeScript's public `lex.next`.
1160    /// Replayed tokens produced by `Context::rewind` are served first.
1161    pub fn next_raw_for_rule(
1162        &mut self,
1163        rule: &mut crate::Rule,
1164        context: &mut crate::Context,
1165    ) -> LexResult<Token> {
1166        if let Some(token) = context.next_replay() {
1167            Ok(token)
1168        } else {
1169            self.next_raw_with(None, Some((rule, context)))
1170        }
1171    }
1172
1173    /// Fetch the next non-ignored token for an imperative parser callback.
1174    pub fn next_for_rule(
1175        &mut self,
1176        rule: &mut crate::Rule,
1177        context: &mut crate::Context,
1178    ) -> LexResult<Token> {
1179        loop {
1180            let token = self.next_raw_for_rule(rule, context)?;
1181            if !self.ignore_tins.contains(&token.tin) {
1182                return Ok(token);
1183            }
1184        }
1185    }
1186
1187    /// Public negotiated-relex entry point for native parser callbacks.
1188    /// A successful recut commits the lexer cursor and returns an opaque undo
1189    /// checkpoint; a failed recut restores all lexer state before returning.
1190    pub fn relex_for_rule(
1191        &mut self,
1192        from: &Token,
1193        wanted: &[crate::Tin],
1194        rule: &mut crate::Rule,
1195        context: &mut crate::Context,
1196    ) -> Option<(Token, RelexCheckpoint)> {
1197        self.relex(from, wanted, rule, context)
1198    }
1199
1200    /// Undo a committed [`Lexer::relex_for_rule`] operation, including the
1201    /// pending tokens hidden while the replacement cut was negotiated.
1202    pub fn unrelex(&mut self, checkpoint: RelexCheckpoint, context: &mut crate::Context) {
1203        self.restore(checkpoint.state);
1204        context.restore_replay(checkpoint.replay);
1205    }
1206
1207    fn record_panic(
1208        &mut self,
1209        payload: Box<dyn std::any::Any + Send>,
1210        api: &str,
1211        point: Point,
1212    ) -> LexResult<Token> {
1213        let error = TabnasError::from_panic(
1214            payload,
1215            api,
1216            self.text(),
1217            point.site.pos,
1218            point.site.ri,
1219            point.site.ci,
1220            &self.options,
1221        );
1222        self.err = Some(error.clone());
1223        Err(Box::new(error))
1224    }
1225
1226    /// Fetch a raw token while restricting non-eager custom token matchers to
1227    /// the exact tins accepted at the parser slot being filled. Builtin and
1228    /// fixed-token matchers are unaffected by this gate.
1229    pub(crate) fn next_rule_token(
1230        &mut self,
1231        expected_match_tins: &[crate::Tin],
1232        rule: &mut crate::Rule,
1233        context: &mut crate::Context,
1234    ) -> LexResult<Token> {
1235        self.next_raw_with(Some(expected_match_tins), Some((rule, context)))
1236    }
1237
1238    /// Step past a bad token the parser has absorbed or skipped, as
1239    /// TypeScript's `advanceLexPast` does (ts/src/rules.ts): a bad token
1240    /// does not advance the cursor by itself, so recovery moves it to the
1241    /// end of the token's span, never backwards, and, for a fault raised
1242    /// inside a compound construct, on past the next row character so
1243    /// lexing resumes on a fresh row. The lexer's own faults latch until
1244    /// this clears them.
1245    pub(crate) fn skip_bad(&mut self, token: &Token, to_line_end: bool) {
1246        let span = token.src.chars().count().max(1);
1247        let mut target = self.idx.max(token.site.pos.saturating_add(span));
1248        if to_line_end {
1249            let mut end = target;
1250            while end < self.char_len && !self.char_sets.row.contains(self.ch(end)) {
1251                end += 1;
1252            }
1253            target = target.max(self.char_len.min(end + 1));
1254        }
1255        while self.idx < target && self.idx < self.char_len {
1256            self.advance();
1257        }
1258        self.err = None;
1259        if self.idx < self.char_len {
1260            self.end_reached = false;
1261        }
1262    }
1263
1264    /// Re-cut an already buffered source span, constrained to the token
1265    /// identities requested by one alternate. On success the cursor remains
1266    /// after the new cut; the returned state can restore the original cut if
1267    /// that alternate later fails.
1268    pub(crate) fn relex(
1269        &mut self,
1270        from: &Token,
1271        wanted: &[crate::Tin],
1272        rule: &mut crate::Rule,
1273        context: &mut crate::Context,
1274    ) -> Option<(Token, RelexCheckpoint)> {
1275        if from.src.is_empty() || from.site.pos > self.char_len || wanted.is_empty() {
1276            return None;
1277        }
1278        // The standing error, if any, is moved into the checkpoint rather
1279        // than copied: the cut below clears it anyway, and a copy is the
1280        // length of the source (`full_source`) for every cut attempted.
1281        let err = self.err.take();
1282        let saved = LexerState {
1283            err,
1284            ..self.state()
1285        };
1286        // TypeScript temporarily replaces the lexer's pending-token queue
1287        // with an empty queue for a negotiated cut. Rust keeps that queue on
1288        // Context, so hide it explicitly and preserve it in the checkpoint.
1289        let replay = context.take_replay();
1290        self.idx = from.site.pos;
1291        self.ri = from.site.ri;
1292        self.ci = from.site.ci;
1293        self.err = None;
1294        self.end_reached = false;
1295        self.want = Some(wanted.to_vec());
1296        // Straight to the matchers, past `next_raw_with`: an error here only
1297        // rejects the cut, and the restore below puts back the lexer's own,
1298        // so the source it would attach, and the copy it would keep, are
1299        // never seen. With them, every rejected cut cost the length of the
1300        // source, and a flat stylesheet parsed in quadratic time.
1301        let recut = self.next_raw_inner(None, Some((rule, context))).ok();
1302        self.want = None;
1303        match recut.filter(|token| wanted.contains(&token.tin)) {
1304            Some(mut token) => {
1305                token.ignored = from.ignored.clone();
1306                Some((
1307                    token,
1308                    RelexCheckpoint {
1309                        state: saved,
1310                        replay,
1311                    },
1312                ))
1313            }
1314            None => {
1315                self.restore(saved);
1316                // Discard any speculative replay generated by an imperative
1317                // matcher and restore the queue that preceded the attempt.
1318                context.restore_replay(replay);
1319                None
1320            }
1321        }
1322    }
1323
1324    pub(crate) fn restore(&mut self, state: LexerState) {
1325        self.idx = state.idx;
1326        self.ri = state.ri;
1327        self.ci = state.ci;
1328        self.err = state.err;
1329        self.end_reached = state.end_reached;
1330        self.want = None;
1331    }
1332
1333    /// Put the cursor back after a custom matcher's speculative attempt and
1334    /// keep the request being negotiated, as TypeScript's `Lex.speculate`
1335    /// restores the point and leaves `want` alone. [`Lexer::restore`] ends
1336    /// a negotiation, so the first custom matcher to decline during a
1337    /// re-cut used to lift the request for every matcher after it, and a
1338    /// string matcher could then cut the `#ST` the re-cut was there to
1339    /// avoid.
1340    fn rewind(&mut self, state: LexerState) {
1341        let want = self.want.take();
1342        self.restore(state);
1343        self.want = want;
1344    }
1345
1346    fn next_raw(&mut self, expected_match_tins: Option<&[crate::Tin]>) -> LexResult<Token> {
1347        // Only a lexer being driven directly needs these, and building
1348        // them costs a whole `Options` clone. A parse reaches the lexer
1349        // through `next_rule_token`, which brings the real rule and
1350        // context with it, so it never wants them at all.
1351        let (mut rule, mut context) = match self.standalone.take() {
1352            Some(pair) => pair,
1353            None => (
1354                crate::Rule::new("#NORULE", Value::Undefined),
1355                crate::Context::new(
1356                    self.options.rewind.history,
1357                    self.text(),
1358                    Value::Undefined,
1359                    Arc::clone(&self.options),
1360                    crate::InstanceInfo::default(),
1361                ),
1362            ),
1363        };
1364        let result = self.next_raw_with(expected_match_tins, Some((&mut rule, &mut context)));
1365        self.standalone = Some((rule, context));
1366        result
1367    }
1368
1369    fn modify_text_value(
1370        &mut self,
1371        mut value: Value,
1372        plugin: &mut Option<(&mut crate::Rule, &mut crate::Context)>,
1373    ) -> Value {
1374        if self.options.text.modify.is_empty() {
1375            return value;
1376        }
1377        let modifiers = self.options.text.modify.clone();
1378        let options = self.options.clone();
1379        let Some((rule, context)) = plugin.as_mut() else {
1380            panic!("imperative text modifier requires an active lexer context");
1381        };
1382        for modifier in modifiers {
1383            value = modifier.run(value, self, rule, context, &options);
1384        }
1385        value
1386    }
1387
1388    fn next_raw_with(
1389        &mut self,
1390        expected_match_tins: Option<&[crate::Tin]>,
1391        plugin: Option<(&mut crate::Rule, &mut crate::Context)>,
1392    ) -> LexResult<Token> {
1393        let result = self.next_raw_inner(expected_match_tins, plugin);
1394        match result {
1395            Ok(token) => Ok(token),
1396            Err(mut error) => {
1397                // The matchers build their errors without the source, and it
1398                // is attached here, where an error leaves the lexer: a
1399                // negotiated cut (`relex`) rejects many candidates, each an
1400                // error nobody sees, and a copy of the whole source apiece
1401                // made that quadratic. A parse from a reader attaches the
1402                // window instead, once the lexer is done and the window has
1403                // read on to the end of its line
1404                // ([`crate::lexer::ReaderSource`]).
1405                if !self.windowed() {
1406                    let (text, row, col) = self.excerpt();
1407                    error.set_excerpt(text, row, col);
1408                }
1409                error.apply_options(&self.options);
1410                self.err = Some((*error).clone());
1411                Err(error)
1412            }
1413        }
1414    }
1415
1416    /// Cut the next token. In a window, the window is filled to its
1417    /// margin ahead of the cursor first, and a token whose cut read to the
1418    /// window's end while the source had more is cut again over a window
1419    /// twice as far ahead, so that it is the token the whole text gives.
1420    fn next_raw_inner(
1421        &mut self,
1422        expected_match_tins: Option<&[crate::Tin]>,
1423        mut plugin: Option<(&mut crate::Rule, &mut crate::Context)>,
1424    ) -> LexResult<Token> {
1425        let Text::Window(window) = &self.src else {
1426            return self.next_raw_cut(expected_match_tins, plugin);
1427        };
1428        let mut ahead = window.margin;
1429        loop {
1430            self.fill_window(ahead);
1431            if let Some(failure) = self.unreadable() {
1432                return Err(Box::new(failure.error()));
1433            }
1434            let saved = self.state();
1435            self.touched_end.set(false);
1436            let result = self.next_raw_cut(
1437                expected_match_tins,
1438                plugin
1439                    .as_mut()
1440                    .map(|(rule, context)| (&mut **rule, &mut **context)),
1441            );
1442            // The cut stands when nothing read to the window's end while
1443            // the source had more, and it ended inside the window or the
1444            // source is all read. A read that touched the end decided on a
1445            // window shorter than the source, even where a later matcher's
1446            // fill has since read the source to its end: made again, over
1447            // the window that fill left, it touches nothing and stands
1448            // (#326).
1449            if !self.touched_end.get() && (!self.window_open() || self.idx < self.char_len) {
1450                return result;
1451            }
1452            self.rewind(saved);
1453            ahead = ahead.saturating_mul(2);
1454        }
1455    }
1456
1457    fn next_raw_cut(
1458        &mut self,
1459        expected_match_tins: Option<&[crate::Tin]>,
1460        mut plugin: Option<(&mut crate::Rule, &mut crate::Context)>,
1461    ) -> LexResult<Token> {
1462        if self.end_reached {
1463            return Ok(Token::new(
1464                "#ZZ",
1465                TIN_ZZ,
1466                Value::Undefined,
1467                "",
1468                self.current_point(),
1469            ));
1470        }
1471
1472        if self.idx >= self.char_len {
1473            self.end_reached = true;
1474            return Ok(Token::new(
1475                "#ZZ",
1476                TIN_ZZ,
1477                Value::Undefined,
1478                "",
1479                self.current_point(),
1480            ));
1481        }
1482
1483        // The fetch begins on `first`, at `entry`. `pnt` and `c`, the point
1484        // a token starts at and the character under the cursor, follow the
1485        // cursor whenever a custom matcher or a check moves it and declines
1486        // (`resync`); `c` is `None` once the cursor is at the end.
1487        let entry = self.idx;
1488        let first = self.ch(entry);
1489        let mut pnt = self.current_point();
1490        let mut c = Some(first);
1491        let mut custom_index = 0;
1492
1493        if let Some(token) = self.run_custom_matchers(&mut custom_index, 1_000_000.0, &mut plugin) {
1494            return Ok(token);
1495        }
1496        self.resync(&mut pnt, &mut c);
1497
1498        // User-declared match tokens occupy the 1e6 matcher priority band.
1499        let match_skipped = if self.options.match_lex
1500            && (!self.options.match_values.is_empty()
1501                || self
1502                    .options
1503                    .match_tokens
1504                    .values()
1505                    .any(|matcher| self.wants(matcher.tin)))
1506        {
1507            match self.run_check(self.options.match_check.clone(), pnt) {
1508                CheckFlow::Continue => false,
1509                CheckFlow::Skip => true,
1510                CheckFlow::Token(token) => return Ok(*token),
1511            }
1512        } else {
1513            false
1514        };
1515        self.resync(&mut pnt, &mut c);
1516        let remaining = self.tail_at_byte(self.byte_position());
1517        let custom_value = (self.options.match_lex && !match_skipped && self.want.is_none())
1518            .then(|| {
1519                self.options
1520                    .match_values
1521                    .values()
1522                    .find_map(|matcher| match &matcher.matcher {
1523                        MatchTokenMatcher::Callback(callback) => callback(remaining)
1524                            .filter(|result| {
1525                                !result.source.is_empty() && remaining.starts_with(&result.source)
1526                            })
1527                            .map(|result| (result.source, result.value)),
1528                        MatchTokenMatcher::Regex(regex) => {
1529                            let captures = regex.captures(remaining)?;
1530                            let found = captures
1531                                .get(0)
1532                                .filter(|found| found.start() == 0 && !found.as_str().is_empty())?;
1533                            let source = found.as_str().to_string();
1534                            let value = matcher.transform.as_ref().map_or_else(
1535                                || {
1536                                    matcher
1537                                        .val
1538                                        .clone()
1539                                        .unwrap_or_else(|| Value::String(source.clone()))
1540                                },
1541                                |transform| {
1542                                    let groups = captures
1543                                        .iter()
1544                                        .map(|capture| {
1545                                            capture.map_or_else(String::new, |value| {
1546                                                value.as_str().into()
1547                                            })
1548                                        })
1549                                        .collect::<Vec<_>>();
1550                                    transform(&groups)
1551                                },
1552                            );
1553                            Some((source, value))
1554                        }
1555                    })
1556            })
1557            .flatten();
1558        if let Some((source, value)) = custom_value {
1559            for _ in source.chars() {
1560                self.advance();
1561            }
1562            return Ok(Token::new("#VL", TIN_VL, value, source, pnt));
1563        }
1564
1565        let remaining = self.tail_at_byte(self.byte_position());
1566        // With no custom matcher there is nothing for the band to do: both
1567        // passes walk an empty table and yield nothing, and `fix_len` is
1568        // read only by that walk. Most grammars register none, and every
1569        // token fetch of theirs paid the eager pass's scan of the fixed
1570        // table (one closure call per fixed literal) to arrive at the
1571        // `None` this guard now hands over directly. TS `makeMatchMatcher`
1572        // returns null on an empty table (ts/src/lexer.ts) and the band is
1573        // never installed; Go reaches the same place by defaulting
1574        // `MatchLex` off unless `Options.Match` is set. Rust defaults
1575        // `match_lex` true as TS does, so the guard is the parity.
1576        let custom = (self.options.match_lex
1577            && !match_skipped
1578            && !self.options.match_tokens.is_empty())
1579        .then(|| {
1580            // Two passes, position-expected before eager, as go/lexer.go
1581            // matchMatch and ts/src/lexer.ts makeMatchMatcher both make.
1582            // One tin-ordered pass in which eagerness merely bypassed the
1583            // slot gate let an eager matcher EARLIER in tin order win over
1584            // an expected one later: with `p = %x31-39` beside
1585            // `d = %x30-39`, the `2` of `12` lexed as the narrower class
1586            // the `*d` loop never asked for. Eagerness is for firing where
1587            // the slot's list is narrower than the grammar, never for
1588            // outbidding what the slot names.
1589            //
1590            // Under a want the alternate's own tin list is the sharper
1591            // gate, so one filtered pass is the whole search. With no
1592            // expected list at all (a standalone lexer, no rule) nothing
1593            // constrains the caller and every matcher is eligible in the
1594            // first pass.
1595            // The longest FIXED literal this slot expects that matches
1596            // here, or 0. Only the eager pass consults it: there, a
1597            // literal the slot names beats an eager-only matcher that
1598            // cuts no further than it does. Without this, a character
1599            // class that CONTAINS a literal the grammar also uses
1600            // swallows it wherever the class is eager (`num = "0" /
1601            // posdigit *digit` beside `digit = %x30-39` rejected
1602            // `0.0.0`). LENGTH decides, not mere existence, so a keyword
1603            // literal cannot truncate a longer word: ties go to the
1604            // literal, and an eager matcher that cuts further still
1605            // wins. TS and Go do the same, in makeMatchMatcher and
1606            // matchMatch.
1607            //
1608            // Computed once per fetch and only when a regex matcher in the
1609            // eager pass has something to weigh against it, as TS
1610            // `expectedFixedLen` does (`fixLen = -1` until asked). The
1611            // scan is the whole fixed table against the slot's list; an
1612            // expected matcher that wins in pass 0, or a fetch under a
1613            // want, never needs it. Nothing the scan reads changes
1614            // between the two passes, so lazy equals eager.
1615            let mut fix_len: Option<usize> = None;
1616            let compute_fix_len = || {
1617                if self.want.is_none() && self.options.fixed.lex {
1618                    expected_match_tins.map_or(0, |expected| {
1619                        self.options
1620                            .fixed
1621                            .tokens
1622                            .values()
1623                            .filter(|token| {
1624                                !token.source.is_empty()
1625                                    && expected.contains(&token.tin)
1626                                    && remaining.starts_with(&token.source)
1627                            })
1628                            .map(|token| token.source.len())
1629                            .max()
1630                            .unwrap_or(0)
1631                    })
1632                } else {
1633                    0
1634                }
1635            };
1636            let passes = if self.want.is_some() { 1 } else { 2 };
1637            (0..passes).find_map(|pass| {
1638                self.options.match_tokens.values().find_map(|matcher| {
1639                    if !self.wants(matcher.tin) {
1640                        return None;
1641                    }
1642                    if self.want.is_none() {
1643                        let expected = expected_match_tins
1644                            .is_none_or(|expected| expected.contains(&matcher.tin));
1645                        if pass == 0 {
1646                            if !expected {
1647                                return None;
1648                            }
1649                        } else if expected || !matcher.eager {
1650                            return None;
1651                        }
1652                    }
1653                    let result = match &matcher.matcher {
1654                        MatchTokenMatcher::Regex(regex) => regex
1655                            .find(remaining)
1656                            .filter(|found| found.start() == 0)
1657                            // The eager pass yields to an expected
1658                            // literal it cannot out-cut; the fixed
1659                            // matcher (2e6) runs next and takes it. See
1660                            // `fix_len` above.
1661                            .filter(|found| {
1662                                pass == 0 || {
1663                                    let fix_len = *fix_len.get_or_insert_with(compute_fix_len);
1664                                    fix_len == 0 || found.len() > fix_len
1665                                }
1666                            })
1667                            .map(|found| {
1668                                let source = found.as_str().to_string();
1669                                (source.clone(), Value::String(source))
1670                            }),
1671                        MatchTokenMatcher::Callback(callback) => callback(remaining)
1672                            .filter(|result| {
1673                                !result.source.is_empty() && remaining.starts_with(&result.source)
1674                            })
1675                            .map(|result| (result.source, result.value)),
1676                    };
1677                    result.map(|(source, value)| (matcher.name.clone(), matcher.tin, source, value))
1678                })
1679            })
1680        });
1681        if let Some(Some((name, tin, matched, value))) = custom {
1682            for _ in matched.chars() {
1683                self.advance();
1684            }
1685            return Ok(Token::new(name, tin, value, matched, pnt));
1686        }
1687
1688        if let Some(token) = self.run_custom_matchers(&mut custom_index, 2_000_000.0, &mut plugin) {
1689            return Ok(token);
1690        }
1691        self.resync(&mut pnt, &mut c);
1692
1693        // Fixed literals occupy the 2e6 band and use longest-match wins.
1694        let fixed_skipped = if self.options.fixed.lex {
1695            match self.run_check(self.options.fixed.check.clone(), pnt) {
1696                CheckFlow::Continue => false,
1697                CheckFlow::Skip => true,
1698                CheckFlow::Token(token) => return Ok(*token),
1699            }
1700        } else {
1701            false
1702        };
1703        self.resync(&mut pnt, &mut c);
1704        let fixed_skipped = fixed_skipped
1705            || !self.listed(entry, first, self.options.fixed.check.is_some(), |ch| {
1706                self.options
1707                    .fixed
1708                    .tokens
1709                    .values()
1710                    .any(|token| token.source.starts_with(ch))
1711            });
1712        let remaining = self.tail_at_byte(self.byte_position());
1713        // The winner is carried out of the table as its position, not as a
1714        // copy of its text. `Token::new` takes the name and the source text
1715        // by reference and stores both inline, so the only owned copy the
1716        // token needs is the one inside `Value::String`. Naming the match
1717        // as three owned values cost three `String` allocations per fixed
1718        // token, two of them freed again before the token was built.
1719        // The first byte decides almost every entry. Asking `wants` and then
1720        // `starts_with` of each fixed token in turn ran a tin lookup and a
1721        // `memcmp` per token in the grammar per token in the input, and a
1722        // grammar with fifty fixed tokens pays fifty of each to reject
1723        // forty-nine. One byte answers the same question, and an empty
1724        // source is kept out by its own check, which only entries that
1725        // already matched the byte ever reach.
1726        let first_byte = remaining.as_bytes().first().copied();
1727        if self.options.fixed.lex && !fixed_skipped && self.window_open() {
1728            // A literal longer than what is left of a window, which what is
1729            // left starts, may yet be there once the window has grown: the
1730            // read meets the window's end, whatever this matcher decides,
1731            // so that a later matcher's fill cannot let the cut stand on it
1732            // (#326).
1733            if self.options.fixed.tokens.values().any(|token| {
1734                remaining.len() < token.source.len() && token.source.starts_with(remaining)
1735            }) {
1736                self.touch_end();
1737            }
1738        }
1739        let fixed = (self.options.fixed.lex && !fixed_skipped)
1740            .then(|| {
1741                self.options
1742                    .fixed
1743                    .tokens
1744                    .values()
1745                    .enumerate()
1746                    .filter(|(_, token)| {
1747                        token.source.as_bytes().first().copied() == first_byte
1748                            && !token.source.is_empty()
1749                            && self.wants(token.tin)
1750                            && remaining.starts_with(&token.source)
1751                    })
1752                    .max_by_key(|(_, token)| token.source.len())
1753                    .map(|(index, token)| (index, token.source.chars().count()))
1754            })
1755            .flatten();
1756        if let Some((index, source_chars)) = fixed {
1757            for _ in 0..source_chars {
1758                self.advance();
1759            }
1760            let (_, token) = self
1761                .options
1762                .fixed
1763                .tokens
1764                .get_index(index)
1765                .expect("index came from this table, which nothing writes to mid-parse");
1766            // A fixed token has no value, as in TypeScript and Go: its
1767            // text is its `src`.
1768            return Ok(Token::new(
1769                &token.name,
1770                token.tin,
1771                Value::Undefined,
1772                token.source.as_str(),
1773                pnt,
1774            ));
1775        }
1776
1777        if let Some(token) = self.run_custom_matchers(&mut custom_index, 3_000_000.0, &mut plugin) {
1778            return Ok(token);
1779        }
1780        self.resync(&mut pnt, &mut c);
1781
1782        // 1. Whitespace
1783        let space_skipped = if self.options.space.lex && self.wants(TIN_SP) {
1784            match self.run_check(self.options.space.check.clone(), pnt) {
1785                CheckFlow::Continue => false,
1786                CheckFlow::Skip => true,
1787                CheckFlow::Token(token) => return Ok(*token),
1788            }
1789        } else {
1790            false
1791        };
1792        self.resync(&mut pnt, &mut c);
1793        if self.options.space.lex
1794            && !space_skipped
1795            && self.wants(TIN_SP)
1796            && c.is_some_and(|ch| self.char_sets.space.contains(ch))
1797            && self.listed(entry, first, self.options.space.check.is_some(), |ch| {
1798                self.char_sets.space.contains(ch)
1799            })
1800        {
1801            let mut src = String::new();
1802            while let Some(ch) = self.peek() {
1803                if self.char_sets.space.contains(ch) {
1804                    src.push(ch);
1805                    self.advance();
1806                } else {
1807                    break;
1808                }
1809            }
1810            return Ok(Token::new(
1811                "#SP",
1812                TIN_SP,
1813                Value::String(src.clone()),
1814                src,
1815                pnt,
1816            ));
1817        }
1818
1819        if let Some(token) = self.run_custom_matchers(&mut custom_index, 4_000_000.0, &mut plugin) {
1820            return Ok(token);
1821        }
1822        self.resync(&mut pnt, &mut c);
1823
1824        // 2. Line ending
1825        let line_skipped = if self.options.line.lex && self.wants(TIN_LN) {
1826            match self.run_check(self.options.line.check.clone(), pnt) {
1827                CheckFlow::Continue => false,
1828                CheckFlow::Skip => true,
1829                CheckFlow::Token(token) => return Ok(*token),
1830            }
1831        } else {
1832            false
1833        };
1834        self.resync(&mut pnt, &mut c);
1835        let line_skipped = line_skipped
1836            || !self.listed(entry, first, self.options.line.check.is_some(), |ch| {
1837                self.char_sets.line_ends.contains(ch)
1838            });
1839        if self.options.line.lex
1840            && !line_skipped
1841            && self.wants(TIN_LN)
1842            && c.is_some_and(|ch| self.char_sets.line_ends.contains(ch))
1843        {
1844            let mut src = String::new();
1845            let mut seen = std::collections::HashSet::new();
1846            while let Some(ch) = self.peek() {
1847                if !self.char_sets.line_ends.contains(ch) {
1848                    break;
1849                }
1850                if self.options.line.single && !seen.insert(ch) {
1851                    break;
1852                }
1853                src.push(self.advance().expect("peeked character must advance"));
1854            }
1855            self.ci = 1;
1856            return Ok(Token::new(
1857                "#LN",
1858                TIN_LN,
1859                Value::String(src.clone()),
1860                src,
1861                pnt,
1862            ));
1863        }
1864
1865        if let Some(bad_char @ ('\u{2028}' | '\u{2029}')) = c {
1866            if self.options.line.lex && !line_skipped && self.wants(TIN_LN) {
1867                self.advance();
1868                let err = TabnasError::new(
1869                    "unexpected",
1870                    bad_char.to_string(),
1871                    "",
1872                    pnt.site.pos,
1873                    pnt.site.ri,
1874                    pnt.site.ci,
1875                );
1876                self.err = Some(err.clone());
1877                return Err(Box::new(err));
1878            }
1879        }
1880
1881        if let Some(token) = self.run_custom_matchers(&mut custom_index, 5_000_000.0, &mut plugin) {
1882            return Ok(token);
1883        }
1884        self.resync(&mut pnt, &mut c);
1885
1886        // 3. Quoted strings. These precede comments in the canonical matcher
1887        // order, so an overlapping quote/comment opener is a string unless
1888        // string matching explicitly abandons the malformed candidate.
1889        let string_skipped = if self.options.string.lex && self.wants(TIN_ST) {
1890            match self.run_check(self.options.string.check.clone(), pnt) {
1891                CheckFlow::Continue => false,
1892                CheckFlow::Skip => true,
1893                CheckFlow::Token(token) => return Ok(*token),
1894            }
1895        } else {
1896            false
1897        };
1898        self.resync(&mut pnt, &mut c);
1899        let quote = c.filter(|ch| self.char_sets.string.contains(*ch));
1900        if let Some(quote) = quote.filter(|_| {
1901            self.options.string.lex
1902                && !string_skipped
1903                && self.wants(TIN_ST)
1904                && self.listed(entry, first, self.options.string.check.is_some(), |ch| {
1905                    self.char_sets.string.contains(ch)
1906                })
1907        }) {
1908            let start = (self.idx, self.ri, self.ci);
1909            match self.match_string(quote, pnt) {
1910                result @ Ok(_) => return result,
1911                Err(error) if !self.options.string.abandon => return Err(error),
1912                Err(_) => {
1913                    (self.idx, self.ri, self.ci) = start;
1914                    self.err = None;
1915                }
1916            }
1917        }
1918
1919        if let Some(token) = self.run_custom_matchers(&mut custom_index, 6_000_000.0, &mut plugin) {
1920            return Ok(token);
1921        }
1922        self.resync(&mut pnt, &mut c);
1923
1924        // 4. Comments (longest opening marker wins; ties sort by name).
1925        let comment_skipped = if self.options.comment.lex && self.wants(TIN_CM) {
1926            match self.run_check(self.options.comment.check.clone(), pnt) {
1927                CheckFlow::Continue => false,
1928                CheckFlow::Skip => true,
1929                CheckFlow::Token(token) => return Ok(*token),
1930            }
1931        } else {
1932            false
1933        };
1934        self.resync(&mut pnt, &mut c);
1935        if self.options.comment.lex
1936            && !comment_skipped
1937            && self.wants(TIN_CM)
1938            && self.listed(entry, first, self.options.comment.check.is_some(), |ch| {
1939                self.options
1940                    .comment
1941                    .definitions
1942                    .values()
1943                    .any(|definition| definition.start.starts_with(ch))
1944            })
1945        {
1946            if let Some(token) = self.match_comment(pnt)? {
1947                return Ok(token);
1948            }
1949        }
1950
1951        if let Some(token) = self.run_custom_matchers(&mut custom_index, 7_000_000.0, &mut plugin) {
1952            return Ok(token);
1953        }
1954        self.resync(&mut pnt, &mut c);
1955
1956        // 5. Numbers
1957        let number_skipped = if self.options.number.lex && self.wants(TIN_NR) {
1958            match self.run_check(self.options.number.check.clone(), pnt) {
1959                CheckFlow::Continue => false,
1960                CheckFlow::Skip => true,
1961                CheckFlow::Token(token) => return Ok(*token),
1962            }
1963        } else {
1964            false
1965        };
1966        self.resync(&mut pnt, &mut c);
1967        let could_start_number =
1968            |ch: char| ch == '-' || ch == '+' || ch == '.' || ch.is_ascii_digit();
1969        if self.options.number.lex
1970            && !number_skipped
1971            && self.wants(TIN_NR)
1972            && c.is_some_and(could_start_number)
1973            && self.listed(
1974                entry,
1975                first,
1976                self.options.number.check.is_some(),
1977                could_start_number,
1978            )
1979        {
1980            if let Some(tkn) = self.match_number(pnt)? {
1981                return Ok(tkn);
1982            }
1983        }
1984
1985        if let Some(token) = self.run_custom_matchers(&mut custom_index, 8_000_000.0, &mut plugin) {
1986            return Ok(token);
1987        }
1988        self.resync(&mut pnt, &mut c);
1989
1990        // 6. Text and named/regex values share the same delimited run.
1991        // Negotiated lexing gates this combined family by its primary token
1992        // identity (#TX), matching the TypeScript and Go dispatchers. Once
1993        // entered, an exact or regexp value definition may still produce
1994        // #VL; the caller rejects and rolls that cut back when #VL was not
1995        // requested.
1996        let text_matcher_wanted = self.wants(TIN_TX);
1997        let value_lex = self.options.value.lex && text_matcher_wanted;
1998        let text_lex = self.options.text.lex && text_matcher_wanted;
1999        let text_skipped = if text_lex || value_lex {
2000            match self.run_check(self.options.text.check.clone(), pnt) {
2001                CheckFlow::Continue => false,
2002                CheckFlow::Skip => true,
2003                CheckFlow::Token(token) => return Ok(*token),
2004            }
2005        } else {
2006            false
2007        };
2008        self.resync(&mut pnt, &mut c);
2009        if (text_lex || value_lex) && !text_skipped && !self.is_text_delimiter_here() {
2010            let start = (self.idx, self.ri, self.ci);
2011            // Only a `value` definition declaring `consume` looks at the
2012            // rest of the document, and the JSON grammar has none: the
2013            // tail is sliced where one asks for it, from the offset the
2014            // token started at, never copied.
2015            let start_byte = self.byte_position();
2016            let mut src = String::new();
2017            while let Some(ch) = self.peek() {
2018                if self.is_text_delimiter_here() {
2019                    break;
2020                }
2021                src.push(ch);
2022                self.advance();
2023            }
2024
2025            let mut output = None;
2026            if value_lex {
2027                if let Some(definition) = self
2028                    .options
2029                    .value
2030                    .definitions
2031                    .get(&src)
2032                    .filter(|definition| definition.matcher.is_none())
2033                    .cloned()
2034                {
2035                    output = Some(Token::new(
2036                        "#VL",
2037                        TIN_VL,
2038                        definition
2039                            .val
2040                            .clone()
2041                            .unwrap_or_else(|| Value::String(src.clone())),
2042                        src.clone(),
2043                        pnt,
2044                    ));
2045                }
2046
2047                if output.is_none() {
2048                    let mut definitions: Vec<_> = self
2049                        .options
2050                        .value
2051                        .definitions
2052                        .iter()
2053                        .filter(|(_, definition)| definition.matcher.is_some())
2054                        .map(|(name, definition)| (name.clone(), definition.clone()))
2055                        .collect();
2056                    definitions.sort_by(|(name_a, _), (name_b, _)| name_a.cmp(name_b));
2057                    for (_, definition) in definitions {
2058                        let regex = definition.matcher.as_ref().expect("filtered matcher");
2059                        let target: &str = if definition.consume {
2060                            self.tail_at_byte(start_byte)
2061                        } else {
2062                            &src
2063                        };
2064                        let Some(captures) = regex.captures(target) else {
2065                            continue;
2066                        };
2067                        let Some(found) = captures.get(0).filter(|found| found.start() == 0) else {
2068                            continue;
2069                        };
2070                        if !definition.consume && found.end() != target.len() {
2071                            continue;
2072                        }
2073                        let matched = found.as_str().to_string();
2074                        let value = definition.transform.as_ref().map_or_else(
2075                            || {
2076                                definition
2077                                    .val
2078                                    .clone()
2079                                    .unwrap_or_else(|| Value::String(matched.clone()))
2080                            },
2081                            |transform| {
2082                                let groups = captures
2083                                    .iter()
2084                                    .map(|capture| {
2085                                        capture
2086                                            .map_or_else(String::new, |value| value.as_str().into())
2087                                    })
2088                                    .collect::<Vec<_>>();
2089                                transform(&groups)
2090                            },
2091                        );
2092                        if definition.consume {
2093                            (self.idx, self.ri, self.ci) = start;
2094                            for _ in matched.chars() {
2095                                self.advance();
2096                            }
2097                        }
2098                        output = Some(Token::new("#VL", TIN_VL, value, matched, pnt));
2099                        break;
2100                    }
2101                }
2102            }
2103
2104            if output.is_none() && (!text_lex || text_skipped) {
2105                (self.idx, self.ri, self.ci) = start;
2106            } else if output.is_none() {
2107                output = Some(Token::new(
2108                    "#TX",
2109                    TIN_TX,
2110                    Value::String(src.clone()),
2111                    src,
2112                    pnt,
2113                ));
2114            }
2115
2116            if let Some(mut token) = output {
2117                let value = std::mem::replace(&mut token.val, Value::Undefined);
2118                token.val = self.modify_text_value(value, &mut plugin);
2119                return Ok(token);
2120            }
2121        }
2122
2123        if let Some(token) = self.run_custom_matchers(&mut custom_index, f64::INFINITY, &mut plugin)
2124        {
2125            return Ok(token);
2126        }
2127        self.resync(&mut pnt, &mut c);
2128
2129        // 7. Unclaimed character -> Error: unexpected, raised where the
2130        // cursor stands, as TypeScript's `Lex.next` builds its #BD at the
2131        // live `pnt` and Go's `nextUnfiltered2` at `l.pnt`. A matcher that
2132        // stepped over the last character and declined leaves no character
2133        // to name: the error then names none, at the end of the source, as
2134        // it does in both. This took a character unconditionally, and so
2135        // panicked on a lone byte-order mark under the xml plugin, whose
2136        // matcher steps over a mark at the start of the source.
2137        let bad_source = self.advance().map_or_else(String::new, String::from);
2138        let err = TabnasError::new(
2139            "unexpected",
2140            bad_source,
2141            "",
2142            pnt.site.pos,
2143            pnt.site.ri,
2144            pnt.site.ci,
2145        );
2146        self.err = Some(err.clone());
2147        Err(Box::new(err))
2148    }
2149
2150    fn match_comment(&mut self, pnt: Point) -> LexResult<Option<Token>> {
2151        let remaining = self.tail_at_byte(self.byte_position());
2152        if self.window_open()
2153            && self.options.comment.definitions.values().any(|definition| {
2154                definition.lex
2155                    && remaining.len() < definition.start.len()
2156                    && definition.start.starts_with(remaining)
2157            })
2158        {
2159            // A marker longer than what is left of a window, which what is
2160            // left starts, may yet start a comment once the window has
2161            // grown (#326).
2162            self.touch_end();
2163        }
2164        let mut definitions: Vec<_> = self
2165            .options
2166            .comment
2167            .definitions
2168            .iter()
2169            .filter(|(_, definition)| {
2170                // Same first-byte test as the fixed-token scan above.
2171                definition.start.as_bytes().first().copied()
2172                    == remaining.as_bytes().first().copied()
2173                    && !definition.start.is_empty()
2174                    && definition.lex
2175                    && remaining.starts_with(&definition.start)
2176            })
2177            .collect();
2178        definitions.sort_by(|(name_a, a), (name_b, b)| {
2179            b.start
2180                .len()
2181                .cmp(&a.start.len())
2182                .then_with(|| name_a.cmp(name_b))
2183        });
2184        let Some((_, definition)) = definitions.first() else {
2185            return Ok(None);
2186        };
2187        let definition = (*definition).clone();
2188        let mut src = String::new();
2189        for _ in definition.start.chars() {
2190            src.push(self.advance().expect("comment marker must advance"));
2191        }
2192
2193        let mut terminated_by_suffix = false;
2194        let mut closed = definition.line;
2195        loop {
2196            let at = self.byte_position();
2197            let remainder = self.tail_at_byte(at);
2198            let suffix = definition
2199                .suffixes
2200                .iter()
2201                .filter(|suffix| !suffix.is_empty() && remainder.starts_with(*suffix))
2202                .max_by_key(|suffix| suffix.len())
2203                .cloned();
2204            let suffix = match (suffix, definition.suffix_matcher.as_ref()) {
2205                (Some(suffix), _) => Some(suffix),
2206                (None, None) => None,
2207                (None, Some(matcher)) => match matcher.run(self.tail_at_byte(at)) {
2208                    Some(effect) => Some(effect),
2209                    None => {
2210                        let saved = self.state();
2211                        let wanted = self.want.clone();
2212                        let token = matcher.run_imperative(self);
2213                        self.restore(saved);
2214                        self.want = wanted;
2215                        token.map(|token| token.src.to_string())
2216                    }
2217                },
2218            };
2219            let remainder = self.tail_at_byte(self.byte_position());
2220            let suffix =
2221                suffix.filter(|suffix| !suffix.is_empty() && remainder.starts_with(suffix));
2222            if let Some(suffix) = suffix {
2223                for _ in suffix.chars() {
2224                    src.push(self.advance().expect("comment suffix must advance"));
2225                }
2226                terminated_by_suffix = true;
2227                closed = true;
2228                break;
2229            }
2230            if !definition.line
2231                && !definition.end.is_empty()
2232                && remainder.starts_with(&definition.end)
2233            {
2234                for _ in definition.end.chars() {
2235                    src.push(self.advance().expect("comment end must advance"));
2236                }
2237                closed = true;
2238                break;
2239            }
2240            let Some(ch) = self.peek() else {
2241                break;
2242            };
2243            if definition.line && (self.char_sets.line_ends.contains(ch)) {
2244                break;
2245            }
2246            src.push(self.advance().expect("comment body must advance"));
2247        }
2248
2249        if !closed {
2250            let err = TabnasError::new(
2251                "unterminated_comment",
2252                src,
2253                "",
2254                pnt.site.pos,
2255                pnt.site.ri,
2256                pnt.site.ci,
2257            );
2258            self.err = Some(err.clone());
2259            return Err(Box::new(err));
2260        }
2261
2262        if definition.eat_line && !terminated_by_suffix {
2263            while let Some(ch) = self.peek() {
2264                if !self.char_sets.line_ends.contains(ch) {
2265                    break;
2266                }
2267                src.push(self.advance().expect("comment line tail must advance"));
2268            }
2269        }
2270
2271        Ok(Some(Token::new(
2272            "#CM",
2273            TIN_CM,
2274            Value::String(src.clone()),
2275            src,
2276            pnt,
2277        )))
2278    }
2279
2280    fn match_number(&mut self, pnt: Point) -> LexResult<Option<Token>> {
2281        let start_idx = self.idx;
2282        let mut src = String::new();
2283
2284        // Optional sign.
2285        if matches!(self.peek(), Some('-' | '+')) {
2286            src.push(self.advance().unwrap());
2287        }
2288
2289        // Base-prefixed integers are complete at the final valid digit.
2290        if self.peek() == Some('0') {
2291            if let Some(prefix) = self.peek_at(1) {
2292                let radix = match prefix {
2293                    'x' | 'X' if self.options.number.hex => Some(16),
2294                    'o' | 'O' if self.options.number.oct => Some(8),
2295                    'b' | 'B' if self.options.number.bin => Some(2),
2296                    _ => None,
2297                };
2298                if let Some(radix) = radix {
2299                    src.push(self.advance().expect("peeked zero"));
2300                    src.push(self.advance().expect("peeked base prefix"));
2301                    let mut saw_digit = false;
2302                    while let Some(ch) = self.peek() {
2303                        if ch.is_digit(radix) {
2304                            saw_digit = true;
2305                            src.push(self.advance().expect("peeked base digit"));
2306                        } else if self
2307                            .options
2308                            .number
2309                            .sep
2310                            .as_ref()
2311                            .is_some_and(|separator| separator.contains(ch))
2312                        {
2313                            src.push(self.advance().expect("peeked base digit"));
2314                        } else {
2315                            break;
2316                        }
2317                    }
2318                    if saw_digit && self.is_text_delimiter_here() {
2319                        if self
2320                            .exclude_regex
2321                            .as_ref()
2322                            .is_some_and(|regex| regex.is_match(&src))
2323                        {
2324                            self.reset_number(start_idx, pnt);
2325                            return Ok(None);
2326                        }
2327                        if self.options.value.lex {
2328                            if let Some(definition) = self
2329                                .options
2330                                .value
2331                                .definitions
2332                                .get(&src)
2333                                .filter(|definition| definition.matcher.is_none())
2334                            {
2335                                return Ok(Some(Token::new(
2336                                    "#VL",
2337                                    TIN_VL,
2338                                    definition
2339                                        .val
2340                                        .clone()
2341                                        .unwrap_or_else(|| Value::String(src.clone())),
2342                                    src,
2343                                    pnt,
2344                                )));
2345                            }
2346                        }
2347                        // The digits of the literal, prefix, sign and any
2348                        // separators removed, folded as they are read. The
2349                        // fold keeps a bounded head, a digit count and a
2350                        // sticky bit, so a literal of any length costs the
2351                        // same handful of bytes: buffering the digits
2352                        // instead would let one long token multiply the
2353                        // memory the source already holds.
2354                        let mut fold = DigitFold::new(radix.trailing_zeros());
2355                        for ch in src.chars().skip_while(|ch| matches!(ch, '-' | '+')).skip(2) {
2356                            if self
2357                                .options
2358                                .number
2359                                .sep
2360                                .as_ref()
2361                                .is_some_and(|separator| separator.contains(ch))
2362                            {
2363                                continue;
2364                            }
2365                            fold.push(ch.to_digit(radix).expect("validated base digit"));
2366                        }
2367                        let mut value = fold.finish();
2368                        if src.starts_with('-') {
2369                            value = -value;
2370                        }
2371                        return Ok(Some(Token::new(
2372                            "#NR",
2373                            TIN_NR,
2374                            Value::Number(value),
2375                            src,
2376                            pnt,
2377                        )));
2378                    }
2379                    self.reset_number(start_idx, pnt);
2380                    return Ok(None);
2381                }
2382            }
2383        }
2384
2385        let Some(ch) = self.peek() else {
2386            self.reset_number(start_idx, pnt);
2387            return Ok(None);
2388        };
2389        if ch == '.' {
2390            if !self.peek_at(1).is_some_and(|next| next.is_ascii_digit()) {
2391                self.reset_number(start_idx, pnt);
2392                return Ok(None);
2393            }
2394            src.push(self.advance().expect("peeked leading decimal point"));
2395        } else if !ch.is_ascii_digit() {
2396            self.reset_number(start_idx, pnt);
2397            return Ok(None);
2398        }
2399
2400        let (has_digits, edge_separator) = self.scan_number_digits(&mut src);
2401        if !has_digits || edge_separator {
2402            self.reset_number(start_idx, pnt);
2403            return Ok(None);
2404        }
2405
2406        // The canonical regexp admits a trailing decimal point and an
2407        // exponent after it (`2.e3`), but declines `0.a` as one text run.
2408        if self.peek() == Some('.') {
2409            let next = self.peek_at(1);
2410            let exponent_after_dot = matches!(next, Some('e' | 'E'))
2411                && match self.peek_at(2) {
2412                    Some('+' | '-') => self.peek_at(3).is_some_and(|ch| ch.is_ascii_digit()),
2413                    Some(ch) => ch.is_ascii_digit(),
2414                    None => false,
2415                };
2416            if next.is_some_and(|ch| ch.is_ascii_digit()) {
2417                src.push(self.advance().expect("peeked decimal point"));
2418                let (_, edge_separator) = self.scan_number_digits(&mut src);
2419                if edge_separator {
2420                    self.reset_number(start_idx, pnt);
2421                    return Ok(None);
2422                }
2423            } else if next.is_some()
2424                && !self.is_text_delimiter_at(self.idx + 1)
2425                && next != Some('.')
2426                && !exponent_after_dot
2427            {
2428                self.reset_number(start_idx, pnt);
2429                return Ok(None);
2430            } else {
2431                src.push(self.advance().expect("peeked trailing decimal point"));
2432            }
2433        }
2434
2435        if matches!(self.peek(), Some('e' | 'E')) {
2436            let exponent_start = self.idx;
2437            let source_len = src.len();
2438            src.push(self.advance().expect("peeked exponent marker"));
2439            if matches!(self.peek(), Some('+' | '-')) {
2440                src.push(self.advance().expect("peeked exponent sign"));
2441            }
2442            let (has_exponent_digits, edge_separator) = self.scan_number_digits(&mut src);
2443            if edge_separator {
2444                self.reset_number(start_idx, pnt);
2445                return Ok(None);
2446            }
2447            if !has_exponent_digits {
2448                self.idx = exponent_start;
2449                src.truncate(source_len);
2450            }
2451        }
2452
2453        if !self.is_text_delimiter_here() {
2454            self.reset_number(start_idx, pnt);
2455            return Ok(None);
2456        }
2457
2458        // Check exclusion regex (e.g. ^00+)
2459        if let Some(ref re) = self.exclude_regex {
2460            if re.is_match(&src) {
2461                // Number is excluded, backtrack
2462                self.reset_number(start_idx, pnt);
2463                return Ok(None);
2464            }
2465        }
2466
2467        if self.options.value.lex {
2468            if let Some(definition) = self
2469                .options
2470                .value
2471                .definitions
2472                .get(&src)
2473                .filter(|definition| definition.matcher.is_none())
2474            {
2475                return Ok(Some(Token::new(
2476                    "#VL",
2477                    TIN_VL,
2478                    definition
2479                        .val
2480                        .clone()
2481                        .unwrap_or_else(|| Value::String(src.clone())),
2482                    src,
2483                    pnt,
2484                )));
2485            }
2486        }
2487
2488        // Parse float
2489        let parse_src = self.options.number.sep.as_ref().map_or_else(
2490            || src.clone(),
2491            |separator| src.chars().filter(|ch| !separator.contains(*ch)).collect(),
2492        );
2493        match parse_src.parse::<f64>() {
2494            Ok(num) => Ok(Some(Token::new(
2495                "#NR",
2496                TIN_NR,
2497                Value::Number(num),
2498                src,
2499                pnt,
2500            ))),
2501            Err(_) => {
2502                self.reset_number(start_idx, pnt);
2503                Ok(None)
2504            }
2505        }
2506    }
2507
2508    fn reset_number(&mut self, start_idx: usize, pnt: Point) {
2509        self.idx = start_idx;
2510        self.ri = pnt.site.ri;
2511        self.ci = pnt.site.ci;
2512    }
2513
2514    /// Consume a decimal digit/separator run. Separators are legal only
2515    /// between digits; a leading or trailing separator makes the whole run
2516    /// fall through to text, matching the TypeScript regexp and Go scanner.
2517    fn scan_number_digits(&mut self, src: &mut String) -> (bool, bool) {
2518        // The run is measured before any of it is consumed. Advancing
2519        // as it goes would hold `&mut self` across a read of
2520        // `self.options.number.sep`, and the way that used to be settled
2521        // was to clone the separator — an allocation and a free for
2522        // every number in the input, for a value that cannot change
2523        // while one number is being scanned.
2524        let run_start = self.idx;
2525        let mut saw_digit = false;
2526        let mut last_was_separator = false;
2527        let mut end = run_start;
2528        {
2529            let separator = self.options.number.sep.as_deref();
2530            while let Some(ch) = self.ch_get(end) {
2531                if ch.is_ascii_digit() {
2532                    saw_digit = true;
2533                    last_was_separator = false;
2534                } else if separator.is_some_and(|separator| separator.contains(ch)) {
2535                    last_was_separator = true;
2536                } else {
2537                    break;
2538                }
2539                end += 1;
2540            }
2541        }
2542        while self.idx < end {
2543            src.push(self.advance().expect("scanned number character"));
2544        }
2545        let starts_with_separator = self.idx > run_start
2546            && self.options.number.sep.as_deref().is_some_and(|separator| {
2547                self.chars[run_start - self.low..self.idx - self.low]
2548                    .first()
2549                    .is_some_and(|ch| separator.contains(*ch))
2550            });
2551        (saw_digit, starts_with_separator || last_was_separator)
2552    }
2553
2554    fn match_string(&mut self, quote: char, pnt: Point) -> LexResult<Token> {
2555        let quote_char = self.advance().unwrap();
2556        let mut out_str = String::new();
2557        let mut raw_src = String::new();
2558        raw_src.push(quote_char);
2559
2560        let mut pending_high_surrogate: Option<u16> = None;
2561
2562        // The body classes of TypeScript's `buildStringBodySpec`
2563        // (ts/src/lexer.ts), which Go's `BuildStringBodySpec` ports: a
2564        // line character is LINE or LINE+ROW only inside a multi-line
2565        // string, where it resets the column and, in `line.rowChars`,
2566        // counts a row; anywhere else in a body it is plain content,
2567        // counted as a column, unless it is a control character, which
2568        // stops the body as `unprintable`. This loop used to count a row
2569        // for any row character it stepped over, string body or not, and
2570        // to refuse any line character inside a single-line string: with
2571        // json5's U+2028 and U+2029 as row characters, `y` in
2572        // `"a<U+2028>b" y` sat on row 2 here and on row 1 in TypeScript
2573        // and Go, and with the two in `line.chars` as well the string was
2574        // `unprintable` (tabnas/parser#263).
2575        let multi_line = self.options.string.multi_chars.contains(quote);
2576
2577        while let Some(c) = self.peek() {
2578            if c == quote {
2579                raw_src.push(self.advance().unwrap());
2580                // Rust strings cannot represent a lone UTF-16 surrogate, so
2581                // preserve the Go-port behavior and fold it to U+FFFD.
2582                self.flush_surrogate(&mut pending_high_surrogate, &mut out_str);
2583                return Ok(Token::new(
2584                    "#ST",
2585                    TIN_ST,
2586                    Value::String(out_str),
2587                    raw_src,
2588                    pnt,
2589                ));
2590            }
2591
2592            if let Some(replacement) = self.options.string.replace.get(&c).cloned() {
2593                raw_src.push(self.advance().expect("peeked character must advance"));
2594                self.flush_surrogate(&mut pending_high_surrogate, &mut out_str);
2595                out_str.push_str(&replacement);
2596                continue;
2597            }
2598
2599            if multi_line && self.char_sets.line.contains(c) {
2600                raw_src.push(
2601                    self.advance_string_line()
2602                        .expect("peeked character must advance"),
2603                );
2604                out_str.push(c);
2605                continue;
2606            }
2607
2608            // A control character stops the body: a line character
2609            // always, since a single-line string cannot hold one, and any
2610            // other unless `string.allowControl` admits it. Sited ON the
2611            // character, as TypeScript does (`pnt.sI = sI; pnt.cI = cI`
2612            // before its `bad()` call, ts/src/lexer.ts). `pnt` is the
2613            // opening quote, and reporting that put every embedded
2614            // newline at the start of its string.
2615            if (c as u32) < 32
2616                && (self.char_sets.line.contains(c) || !self.options.string.allow_control)
2617            {
2618                let site = self.current_point().site;
2619                let err =
2620                    TabnasError::new("unprintable", c.to_string(), "", site.pos, site.ri, site.ci);
2621                self.err = Some(err.clone());
2622                return Err(Box::new(err));
2623            }
2624
2625            if c == self.options.string.escape_char {
2626                raw_src.push(self.advance().unwrap());
2627                let esc_point = self.current_point();
2628                if let Some(esc) = self.advance() {
2629                    raw_src.push(esc);
2630                    if let Some(replacement) = self.options.string.escape.get(&esc).cloned() {
2631                        self.flush_surrogate(&mut pending_high_surrogate, &mut out_str);
2632                        out_str.push_str(&replacement);
2633                        continue;
2634                    }
2635                    match esc {
2636                        'u' => {
2637                            // Unicode escape: \uXXXX or \u{X...}. An
2638                            // invalid one is reported on the backslash
2639                            // with the span TypeScript cuts: six source
2640                            // characters for the fixed-width form, four
2641                            // for `\x`, and through the closing brace
2642                            // (or to the end of the source) for the
2643                            // braced form -- clipped, never padded, so a
2644                            // truncated escape at end of input reports
2645                            // exactly the characters that are there.
2646                            if self.peek() == Some('{') && !self.options.string.escape_strict {
2647                                raw_src.push(self.advance().unwrap()); // '{'
2648                                let mut hex = String::new();
2649                                let mut closed = false;
2650                                while let Some(h) = self.peek() {
2651                                    if h == '}' {
2652                                        raw_src.push(self.advance().unwrap());
2653                                        closed = true;
2654                                        break;
2655                                    }
2656                                    raw_src.push(self.advance().unwrap());
2657                                    hex.push(h);
2658                                }
2659
2660                                if !closed
2661                                    || hex.is_empty()
2662                                    || hex.len() > 6
2663                                    || !hex.chars().all(|ch| ch.is_ascii_hexdigit())
2664                                {
2665                                    let err = TabnasError::new(
2666                                        "invalid_unicode",
2667                                        self.source_span(esc_point.site.pos - 1, self.idx),
2668                                        "",
2669                                        esc_point.site.pos - 1,
2670                                        esc_point.site.ri,
2671                                        esc_point.site.ci - 1,
2672                                    );
2673                                    self.err = Some(err.clone());
2674                                    return Err(Box::new(err));
2675                                }
2676
2677                                let cp = match u32::from_str_radix(&hex, 16) {
2678                                    Ok(val) if val <= 0x10FFFF => val,
2679                                    _ => {
2680                                        let err = TabnasError::new(
2681                                            "invalid_unicode",
2682                                            self.source_span(esc_point.site.pos - 1, self.idx),
2683                                            "",
2684                                            esc_point.site.pos - 1,
2685                                            esc_point.site.ri,
2686                                            esc_point.site.ci - 1,
2687                                        );
2688                                        self.err = Some(err.clone());
2689                                        return Err(Box::new(err));
2690                                    }
2691                                };
2692
2693                                self.emit_unicode_escape(
2694                                    cp,
2695                                    &mut pending_high_surrogate,
2696                                    &mut out_str,
2697                                );
2698                            } else {
2699                                // Exactly 4 hex digits: \uXXXX
2700                                let mut hex = String::new();
2701                                for _ in 0..4 {
2702                                    if let Some(h) = self.peek() {
2703                                        if h.is_ascii_hexdigit() {
2704                                            raw_src.push(self.advance().unwrap());
2705                                            hex.push(h);
2706                                        } else {
2707                                            break;
2708                                        }
2709                                    } else {
2710                                        break;
2711                                    }
2712                                }
2713
2714                                if hex.len() != 4 {
2715                                    let err = TabnasError::new(
2716                                        "invalid_unicode",
2717                                        self.source_span(
2718                                            esc_point.site.pos - 1,
2719                                            esc_point.site.pos + 5,
2720                                        ),
2721                                        "",
2722                                        esc_point.site.pos - 1,
2723                                        esc_point.site.ri,
2724                                        esc_point.site.ci - 1,
2725                                    );
2726                                    self.err = Some(err.clone());
2727                                    return Err(Box::new(err));
2728                                }
2729
2730                                let cp = u16::from_str_radix(&hex, 16).map_err(|_| {
2731                                    let err = TabnasError::new(
2732                                        "invalid_unicode",
2733                                        self.source_span(
2734                                            esc_point.site.pos - 1,
2735                                            esc_point.site.pos + 5,
2736                                        ),
2737                                        "",
2738                                        esc_point.site.pos - 1,
2739                                        esc_point.site.ri,
2740                                        esc_point.site.ci - 1,
2741                                    );
2742                                    self.err = Some(err.clone());
2743                                    err
2744                                })?;
2745
2746                                self.emit_unicode_escape(
2747                                    u32::from(cp),
2748                                    &mut pending_high_surrogate,
2749                                    &mut out_str,
2750                                );
2751                            }
2752                        }
2753                        'x' if !self.options.string.escape_strict => {
2754                            let mut hex = String::new();
2755                            for _ in 0..2 {
2756                                if let Some(h) = self.peek() {
2757                                    if h.is_ascii_hexdigit() {
2758                                        raw_src.push(
2759                                            self.advance().expect("peeked character must advance"),
2760                                        );
2761                                        hex.push(h);
2762                                    }
2763                                }
2764                            }
2765                            if hex.len() != 2 {
2766                                let err = TabnasError::new(
2767                                    "invalid_ascii",
2768                                    self.source_span(
2769                                        esc_point.site.pos - 1,
2770                                        esc_point.site.pos + 3,
2771                                    ),
2772                                    "",
2773                                    esc_point.site.pos - 1,
2774                                    esc_point.site.ri,
2775                                    esc_point.site.ci - 1,
2776                                );
2777                                self.err = Some(err.clone());
2778                                return Err(Box::new(err));
2779                            }
2780                            let byte = u8::from_str_radix(&hex, 16).expect("validated ASCII hex");
2781                            self.flush_surrogate(&mut pending_high_surrogate, &mut out_str);
2782                            out_str.push(char::from(byte));
2783                        }
2784                        other => {
2785                            if !self.options.string.allow_unknown {
2786                                // Sited on the escape CHARACTER with a
2787                                // one-character span, as TypeScript
2788                                // (`pnt.sI = sI; pnt.cI = cI; lex.bad(
2789                                // S.unexpected, sI, sI + 1)`) and Go do;
2790                                // the other escape errors sit on the
2791                                // backslash and span the construct.
2792                                let err = TabnasError::new(
2793                                    "unexpected",
2794                                    other.to_string(),
2795                                    "",
2796                                    esc_point.site.pos,
2797                                    esc_point.site.ri,
2798                                    esc_point.site.ci,
2799                                );
2800                                self.err = Some(err.clone());
2801                                return Err(Box::new(err));
2802                            }
2803                            self.flush_surrogate(&mut pending_high_surrogate, &mut out_str);
2804                            out_str.push(other);
2805                        }
2806                    }
2807                } else {
2808                    let err = TabnasError::new(
2809                        "unterminated_string",
2810                        raw_src,
2811                        "",
2812                        pnt.site.pos,
2813                        pnt.site.ri,
2814                        pnt.site.ci,
2815                    );
2816                    self.err = Some(err.clone());
2817                    return Err(Box::new(err));
2818                }
2819            } else {
2820                self.flush_surrogate(&mut pending_high_surrogate, &mut out_str);
2821                raw_src.push(self.advance_body().expect("peeked character must advance"));
2822                out_str.push(c);
2823            }
2824        }
2825
2826        let err = TabnasError::new(
2827            "unterminated_string",
2828            raw_src,
2829            "",
2830            pnt.site.pos,
2831            pnt.site.ri,
2832            pnt.site.ci,
2833        );
2834        self.err = Some(err.clone());
2835        Err(Box::new(err))
2836    }
2837
2838    fn flush_surrogate(&self, pending: &mut Option<u16>, out: &mut String) {
2839        if pending.take().is_some() {
2840            out.push('\u{FFFD}');
2841        }
2842    }
2843
2844    /// Emit one decoded Unicode escape while pairing UTF-16 surrogate code
2845    /// units across both `\\uXXXX` and `\\u{...}` spellings.
2846    fn emit_unicode_escape(&self, cp: u32, pending: &mut Option<u16>, out: &mut String) {
2847        if (0xD800..=0xDBFF).contains(&cp) {
2848            self.flush_surrogate(pending, out);
2849            *pending = Some(cp as u16);
2850        } else if (0xDC00..=0xDFFF).contains(&cp) {
2851            if let Some(high) = pending.take() {
2852                let scalar = 0x10000 + (((u32::from(high)) - 0xD800) << 10) + (cp - 0xDC00);
2853                out.push(char::from_u32(scalar).expect("paired surrogates form a Unicode scalar"));
2854            } else {
2855                out.push('\u{FFFD}');
2856            }
2857        } else {
2858            self.flush_surrogate(pending, out);
2859            out.push(char::from_u32(cp).expect("validated escape is a Unicode scalar"));
2860        }
2861    }
2862}
2863
2864// ---------------------------------------------------------------------------
2865// Base-prefixed integer literals.
2866//
2867// A `0x`, `0o` or `0b` literal is read as an EXACT integer and rounded to
2868// a double ONCE. The obvious fold -- `value = value * radix + digit` in
2869// `f64` -- rounds at every digit, and past the 53-bit exact integer range
2870// those roundings accumulate: `0Xa6f2f78f4f9bf44` came out as
2871// `43a4de5ef1e9f37e` where canonical TypeScript and the Go port both
2872// answer `43a4de5ef1e9f37f`, one unit in the last place low. That is
2873// silently altered data, not a formatting difference.
2874//
2875// TypeScript coerces the literal with unary `+`, whose StringNumericValue
2876// is the exact mathematical value of the digits rounded once, half to
2877// even; Go reads it through `big.Int` and `big.Float.Float64()`, which is
2878// the same rule. These reproduce it. Only the VALUE is affected: which
2879// literals are accepted, and the token they become, are settled by
2880// `match_number` before any of this runs.
2881//
2882// The decimal path needs none of it -- `str::parse::<f64>` is correctly
2883// rounded for a digit string of any length.
2884// ---------------------------------------------------------------------------
2885
2886/// `2^k` for a non-negative `k`, exactly, saturating to infinity above the
2887/// double range. A repeated multiply would round on the way up.
2888fn pow2(k: i64) -> f64 {
2889    debug_assert!(k >= 0, "only non-negative exponents arise here");
2890    if k > 1023 {
2891        f64::INFINITY
2892    } else {
2893        f64::from_bits(((k + 1023) as u64) << 52)
2894    }
2895}
2896
2897/// Folds the digits of a base-prefixed literal into the NEAREST double,
2898/// rounding half to even, without holding the digits.
2899///
2900/// `bits` is the width of one digit, so the base is a power of two: 1 for
2901/// binary, 3 for octal, 4 for hexadecimal. Those are the only bases
2902/// `match_number` reads, which is what lets a `u128` head plus a sticky
2903/// bit stand in for arbitrary-precision arithmetic.
2904///
2905/// Only three things about a literal can change the answer: the top
2906/// `128 / bits` significant digits, how many digits follow them, and
2907/// whether any of those is non-zero. This keeps exactly those, so the
2908/// space it costs does not grow with the literal, however long an
2909/// untrusted document makes one.
2910struct DigitFold {
2911    bits: u32,
2912    /// The significant digits packed so far, at most `head_len` of them.
2913    head: u128,
2914    /// How many significant digits have been pushed, head and tail alike.
2915    len: usize,
2916    /// Whether any digit past the head was non-zero.
2917    sticky: bool,
2918    /// Whether a non-zero digit has been seen. Leading zeros carry no
2919    /// value, and dropping them is what makes the head wider than the 54
2920    /// significant bits the rounding needs.
2921    started: bool,
2922}
2923
2924impl DigitFold {
2925    fn new(bits: u32) -> Self {
2926        debug_assert!(
2927            (1..=4).contains(&bits),
2928            "only the power-of-two bases the lexer reads"
2929        );
2930        DigitFold {
2931            bits,
2932            head: 0,
2933            len: 0,
2934            sticky: false,
2935            started: false,
2936        }
2937    }
2938
2939    /// A `u128` holds exactly this many digits of the base.
2940    fn head_len(&self) -> usize {
2941        (128 / self.bits) as usize
2942    }
2943
2944    fn push(&mut self, digit: u32) {
2945        if !self.started {
2946            if 0 == digit {
2947                return;
2948            }
2949            self.started = true;
2950        }
2951        if self.len < self.head_len() {
2952            self.head = (self.head << self.bits) | u128::from(digit);
2953        } else if 0 != digit {
2954            self.sticky = true;
2955        }
2956        self.len += 1;
2957    }
2958
2959    fn finish(&self) -> f64 {
2960        if !self.started {
2961            return 0.0;
2962        }
2963        let head_len = self.head_len();
2964        if self.len <= head_len {
2965            // A `u128` to `f64` cast rounds to nearest, ties to even, which
2966            // is the rule the canonical runtime follows.
2967            return self.head as f64;
2968        }
2969
2970        // Longer than a u128: the head holds the top `head_len` digits and
2971        // `sticky` remembers whether anything below them was set. Those two
2972        // are all the rounding can depend on. The leading digit is
2973        // non-zero, so the head is at least 121 bits wide in every base
2974        // here and `shift` is comfortably positive.
2975        let dropped = i64::from(self.bits) * (self.len - head_len) as i64;
2976        let shift = 128 - self.head.leading_zeros() - 53;
2977
2978        let mut mantissa = (self.head >> shift) as u64;
2979        let half = (self.head >> (shift - 1)) & 1 == 1;
2980        let sticky = self.head & ((1u128 << (shift - 1)) - 1) != 0 || self.sticky;
2981        if half && (sticky || mantissa & 1 == 1) {
2982            // At most 2^53, which is still an exact double.
2983            mantissa += 1;
2984        }
2985        // The mantissa carries at most 53 significant bits, so the scaling
2986        // is exact inside the double range and overflows to infinity
2987        // outside it.
2988        mantissa as f64 * pow2(dropped + i64::from(shift))
2989    }
2990}
2991
2992#[cfg(test)]
2993mod window_tests {
2994    use super::{WindowSizes, WINDOW_PEAK};
2995
2996    /// A parse from a reader holds no more of the source than its window:
2997    /// a block behind the cursor and a margin or two ahead of it, however
2998    /// long the source, flat from 2,000 records to 20,000.
2999    #[test]
3000    fn a_parse_from_a_reader_holds_a_bounded_window() {
3001        let mut parser = crate::Tabnas::make_json();
3002        parser
3003            .set_options(|options| options.lex.window = true)
3004            .expect("lex.window sets");
3005        let sizes = WindowSizes {
3006            margin: 1024,
3007            block: 1024,
3008            chunk: 1024,
3009        };
3010        let mut peaks = Vec::new();
3011        for count in [2_000, 20_000] {
3012            let records: Vec<String> = (0..count)
3013                .map(|i| format!("{{\"id\":{i},\"name\":\"n{i}\",\"tags\":[\"a\",\"b\"]}}"))
3014                .collect();
3015            let text = format!("[{}]", records.join(","));
3016            WINDOW_PEAK.with(|peak| peak.set(0));
3017            parser
3018                .parse_reader_with_sizes(text.as_bytes(), sizes)
3019                .expect("the reader parses");
3020            let peak = WINDOW_PEAK.with(std::cell::Cell::get);
3021            assert!(peak <= 4 * 1024, "{count} records held {peak} characters");
3022            peaks.push(peak);
3023        }
3024        // Ten times the records hold no more than a few records' more:
3025        // where the blocks fall against the records moves the peak a
3026        // little, the length not at all.
3027        assert!(peaks[1] <= peaks[0] + 256, "{peaks:?}");
3028    }
3029}