tabnas/lexer.rs
1// Copyright (c) 2013-2026 Richard Rodger, MIT License
2
3use crate::error::TabnasError;
4use crate::options::{LexCheck, LexCheckResult, MatchTokenMatcher, Options};
5use crate::token::{
6 Point, Token, TIN_BD, TIN_CM, TIN_LN, TIN_NR, TIN_SP, TIN_ST, TIN_TX, TIN_VL, TIN_ZZ,
7};
8use crate::value::Value;
9use regex::Regex;
10use std::panic::{catch_unwind, AssertUnwindSafe};
11use std::sync::Arc;
12
13pub struct Lexer<'a> {
14 /// The text: the whole source, or the window of one read from a
15 /// reader ([`Text`]).
16 src: Text<'a>,
17 /// The characters from `low` on: the whole source's, or the window's.
18 chars: Vec<char>,
19 /// The byte offset of each character, or `None` for an ASCII source,
20 /// where a character's index is its byte offset (`byte_at`). Offsets
21 /// and indexes are the source's, from its start, in a window too.
22 byte_indices: Option<Vec<usize>>,
23 /// The end of the characters read: the source's length, or the
24 /// window's end.
25 char_len: usize,
26 /// The index and the byte offset of the first character `chars`
27 /// holds: zero for the whole source, the window's start otherwise.
28 low: usize,
29 low_byte: usize,
30 /// A read reached the window's end while the source had more: the
31 /// token being cut may be cut short, and is cut again over a larger
32 /// window ([`Lexer::next_raw_inner`]).
33 touched_end: std::cell::Cell<bool>,
34 idx: usize,
35 ri: usize,
36 ci: usize,
37 options: Arc<Options>,
38 ignore_tins: Vec<crate::Tin>,
39 char_sets: crate::text::CharSets,
40 err: Option<TabnasError>,
41 end_reached: bool,
42 /// `number.exclude`, compiled once per grammar and shared by the
43 /// parser that owns it; see [`compile_number_exclude`].
44 exclude_regex: Option<Arc<Regex>>,
45 want: Option<Vec<crate::Tin>>,
46 standalone: Option<(crate::Rule, crate::Context)>,
47}
48
49#[derive(Clone)]
50pub(crate) struct LexerState {
51 idx: usize,
52 ri: usize,
53 ci: usize,
54 err: Option<TabnasError>,
55 end_reached: bool,
56}
57
58/// Opaque snapshot returned by [`Lexer::relex_for_rule`]. Pass it to
59/// [`Lexer::unrelex`] if the caller later rejects the committed recut.
60#[derive(Clone)]
61pub struct RelexCheckpoint {
62 state: LexerState,
63 replay: std::collections::VecDeque<Token>,
64}
65
66enum CheckFlow {
67 Continue,
68 Skip,
69 Token(Box<Token>),
70}
71
72/// The result the lexer passes through its own internals.
73///
74/// `TabnasError` is 664 bytes, so `LexResult<Token>` is 664 bytes
75/// too: three times the token it carries, and moved on the SUCCESS path
76/// through every frame of the lexer chain. Boxing the error inside the
77/// engine makes those results the size of a token again, and costs an
78/// allocation only when there is an error to report, which is the path that
79/// is already building a 664-byte diagnostic.
80///
81/// The public entry points still hand back an unboxed `TabnasError`, so no
82/// caller of this crate sees the box.
83type LexResult<T> = Result<T, Box<TabnasError>>;
84
85/// The compiled form of `number.exclude`, or `None` when there is no
86/// pattern or it does not compile (an invalid pattern excludes nothing,
87/// as it always has).
88///
89/// Compiling a regex costs about 700K instructions, which is more than
90/// a small document costs to parse, so the parser compiles it once per
91/// grammar and hands every lexer the same automaton through the `Arc`.
92/// Cloning a `Regex` would not do: it shares the automaton but builds a
93/// fresh scratch-cache pool, and the first search from each clone fills
94/// it. Go and TypeScript compile the pattern once, at configuration time.
95pub(crate) fn compile_number_exclude(options: &Options) -> Option<Arc<Regex>> {
96 let pattern = options.number.exclude.as_deref()?;
97 Regex::new(pattern).ok().map(Arc::new)
98}
99
100/// What a lexer reads: the whole source, borrowed, or a window of a
101/// source it reads from a reader a chunk at a time, filling the window as
102/// it needs more and dropping it behind the oldest token the parser may
103/// still re-cut ([`Lexer::release_before`]).
104enum Text<'a> {
105 Whole(&'a str),
106 Window(Box<Window<'a>>),
107}
108
109/// The text an error shows its line from, and the row and column it
110/// starts at ([`Lexer::excerpt`]).
111pub(crate) type Excerpt = (String, usize, usize);
112
113/// A source read from a reader, as the parser hands it to the lexer: the
114/// reader, the window's sizes, and a cell the window leaves its text in
115/// when the lexer is done, so that an error the parser raised after the
116/// lexer is gone still shows its line from it.
117pub(crate) struct ReaderSource<'a> {
118 pub(crate) reader: Box<dyn std::io::Read + 'a>,
119 pub(crate) sizes: WindowSizes,
120 pub(crate) excerpt: std::rc::Rc<std::cell::RefCell<Option<Excerpt>>>,
121}
122
123impl<'a> ReaderSource<'a> {
124 pub(crate) fn new(reader: Box<dyn std::io::Read + 'a>, sizes: WindowSizes) -> Self {
125 ReaderSource {
126 reader,
127 sizes,
128 excerpt: std::rc::Rc::default(),
129 }
130 }
131}
132
133/// A source read from a reader ([`Text::Window`]).
134struct Window<'a> {
135 /// The text read and kept, from the lexer's `low_byte` on.
136 text: String,
137 reader: Box<dyn std::io::Read + 'a>,
138 /// Bytes read and not yet decoded: the start of a character a read
139 /// split.
140 pending: Vec<u8>,
141 /// The buffer a read fills, kept from one read to the next.
142 buffer: Vec<u8>,
143 exhausted: bool,
144 /// Why the source ended early, and where: a read that failed, or bytes
145 /// that are not UTF-8. The next token raises it ([`Lexer::unreadable`]).
146 failure: Option<Unreadable>,
147 /// The oldest character the parser may still ask for again.
148 keep: usize,
149 /// The row and column of the window's first character, counted as an
150 /// error's excerpt counts them: a line feed starts a row.
151 low_row: usize,
152 low_col: usize,
153 /// The row and column after the last character read, counted the same
154 /// way, which place a failure.
155 read_row: usize,
156 read_col: usize,
157 /// How far ahead of the cursor the window reaches before each token.
158 margin: usize,
159 /// How much dead text gathers before it is dropped, and how much a
160 /// read asks for.
161 block: usize,
162 chunk: usize,
163 /// Where the window leaves its text when the lexer is dropped
164 /// ([`ReaderSource::excerpt`]).
165 excerpt: std::rc::Rc<std::cell::RefCell<Option<Excerpt>>>,
166}
167
168impl Window<'_> {
169 /// Read on to the end of the window's last line, no further than a
170 /// margin, so that an excerpt shows the line an error is on as a parse
171 /// of the whole text shows it, but for a line longer than the window.
172 /// The lexer is done with the window, so this only adds to its text.
173 fn finish_line(&mut self) {
174 let mut read = 0;
175 while !self.exhausted && !self.text.ends_with('\n') && read < self.margin {
176 if self.buffer.len() != self.chunk {
177 self.buffer = vec![0u8; self.chunk];
178 }
179 match self.reader.read(&mut self.buffer) {
180 Ok(0) | Err(_) => self.exhausted = true,
181 Ok(n) => self.pending.extend_from_slice(&self.buffer[..n]),
182 }
183 let valid = match std::str::from_utf8(&self.pending) {
184 Ok(_) => self.pending.len(),
185 Err(e) => {
186 if e.error_len().is_some() {
187 self.exhausted = true;
188 }
189 e.valid_up_to()
190 }
191 };
192 let decoded = std::str::from_utf8(&self.pending[..valid]).expect("the prefix is UTF-8");
193 for c in decoded.chars() {
194 if self.text.ends_with('\n') || read >= self.margin {
195 break;
196 }
197 self.text.push(c);
198 read += 1;
199 }
200 self.pending.drain(..valid);
201 }
202 }
203}
204
205impl Drop for Window<'_> {
206 fn drop(&mut self) {
207 // A parse that ended before its source did (an error) shows its
208 // line read to its end. The text is moved, not copied: the lexer
209 // is done with it.
210 self.finish_line();
211 *self.excerpt.borrow_mut() =
212 Some((std::mem::take(&mut self.text), self.low_row, self.low_col));
213 }
214}
215
216/// A source that could not be read to its end: what went wrong, and the
217/// character, row and column it went wrong at.
218#[derive(Clone, Debug)]
219pub(crate) struct Unreadable {
220 pub(crate) reason: String,
221 pub(crate) pos: usize,
222 pub(crate) row: usize,
223 pub(crate) col: usize,
224}
225
226impl Unreadable {
227 /// The error a parse fails with: `unreadable`, the reason as its source
228 /// text, at the character the source failed at.
229 pub(crate) fn error(&self) -> TabnasError {
230 TabnasError::new(
231 "unreadable",
232 self.reason.as_str(),
233 "",
234 self.pos,
235 self.row,
236 self.col,
237 )
238 }
239}
240
241#[cfg(test)]
242thread_local! {
243 /// The most characters a window has held on this thread, for the test
244 /// that holds a parse from a reader to a bounded window.
245 static WINDOW_PEAK: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
246}
247
248/// The reason a source is not UTF-8, as a reader's own `read_to_string`
249/// gives it.
250pub(crate) const NOT_UTF8: &str = "stream did not contain valid UTF-8";
251
252/// Where `text` ends, as a failure's position counts it: its characters,
253/// and the row and column after its last one, a line feed starting a row.
254pub(crate) fn position_after(text: &str) -> (usize, usize, usize) {
255 let (mut pos, mut row, mut col) = (0, 1, 1);
256 for c in text.chars() {
257 pos += 1;
258 if c == '\n' {
259 row += 1;
260 col = 1;
261 } else {
262 col += 1;
263 }
264 }
265 (pos, row, col)
266}
267
268/// A window's sizes: the margin ahead of the cursor, the dead text
269/// gathered before it is dropped, and a read's length. Each is 64 KiB
270/// (admin ADR-30), and none is an option: only the differential test makes
271/// them a few bytes, so that every matcher meets the window's edge
272/// ([`crate::Tabnas::parse_reader_with_sizes`]).
273#[doc(hidden)]
274#[derive(Clone, Copy, Debug)]
275pub struct WindowSizes {
276 pub margin: usize,
277 pub block: usize,
278 pub chunk: usize,
279}
280
281impl Default for WindowSizes {
282 fn default() -> Self {
283 WindowSizes {
284 margin: 64 * 1024,
285 block: 64 * 1024,
286 chunk: 64 * 1024,
287 }
288 }
289}
290
291impl<'a> Lexer<'a> {
292 pub fn new(src: &'a str, mut options: Options) -> Self {
293 if let Err(error) = options.validate_comment_definitions() {
294 panic!("invalid options: {error}");
295 }
296 // A lexer built directly may be handed options nobody has
297 // ordered yet. The parser's own lexer comes through
298 // `with_shared`, whose options were ordered when they were
299 // prepared, and whose exclude pattern was compiled then too.
300 options.sort_for_lexing();
301 let exclude_regex = compile_number_exclude(&options);
302 Self::with_shared(src, Arc::new(options), exclude_regex)
303 }
304
305 /// Lex against options the parser already owns and has ordered, with
306 /// the `number.exclude` pattern it compiled from them.
307 pub(crate) fn with_shared(
308 src: &'a str,
309 options: Arc<Options>,
310 exclude_regex: Option<Arc<Regex>>,
311 ) -> Self {
312 // Both tables are sized before they are filled: grown by doubling,
313 // a long source's peak was twice its tables.
314 let (chars, byte_indices) = if src.is_ascii() {
315 (src.bytes().map(char::from).collect::<Vec<char>>(), None)
316 } else {
317 let count = src.chars().count();
318 let mut chars = Vec::with_capacity(count);
319 let mut byte_indices = Vec::with_capacity(count);
320 for (b_idx, c) in src.char_indices() {
321 chars.push(c);
322 byte_indices.push(b_idx);
323 }
324 (chars, Some(byte_indices))
325 };
326 let char_len = chars.len();
327
328 Lexer {
329 src: Text::Whole(src),
330 chars,
331 byte_indices,
332 char_len,
333 low: 0,
334 low_byte: 0,
335 touched_end: std::cell::Cell::new(false),
336 idx: 0,
337 ri: 1,
338 ci: 1,
339 ignore_tins: options.ignore_tins(),
340 char_sets: options.char_sets(),
341 options,
342 err: None,
343 end_reached: false,
344 exclude_regex,
345 want: None,
346 standalone: None,
347 }
348 }
349
350 /// Lex a source read from `reader`, a window of it at a time
351 /// ([`Text::Window`]).
352 pub(crate) fn with_reader(
353 source: ReaderSource<'a>,
354 options: Arc<Options>,
355 exclude_regex: Option<Arc<Regex>>,
356 ) -> Self {
357 let ReaderSource {
358 reader,
359 sizes,
360 excerpt,
361 } = source;
362 Lexer {
363 src: Text::Window(Box::new(Window {
364 text: String::new(),
365 reader,
366 pending: Vec::new(),
367 buffer: Vec::new(),
368 exhausted: false,
369 failure: None,
370 keep: 0,
371 low_row: 1,
372 low_col: 1,
373 read_row: 1,
374 read_col: 1,
375 margin: sizes.margin.max(1),
376 block: sizes.block.max(1),
377 chunk: sizes.chunk.max(1),
378 excerpt,
379 })),
380 chars: Vec::new(),
381 byte_indices: None,
382 char_len: 0,
383 low: 0,
384 low_byte: 0,
385 touched_end: std::cell::Cell::new(false),
386 idx: 0,
387 ri: 1,
388 ci: 1,
389 ignore_tins: options.ignore_tins(),
390 char_sets: options.char_sets(),
391 options,
392 err: None,
393 end_reached: false,
394 exclude_regex,
395 want: None,
396 standalone: None,
397 }
398 }
399
400 /// The text the lexer holds: the whole source, or the window.
401 fn text(&self) -> &str {
402 match &self.src {
403 Text::Whole(src) => src,
404 Text::Window(window) => &window.text,
405 }
406 }
407
408 /// The source's length in bytes, or, in a window, the length read.
409 fn src_len(&self) -> usize {
410 match &self.src {
411 Text::Whole(src) => src.len(),
412 Text::Window(window) => self.low_byte + window.text.len(),
413 }
414 }
415
416 /// The text from the source's byte offset `byte` on, as far as the
417 /// lexer holds it.
418 fn tail_at_byte(&self, byte: usize) -> &str {
419 &self.text()[byte - self.low_byte..]
420 }
421
422 /// Whether the lexer reads a window of a source that has more to give.
423 #[inline]
424 fn window_open(&self) -> bool {
425 matches!(&self.src, Text::Window(window) if !window.exhausted)
426 }
427
428 /// Note a read at the window's end, where the source has more.
429 #[inline]
430 fn touch_end(&self) {
431 if self.window_open() {
432 self.touched_end.set(true);
433 }
434 }
435
436 /// The character at the source's index `index`, which the lexer holds.
437 #[inline]
438 fn ch(&self, index: usize) -> char {
439 self.chars[index - self.low]
440 }
441
442 /// The character at `index`, or `None` past what the lexer holds.
443 #[inline]
444 fn ch_get(&self, index: usize) -> Option<char> {
445 let found = self.chars.get(index - self.low).copied();
446 if found.is_none() {
447 self.touch_end();
448 }
449 found
450 }
451
452 /// Whether the lexer reads a window of its source.
453 pub fn windowed(&self) -> bool {
454 matches!(self.src, Text::Window(_))
455 }
456
457 /// The characters read so far, counted as UTF-16 units: in a window,
458 /// what the parser's step budget is measured against as it grows.
459 pub(crate) fn utf16_read(&self) -> usize {
460 match &self.src {
461 Text::Whole(src) => src.encode_utf16().count(),
462 Text::Window(_) => self.char_len,
463 }
464 }
465
466 /// Whether a source read from a reader turns out empty: the window is
467 /// filled until it holds a character or the reader is done. A source
468 /// that fails before its first character is not empty: its failure is
469 /// the parse's error.
470 pub(crate) fn is_empty_source(&mut self) -> bool {
471 self.fill(1);
472 self.char_len == 0 && !matches!(&self.src, Text::Window(window) if window.failure.is_some())
473 }
474
475 /// The failure that ended a source read from a reader early, taken
476 /// once: the parse fails with it at the next token.
477 pub(crate) fn unreadable(&mut self) -> Option<Unreadable> {
478 match &mut self.src {
479 Text::Window(window) => window.failure.take(),
480 Text::Whole(_) => None,
481 }
482 }
483
484 /// Whether the window has gathered a block of text behind the cursor,
485 /// so that dropping what the parser no longer holds may pay. Until it
486 /// has, the parser need not find the oldest token it holds.
487 #[inline]
488 pub(crate) fn may_release(&self) -> bool {
489 matches!(&self.src, Text::Window(window) if self.idx >= self.low + window.block)
490 }
491
492 /// The least position a window drops its text up to: it drops nothing
493 /// unless the oldest token the parser holds starts at or past it.
494 #[inline]
495 pub(crate) fn release_floor(&self) -> usize {
496 match &self.src {
497 Text::Window(window) => self.low + window.block,
498 Text::Whole(_) => usize::MAX,
499 }
500 }
501
502 /// The parser holds no token that starts before `pos`, so the window
503 /// may drop the text before it (or before the cursor, if that is
504 /// earlier).
505 pub(crate) fn release_before(&mut self, pos: usize) {
506 let keep = pos.min(self.idx);
507 if let Text::Window(window) = &mut self.src {
508 window.keep = keep;
509 }
510 }
511
512 /// The text an error shows its line from, and the row and column that
513 /// text starts at: the whole source from row 1 and column 1, or in a
514 /// parse from a reader the window, whose first line is cut where the
515 /// window begins.
516 pub(crate) fn excerpt(&self) -> (String, usize, usize) {
517 match &self.src {
518 Text::Whole(src) => ((*src).to_string(), 1, 1),
519 Text::Window(window) => (window.text.clone(), window.low_row, window.low_col),
520 }
521 }
522
523 /// The text from the cursor to the end of what the lexer holds: in a
524 /// parse from a reader, the end of the window. A matcher in a grammar
525 /// that declares `lex.window` reads the source through this and
526 /// [`Lexer::fill`], never past them.
527 pub fn window(&self) -> &str {
528 self.tail_at_byte(self.byte_position())
529 }
530
531 /// Read the source until `ahead` characters lie past the cursor, or it
532 /// ends, and say whether they do. A whole source is all read, so this
533 /// only answers. In a parse from a reader the window may drop text
534 /// behind the oldest token the parser holds, never the token being
535 /// cut, and a source that fails to read ends where it failed, the parse
536 /// failing with it at the next token.
537 pub fn fill(&mut self, ahead: usize) -> bool {
538 self.fill_window(ahead);
539 self.char_len - self.idx >= ahead
540 }
541
542 /// Fill the window until `ahead` characters lie past the cursor or the
543 /// source is exhausted, dropping first the text before what the parser
544 /// may still ask for, a block at a time.
545 fn fill_window(&mut self, ahead: usize) {
546 let Text::Window(window) = &mut self.src else {
547 return;
548 };
549 let window: &mut Window<'a> = window;
550 let keep = window.keep.min(self.idx);
551 // Most tokens find the window long enough and nothing to drop.
552 if (window.exhausted || self.char_len - self.idx >= ahead) && keep < self.low + window.block
553 {
554 return;
555 }
556 if keep >= self.low + window.block {
557 let chars = keep - self.low;
558 // A drop to the end of what has been read, where the parser
559 // holds no token and the cursor stands at the end, drops the
560 // whole text: there is no character there to take the offset
561 // of (#322).
562 let bytes = match &self.byte_indices {
563 Some(offsets) => offsets
564 .get(chars)
565 .map_or(window.text.len(), |offset| offset - self.low_byte),
566 None => chars,
567 };
568 let dropped = &window.text[..bytes];
569 match dropped.rfind('\n') {
570 Some(last) => {
571 window.low_row += dropped.bytes().filter(|&b| b == b'\n').count();
572 window.low_col = 1 + dropped[last + 1..].chars().count();
573 }
574 None => window.low_col += chars,
575 }
576 self.chars.drain(..chars);
577 if let Some(offsets) = &mut self.byte_indices {
578 offsets.drain(..chars);
579 }
580 window.text.drain(..bytes);
581 self.low = keep;
582 self.low_byte += bytes;
583 }
584 if window.buffer.len() != window.chunk {
585 window.buffer = vec![0u8; window.chunk];
586 }
587 while !window.exhausted && self.char_len - self.idx < ahead {
588 let read = loop {
589 match window.reader.read(&mut window.buffer) {
590 Ok(read) => break Ok(read),
591 Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
592 Err(e) => break Err(e.to_string()),
593 }
594 };
595 let read = match read {
596 Ok(read) => read,
597 Err(reason) => {
598 window.exhausted = true;
599 window.failure = Some(Unreadable {
600 reason,
601 pos: self.char_len,
602 row: window.read_row,
603 col: window.read_col,
604 });
605 break;
606 }
607 };
608 if read == 0 {
609 window.exhausted = true;
610 }
611 window.pending.extend_from_slice(&window.buffer[..read]);
612 // The longest prefix that decodes. A character a read split
613 // waits for its last byte; bytes that are no UTF-8, or a
614 // character the source ends inside, end the source there, as
615 // `read_to_string` refuses them.
616 let valid = match std::str::from_utf8(&window.pending) {
617 Ok(_) => window.pending.len(),
618 Err(e) => {
619 if e.error_len().is_some() || window.exhausted {
620 window.exhausted = true;
621 window.failure = Some(Unreadable {
622 reason: NOT_UTF8.to_string(),
623 pos: 0,
624 row: 0,
625 col: 0,
626 });
627 }
628 e.valid_up_to()
629 }
630 };
631 let decoded =
632 std::str::from_utf8(&window.pending[..valid]).expect("the prefix is UTF-8");
633 for c in decoded.chars() {
634 if self.byte_indices.is_none() && !c.is_ascii() {
635 // Every character so far was ASCII, its offset its
636 // index.
637 let mut offsets = Vec::with_capacity(self.chars.capacity());
638 offsets.extend((0..self.chars.len()).map(|k| self.low + k));
639 self.byte_indices = Some(offsets);
640 }
641 if let Some(offsets) = &mut self.byte_indices {
642 offsets.push(self.low_byte + window.text.len());
643 }
644 if c == '\n' {
645 window.read_row += 1;
646 window.read_col = 1;
647 } else {
648 window.read_col += 1;
649 }
650 window.text.push(c);
651 self.chars.push(c);
652 }
653 window.pending.drain(..valid);
654 self.char_len = self.low + self.chars.len();
655 #[cfg(test)]
656 WINDOW_PEAK.with(|peak| peak.set(peak.get().max(self.chars.len())));
657 if let Some(failure) = &mut window.failure {
658 if failure.row == 0 {
659 // The bytes that are no UTF-8 start after the last
660 // character decoded.
661 failure.pos = self.char_len;
662 failure.row = window.read_row;
663 failure.col = window.read_col;
664 }
665 }
666 }
667 }
668
669 fn current_point(&self) -> Point {
670 Point {
671 len: self.src_len(),
672 site: crate::Site {
673 si: self.byte_position(),
674 pos: self.idx,
675 ri: self.ri,
676 ci: self.ci,
677 },
678 }
679 }
680
681 /// The source from char index `start` up to `end`, clipped to the end
682 /// of the source: the span of a bad token, cut as TypeScript's
683 /// `lex.bad(why, pstart, pend)` cuts it (`ts/src/lexer.ts`).
684 ///
685 /// Only the END is clipped. The caller must supply `start <= end`
686 /// and `start <= self.char_len`, or the indexing panics. Every call
687 /// site below passes `esc_point.site.pos - 1`, which additionally
688 /// needs `1 <= pos`; that holds because `esc_point` is captured
689 /// after the escape character has been consumed. These three are
690 /// the preconditions `rs/verus/lexer_span.rs` ASSUMES: it proves the
691 /// span arithmetic is in bounds given them, and does not model the
692 /// call sites, so nothing machine-checks that they hold here. A
693 /// refactor that captured the point before the advance would still
694 /// pass `rs/verus/run.sh`.
695 fn source_span(&self, start: usize, end: usize) -> String {
696 if end > self.char_len {
697 // A span cut at a window's end may be cut short.
698 self.touch_end();
699 }
700 self.chars[start - self.low..end.min(self.char_len) - self.low]
701 .iter()
702 .collect()
703 }
704
705 fn state(&self) -> LexerState {
706 LexerState {
707 idx: self.idx,
708 ri: self.ri,
709 ci: self.ci,
710 err: self.err.clone(),
711 end_reached: self.end_reached,
712 }
713 }
714
715 /// Full immutable source supplied to this lexer; in a parse from a
716 /// reader, the window the lexer holds.
717 pub fn source(&self) -> &str {
718 self.text()
719 }
720
721 /// Source remaining at the live cursor; in a parse from a reader, to
722 /// the window's end.
723 pub fn remaining(&self) -> &str {
724 self.tail_at_byte(self.byte_position())
725 }
726
727 /// Return at most `max_chars` Unicode scalar values from the live cursor.
728 /// This is the Rust counterpart of the public `lex.fwd`/`Lex.Fwd` helper.
729 pub fn forward(&self, max_chars: usize) -> &str {
730 let remaining = self.remaining();
731 let end = remaining
732 .char_indices()
733 .nth(max_chars)
734 .map_or(remaining.len(), |(index, _)| index);
735 &remaining[..end]
736 }
737
738 /// Snapshot the live cursor for token construction.
739 pub fn point(&self) -> Point {
740 self.current_point()
741 }
742
743 /// Advance by Unicode scalar values. Returns false without moving when
744 /// the requested count extends beyond end-of-source.
745 pub fn advance_chars(&mut self, count: usize) -> bool {
746 if self.idx.saturating_add(count) > self.char_len {
747 // In a window, past its end may yet be in the source: the cut
748 // is made again over a larger window.
749 self.touch_end();
750 return false;
751 }
752 for _ in 0..count {
753 self.advance();
754 }
755 true
756 }
757
758 /// Construct a token from a point captured before cursor advancement.
759 pub fn token(
760 &self,
761 name: impl AsRef<str>,
762 tin: crate::Tin,
763 value: Value,
764 source: impl Into<crate::TokenText>,
765 point: Point,
766 ) -> Token {
767 Token::new(name, tin, value, source, point)
768 }
769
770 /// Resolve or allocate a token identity in this lexer's configuration.
771 pub fn token_tin(&mut self, name: impl Into<String>) -> crate::Tin {
772 Arc::make_mut(&mut self.options).register_token(name)
773 }
774
775 /// Resolve a token identity back to its configured name.
776 pub fn token_name(&self, tin: crate::Tin) -> String {
777 self.options.token_name(tin)
778 }
779
780 /// Whether the options this lexer runs under enable `alt`: not when
781 /// `rule.exclude` names one of its groups, nor when `rule.include`
782 /// lists groups and it declares none of them. The parser skips such an
783 /// alternate, and TypeScript removes it from the rule spec
784 /// (`filterRules`) before anything reads the spec, so a custom matcher
785 /// that reads a rule's alternates, to tell a key position from a value
786 /// one, asks this to see the same alternates TypeScript does.
787 pub fn alt_enabled(&self, alt: &crate::AltSpec) -> bool {
788 crate::parser::groups_enabled(alt, &self.options)
789 }
790
791 /// Construct a bad token at the current cursor.
792 ///
793 /// A custom matcher that returns it reports a lexer fault, which the
794 /// parser handles as it handles its own lexer's: a rule's fetch raises
795 /// it at once with `why` as the code, at this position, or, under
796 /// recovery, records it, coalescing a run of them into one error, and
797 /// moves the cursor past its source; relexing leaves it for an
798 /// alternate to re-cut. The cursor need not be advanced here.
799 pub fn bad(&self, why: impl Into<String>) -> Token {
800 let point = self.current_point();
801 let source = self
802 .peek()
803 .map_or_else(String::new, |character| character.to_string());
804 let mut token = Token::new("#BD", TIN_BD, Value::Undefined, source, point);
805 token.err = crate::TokenCode::from(why.into());
806 token.why = token.err.clone();
807 token
808 }
809
810 /// Construct a bad token whose displayed source is a scalar-indexed span.
811 /// As in TypeScript, the diagnostic point remains the live cursor.
812 /// Returned from a matcher it is handled as [`Lexer::bad`] describes;
813 /// recovery steps past the span, counted from that cursor.
814 pub fn bad_span(&self, why: impl Into<String>, start: usize, end: usize) -> Token {
815 let point = self.current_point();
816 let source = if start <= end && end <= self.char_len {
817 self.text()[self.byte_at(start) - self.low_byte..self.byte_at(end) - self.low_byte]
818 .to_string()
819 } else {
820 self.peek()
821 .map_or_else(String::new, |character| character.to_string())
822 };
823 let mut token = Token::new("#BD", TIN_BD, Value::Undefined, source, point);
824 token.err = crate::TokenCode::from(why.into());
825 token.why = token.err.clone();
826 token
827 }
828
829 fn byte_position(&self) -> usize {
830 self.byte_at(self.idx)
831 }
832
833 /// The byte offset of the character at `index`, or the source's length
834 /// from its end on.
835 fn byte_at(&self, index: usize) -> usize {
836 match &self.byte_indices {
837 Some(offsets) => offsets
838 .get(index - self.low)
839 .copied()
840 .unwrap_or(self.src_len()),
841 None => index.min(self.src_len()),
842 }
843 }
844
845 fn advance(&mut self) -> Option<char> {
846 if self.idx < self.char_len {
847 let c = self.chars[self.idx - self.low];
848 self.idx += 1;
849 if self.char_sets.row.contains(c) {
850 self.ri += 1;
851 self.ci = 1;
852 } else {
853 self.ci += 1;
854 }
855 Some(c)
856 } else {
857 self.touch_end();
858 None
859 }
860 }
861
862 /// Step over one character of a string body, counting a column and
863 /// never a row. `advance` counts a row for any character in
864 /// `line.rowChars`, which is right between tokens and wrong inside a
865 /// string: TypeScript's `buildStringBodySpec` (ts/src/lexer.ts) makes
866 /// a row character plain body there unless the string is multi-line
867 /// and the character is in `line.chars` as well, which is
868 /// [`Lexer::advance_string_line`].
869 fn advance_body(&mut self) -> Option<char> {
870 let c = self.peek()?;
871 self.idx += 1;
872 self.ci += 1;
873 Some(c)
874 }
875
876 /// Step over a line character inside a multi-line string: the column
877 /// resets, and a character in `line.rowChars` counts a row, as
878 /// TypeScript's string-body classes LINE and LINE+ROW do
879 /// (`STRING_BODY_TABLE` in ts/src/lexer.ts) and as Go's
880 /// `BuildStringBodySpec` ports them.
881 fn advance_string_line(&mut self) -> Option<char> {
882 let c = self.peek()?;
883 self.idx += 1;
884 if self.char_sets.row.contains(c) {
885 self.ri += 1;
886 }
887 self.ci = 1;
888 Some(c)
889 }
890
891 fn peek(&self) -> Option<char> {
892 if self.idx < self.char_len {
893 Some(self.chars[self.idx - self.low])
894 } else {
895 self.touch_end();
896 None
897 }
898 }
899
900 fn peek_at(&self, offset: usize) -> Option<char> {
901 let i = self.idx + offset;
902 if i < self.char_len {
903 Some(self.chars[i - self.low])
904 } else {
905 self.touch_end();
906 None
907 }
908 }
909
910 fn wants(&self, tin: crate::Tin) -> bool {
911 self.want
912 .as_ref()
913 .is_none_or(|wanted| wanted.contains(&tin))
914 }
915
916 /// Bring a fetch's point and current character up to the live cursor.
917 ///
918 /// A custom matcher, or an imperative check, may move the cursor and
919 /// then decline: xml's steps over a byte-order mark and goes on to
920 /// look for a tag. Every matcher after it starts where it left the
921 /// cursor, as TypeScript's do (each reads `lex.pnt`) and Go's do (each
922 /// reads `l.pnt`), and the current character is `None` once the cursor
923 /// is at the end of the source. Reading the character the fetch began
924 /// on instead put the built-in matchers' tokens at the wrong place,
925 /// sent them looking for a character that was no longer there, and,
926 /// when the matcher had stepped over the LAST character, panicked on
927 /// the missing one.
928 #[inline]
929 fn resync(&self, point: &mut Point, current: &mut Option<char>) {
930 if self.idx != point.site.pos {
931 *point = self.current_point();
932 *current = self.peek();
933 }
934 }
935
936 /// Whether TypeScript would try a built-in matcher at all in a fetch
937 /// that began on `first` and whose cursor has since been moved by a
938 /// matcher that declined.
939 ///
940 /// TypeScript chooses the matchers a fetch tries from a table indexed
941 /// by the character the fetch begins on (`buildLexDispatch` in
942 /// ts/src/utility.ts, read in `Lex.next`): a built-in is listed for a
943 /// Latin-1 character only when a token it makes could start with it,
944 /// unless it carries a check, and every built-in is listed for a
945 /// character from U+0100 up. The table is not consulted again when a
946 /// custom matcher moves the cursor, so this asks it about `first`, and
947 /// the matcher then tests the character under the cursor as usual.
948 /// Before the cursor moves the two are the same character and the
949 /// matcher's own test is the whole answer, so this is `true`.
950 #[inline]
951 fn listed(
952 &self,
953 entry: usize,
954 first: char,
955 has_check: bool,
956 could_start: impl FnOnce(char) -> bool,
957 ) -> bool {
958 self.idx == entry || u32::from(first) >= 256 || has_check || could_start(first)
959 }
960
961 fn run_check(&mut self, check: Option<LexCheck>, point: Point) -> CheckFlow {
962 let Some(check) = check else {
963 return CheckFlow::Continue;
964 };
965 let at = self.byte_position();
966 let result = match check.run_imperative(self) {
967 Some(result) => result,
968 None => check
969 .run(self.tail_at_byte(at))
970 .unwrap_or(LexCheckResult::Continue),
971 };
972 match result {
973 LexCheckResult::Continue => CheckFlow::Continue,
974 LexCheckResult::Skip => CheckFlow::Skip,
975 LexCheckResult::NativeToken(token) => CheckFlow::Token(token),
976 LexCheckResult::Token(token)
977 if !token.source.is_empty() && self.tail_at_byte(at).starts_with(&token.source) =>
978 {
979 let tin = if token.tin < 0 {
980 self.options.token(&token.name).unwrap_or(token.tin)
981 } else {
982 token.tin
983 };
984 if tin < 0 {
985 return CheckFlow::Skip;
986 }
987 for _ in token.source.chars() {
988 self.advance();
989 }
990 CheckFlow::Token(Box::new(Token::new(
991 token.name,
992 tin,
993 token.value,
994 token.source,
995 point,
996 )))
997 }
998 LexCheckResult::Token(_) => CheckFlow::Skip,
999 }
1000 }
1001
1002 /// Give any plugin matcher whose order is below `before` its turn.
1003 ///
1004 /// Nine sites in the lexer call this per token, once at each stage a
1005 /// matcher is allowed to intervene. A grammar with no custom matcher,
1006 /// which is most of them, was paying nine index lookups per token to be
1007 /// told nine times that there is nothing to run. The guard is inline so
1008 /// those sites skip the call itself; the walk stays out of line.
1009 #[inline]
1010 fn run_custom_matchers(
1011 &mut self,
1012 index: &mut usize,
1013 before: f64,
1014 plugin: &mut Option<(&mut crate::Rule, &mut crate::Context)>,
1015 ) -> Option<Token> {
1016 if *index >= self.options.lex.matchers.len() {
1017 return None;
1018 }
1019 self.run_remaining_custom_matchers(index, before, plugin)
1020 }
1021
1022 #[inline(never)]
1023 fn run_remaining_custom_matchers(
1024 &mut self,
1025 index: &mut usize,
1026 before: f64,
1027 plugin: &mut Option<(&mut crate::Rule, &mut crate::Context)>,
1028 ) -> Option<Token> {
1029 while let Some(matcher) = self
1030 .options
1031 .lex
1032 .matchers
1033 .get_index(*index)
1034 .map(|(_, matcher)| matcher)
1035 .filter(|matcher| matcher.order < before)
1036 .cloned()
1037 {
1038 *index += 1;
1039 // Each matcher starts where the one before it left the cursor:
1040 // one may step over a character and decline, as xml's does over
1041 // a byte-order mark, and TypeScript's matchers all read the
1042 // live `lex.pnt`.
1043 let point = self.current_point();
1044 let at = self.byte_position();
1045 let saved = self.state();
1046 let token = if let Some(callback) = matcher.imperative.as_ref() {
1047 let Some((rule, context)) = plugin.as_mut() else {
1048 continue;
1049 };
1050 callback(self, rule, context)
1051 } else {
1052 let remaining = self.tail_at_byte(at);
1053 matcher
1054 .matcher
1055 .as_ref()
1056 .and_then(|callback| callback(remaining))
1057 .filter(|token| {
1058 !token.source.is_empty() && remaining.starts_with(&token.source)
1059 })
1060 .map(|token| {
1061 Token::new(token.name, token.tin, token.value, token.source, point)
1062 })
1063 };
1064 let Some(mut token) = token else {
1065 if self.want.is_some() {
1066 self.rewind(saved);
1067 }
1068 continue;
1069 };
1070
1071 // TypeScript and Go run opaque custom matchers speculatively for
1072 // a negotiated cut. An unwanted result rolls back locally so a
1073 // later matcher can still satisfy the request.
1074 let tin = if token.tin < 0 {
1075 self.options.token(&token.name).unwrap_or(token.tin)
1076 } else {
1077 token.tin
1078 };
1079 if tin < 0 || !self.wants(tin) {
1080 self.rewind(saved);
1081 continue;
1082 }
1083 if matcher.imperative.is_none() {
1084 for _ in token.src.chars() {
1085 self.advance();
1086 }
1087 }
1088 token.tin = tin;
1089 return Some(token);
1090 }
1091 None
1092 }
1093
1094 fn is_text_delimiter_here(&self) -> bool {
1095 self.is_text_delimiter_at(self.idx)
1096 }
1097
1098 fn is_text_delimiter_at(&self, index: usize) -> bool {
1099 let Some(ch) = self.ch_get(index) else {
1100 return true;
1101 };
1102 let remaining = self.tail_at_byte(self.byte_at(index));
1103 (self.options.space.lex && self.char_sets.space.contains(ch))
1104 || (self.options.fixed.lex
1105 && self
1106 .options
1107 .fixed
1108 .tokens
1109 .values()
1110 .any(|token| !token.source.is_empty() && remaining.starts_with(&token.source)))
1111 || (self.options.line.lex
1112 && (self.char_sets.line_ends.contains(ch) || matches!(ch, '\u{2028}' | '\u{2029}')))
1113 || (self.options.comment.lex
1114 && self.options.comment.definitions.values().any(|definition| {
1115 definition.lex
1116 && !definition.start.is_empty()
1117 && remaining.starts_with(&definition.start)
1118 }))
1119 || self
1120 .options
1121 .ender
1122 .iter()
1123 .any(|ender| !ender.is_empty() && remaining.starts_with(ender))
1124 }
1125
1126 /// Fetches the next non-IGNORE token (skipping spaces, lines, comments).
1127 pub fn next_token(&mut self) -> Result<Token, TabnasError> {
1128 let point = self.current_point();
1129 let result = match catch_unwind(AssertUnwindSafe(|| {
1130 if let Some(ref error) = self.err {
1131 return Err(Box::new(error.clone()));
1132 }
1133
1134 loop {
1135 let token = self.next_raw(None)?;
1136 if !self.ignore_tins.contains(&token.tin) {
1137 return Ok(token);
1138 }
1139 }
1140 })) {
1141 Ok(result) => result,
1142 Err(payload) => self.record_panic(payload, "Lexer::next_token", point),
1143 };
1144 // The box is internal to the engine; a caller gets the error itself.
1145 result.map_err(|error| *error)
1146 }
1147
1148 /// Fetch the next token without discarding whitespace, line, or comment tokens.
1149 pub fn next_raw_token(&mut self) -> Result<Token, TabnasError> {
1150 let point = self.current_point();
1151 let result = match catch_unwind(AssertUnwindSafe(|| self.next_raw(None))) {
1152 Ok(result) => result,
1153 Err(payload) => self.record_panic(payload, "Lexer::next_raw_token", point),
1154 };
1155 result.map_err(|error| *error)
1156 }
1157
1158 /// Fetch one token for an imperative parser callback, preserving ignored
1159 /// space/line/comment tokens just like TypeScript's public `lex.next`.
1160 /// Replayed tokens produced by `Context::rewind` are served first.
1161 pub fn next_raw_for_rule(
1162 &mut self,
1163 rule: &mut crate::Rule,
1164 context: &mut crate::Context,
1165 ) -> LexResult<Token> {
1166 if let Some(token) = context.next_replay() {
1167 Ok(token)
1168 } else {
1169 self.next_raw_with(None, Some((rule, context)))
1170 }
1171 }
1172
1173 /// Fetch the next non-ignored token for an imperative parser callback.
1174 pub fn next_for_rule(
1175 &mut self,
1176 rule: &mut crate::Rule,
1177 context: &mut crate::Context,
1178 ) -> LexResult<Token> {
1179 loop {
1180 let token = self.next_raw_for_rule(rule, context)?;
1181 if !self.ignore_tins.contains(&token.tin) {
1182 return Ok(token);
1183 }
1184 }
1185 }
1186
1187 /// Public negotiated-relex entry point for native parser callbacks.
1188 /// A successful recut commits the lexer cursor and returns an opaque undo
1189 /// checkpoint; a failed recut restores all lexer state before returning.
1190 pub fn relex_for_rule(
1191 &mut self,
1192 from: &Token,
1193 wanted: &[crate::Tin],
1194 rule: &mut crate::Rule,
1195 context: &mut crate::Context,
1196 ) -> Option<(Token, RelexCheckpoint)> {
1197 self.relex(from, wanted, rule, context)
1198 }
1199
1200 /// Undo a committed [`Lexer::relex_for_rule`] operation, including the
1201 /// pending tokens hidden while the replacement cut was negotiated.
1202 pub fn unrelex(&mut self, checkpoint: RelexCheckpoint, context: &mut crate::Context) {
1203 self.restore(checkpoint.state);
1204 context.restore_replay(checkpoint.replay);
1205 }
1206
1207 fn record_panic(
1208 &mut self,
1209 payload: Box<dyn std::any::Any + Send>,
1210 api: &str,
1211 point: Point,
1212 ) -> LexResult<Token> {
1213 let error = TabnasError::from_panic(
1214 payload,
1215 api,
1216 self.text(),
1217 point.site.pos,
1218 point.site.ri,
1219 point.site.ci,
1220 &self.options,
1221 );
1222 self.err = Some(error.clone());
1223 Err(Box::new(error))
1224 }
1225
1226 /// Fetch a raw token while restricting non-eager custom token matchers to
1227 /// the exact tins accepted at the parser slot being filled. Builtin and
1228 /// fixed-token matchers are unaffected by this gate.
1229 pub(crate) fn next_rule_token(
1230 &mut self,
1231 expected_match_tins: &[crate::Tin],
1232 rule: &mut crate::Rule,
1233 context: &mut crate::Context,
1234 ) -> LexResult<Token> {
1235 self.next_raw_with(Some(expected_match_tins), Some((rule, context)))
1236 }
1237
1238 /// Step past a bad token the parser has absorbed or skipped, as
1239 /// TypeScript's `advanceLexPast` does (ts/src/rules.ts): a bad token
1240 /// does not advance the cursor by itself, so recovery moves it to the
1241 /// end of the token's span, never backwards, and, for a fault raised
1242 /// inside a compound construct, on past the next row character so
1243 /// lexing resumes on a fresh row. The lexer's own faults latch until
1244 /// this clears them.
1245 pub(crate) fn skip_bad(&mut self, token: &Token, to_line_end: bool) {
1246 let span = token.src.chars().count().max(1);
1247 let mut target = self.idx.max(token.site.pos.saturating_add(span));
1248 if to_line_end {
1249 let mut end = target;
1250 while end < self.char_len && !self.char_sets.row.contains(self.ch(end)) {
1251 end += 1;
1252 }
1253 target = target.max(self.char_len.min(end + 1));
1254 }
1255 while self.idx < target && self.idx < self.char_len {
1256 self.advance();
1257 }
1258 self.err = None;
1259 if self.idx < self.char_len {
1260 self.end_reached = false;
1261 }
1262 }
1263
1264 /// Re-cut an already buffered source span, constrained to the token
1265 /// identities requested by one alternate. On success the cursor remains
1266 /// after the new cut; the returned state can restore the original cut if
1267 /// that alternate later fails.
1268 pub(crate) fn relex(
1269 &mut self,
1270 from: &Token,
1271 wanted: &[crate::Tin],
1272 rule: &mut crate::Rule,
1273 context: &mut crate::Context,
1274 ) -> Option<(Token, RelexCheckpoint)> {
1275 if from.src.is_empty() || from.site.pos > self.char_len || wanted.is_empty() {
1276 return None;
1277 }
1278 // The standing error, if any, is moved into the checkpoint rather
1279 // than copied: the cut below clears it anyway, and a copy is the
1280 // length of the source (`full_source`) for every cut attempted.
1281 let err = self.err.take();
1282 let saved = LexerState {
1283 err,
1284 ..self.state()
1285 };
1286 // TypeScript temporarily replaces the lexer's pending-token queue
1287 // with an empty queue for a negotiated cut. Rust keeps that queue on
1288 // Context, so hide it explicitly and preserve it in the checkpoint.
1289 let replay = context.take_replay();
1290 self.idx = from.site.pos;
1291 self.ri = from.site.ri;
1292 self.ci = from.site.ci;
1293 self.err = None;
1294 self.end_reached = false;
1295 self.want = Some(wanted.to_vec());
1296 // Straight to the matchers, past `next_raw_with`: an error here only
1297 // rejects the cut, and the restore below puts back the lexer's own,
1298 // so the source it would attach, and the copy it would keep, are
1299 // never seen. With them, every rejected cut cost the length of the
1300 // source, and a flat stylesheet parsed in quadratic time.
1301 let recut = self.next_raw_inner(None, Some((rule, context))).ok();
1302 self.want = None;
1303 match recut.filter(|token| wanted.contains(&token.tin)) {
1304 Some(mut token) => {
1305 token.ignored = from.ignored.clone();
1306 Some((
1307 token,
1308 RelexCheckpoint {
1309 state: saved,
1310 replay,
1311 },
1312 ))
1313 }
1314 None => {
1315 self.restore(saved);
1316 // Discard any speculative replay generated by an imperative
1317 // matcher and restore the queue that preceded the attempt.
1318 context.restore_replay(replay);
1319 None
1320 }
1321 }
1322 }
1323
1324 pub(crate) fn restore(&mut self, state: LexerState) {
1325 self.idx = state.idx;
1326 self.ri = state.ri;
1327 self.ci = state.ci;
1328 self.err = state.err;
1329 self.end_reached = state.end_reached;
1330 self.want = None;
1331 }
1332
1333 /// Put the cursor back after a custom matcher's speculative attempt and
1334 /// keep the request being negotiated, as TypeScript's `Lex.speculate`
1335 /// restores the point and leaves `want` alone. [`Lexer::restore`] ends
1336 /// a negotiation, so the first custom matcher to decline during a
1337 /// re-cut used to lift the request for every matcher after it, and a
1338 /// string matcher could then cut the `#ST` the re-cut was there to
1339 /// avoid.
1340 fn rewind(&mut self, state: LexerState) {
1341 let want = self.want.take();
1342 self.restore(state);
1343 self.want = want;
1344 }
1345
1346 fn next_raw(&mut self, expected_match_tins: Option<&[crate::Tin]>) -> LexResult<Token> {
1347 // Only a lexer being driven directly needs these, and building
1348 // them costs a whole `Options` clone. A parse reaches the lexer
1349 // through `next_rule_token`, which brings the real rule and
1350 // context with it, so it never wants them at all.
1351 let (mut rule, mut context) = match self.standalone.take() {
1352 Some(pair) => pair,
1353 None => (
1354 crate::Rule::new("#NORULE", Value::Undefined),
1355 crate::Context::new(
1356 self.options.rewind.history,
1357 self.text(),
1358 Value::Undefined,
1359 Arc::clone(&self.options),
1360 crate::InstanceInfo::default(),
1361 ),
1362 ),
1363 };
1364 let result = self.next_raw_with(expected_match_tins, Some((&mut rule, &mut context)));
1365 self.standalone = Some((rule, context));
1366 result
1367 }
1368
1369 fn modify_text_value(
1370 &mut self,
1371 mut value: Value,
1372 plugin: &mut Option<(&mut crate::Rule, &mut crate::Context)>,
1373 ) -> Value {
1374 if self.options.text.modify.is_empty() {
1375 return value;
1376 }
1377 let modifiers = self.options.text.modify.clone();
1378 let options = self.options.clone();
1379 let Some((rule, context)) = plugin.as_mut() else {
1380 panic!("imperative text modifier requires an active lexer context");
1381 };
1382 for modifier in modifiers {
1383 value = modifier.run(value, self, rule, context, &options);
1384 }
1385 value
1386 }
1387
1388 fn next_raw_with(
1389 &mut self,
1390 expected_match_tins: Option<&[crate::Tin]>,
1391 plugin: Option<(&mut crate::Rule, &mut crate::Context)>,
1392 ) -> LexResult<Token> {
1393 let result = self.next_raw_inner(expected_match_tins, plugin);
1394 match result {
1395 Ok(token) => Ok(token),
1396 Err(mut error) => {
1397 // The matchers build their errors without the source, and it
1398 // is attached here, where an error leaves the lexer: a
1399 // negotiated cut (`relex`) rejects many candidates, each an
1400 // error nobody sees, and a copy of the whole source apiece
1401 // made that quadratic. A parse from a reader attaches the
1402 // window instead, once the lexer is done and the window has
1403 // read on to the end of its line
1404 // ([`crate::lexer::ReaderSource`]).
1405 if !self.windowed() {
1406 let (text, row, col) = self.excerpt();
1407 error.set_excerpt(text, row, col);
1408 }
1409 error.apply_options(&self.options);
1410 self.err = Some((*error).clone());
1411 Err(error)
1412 }
1413 }
1414 }
1415
1416 /// Cut the next token. In a window, the window is filled to its
1417 /// margin ahead of the cursor first, and a token whose cut read to the
1418 /// window's end while the source had more is cut again over a window
1419 /// twice as far ahead, so that it is the token the whole text gives.
1420 fn next_raw_inner(
1421 &mut self,
1422 expected_match_tins: Option<&[crate::Tin]>,
1423 mut plugin: Option<(&mut crate::Rule, &mut crate::Context)>,
1424 ) -> LexResult<Token> {
1425 let Text::Window(window) = &self.src else {
1426 return self.next_raw_cut(expected_match_tins, plugin);
1427 };
1428 let mut ahead = window.margin;
1429 loop {
1430 self.fill_window(ahead);
1431 if let Some(failure) = self.unreadable() {
1432 return Err(Box::new(failure.error()));
1433 }
1434 let saved = self.state();
1435 self.touched_end.set(false);
1436 let result = self.next_raw_cut(
1437 expected_match_tins,
1438 plugin
1439 .as_mut()
1440 .map(|(rule, context)| (&mut **rule, &mut **context)),
1441 );
1442 // The cut stands when nothing read to the window's end while
1443 // the source had more, and it ended inside the window or the
1444 // source is all read. A read that touched the end decided on a
1445 // window shorter than the source, even where a later matcher's
1446 // fill has since read the source to its end: made again, over
1447 // the window that fill left, it touches nothing and stands
1448 // (#326).
1449 if !self.touched_end.get() && (!self.window_open() || self.idx < self.char_len) {
1450 return result;
1451 }
1452 self.rewind(saved);
1453 ahead = ahead.saturating_mul(2);
1454 }
1455 }
1456
1457 fn next_raw_cut(
1458 &mut self,
1459 expected_match_tins: Option<&[crate::Tin]>,
1460 mut plugin: Option<(&mut crate::Rule, &mut crate::Context)>,
1461 ) -> LexResult<Token> {
1462 if self.end_reached {
1463 return Ok(Token::new(
1464 "#ZZ",
1465 TIN_ZZ,
1466 Value::Undefined,
1467 "",
1468 self.current_point(),
1469 ));
1470 }
1471
1472 if self.idx >= self.char_len {
1473 self.end_reached = true;
1474 return Ok(Token::new(
1475 "#ZZ",
1476 TIN_ZZ,
1477 Value::Undefined,
1478 "",
1479 self.current_point(),
1480 ));
1481 }
1482
1483 // The fetch begins on `first`, at `entry`. `pnt` and `c`, the point
1484 // a token starts at and the character under the cursor, follow the
1485 // cursor whenever a custom matcher or a check moves it and declines
1486 // (`resync`); `c` is `None` once the cursor is at the end.
1487 let entry = self.idx;
1488 let first = self.ch(entry);
1489 let mut pnt = self.current_point();
1490 let mut c = Some(first);
1491 let mut custom_index = 0;
1492
1493 if let Some(token) = self.run_custom_matchers(&mut custom_index, 1_000_000.0, &mut plugin) {
1494 return Ok(token);
1495 }
1496 self.resync(&mut pnt, &mut c);
1497
1498 // User-declared match tokens occupy the 1e6 matcher priority band.
1499 let match_skipped = if self.options.match_lex
1500 && (!self.options.match_values.is_empty()
1501 || self
1502 .options
1503 .match_tokens
1504 .values()
1505 .any(|matcher| self.wants(matcher.tin)))
1506 {
1507 match self.run_check(self.options.match_check.clone(), pnt) {
1508 CheckFlow::Continue => false,
1509 CheckFlow::Skip => true,
1510 CheckFlow::Token(token) => return Ok(*token),
1511 }
1512 } else {
1513 false
1514 };
1515 self.resync(&mut pnt, &mut c);
1516 let remaining = self.tail_at_byte(self.byte_position());
1517 let custom_value = (self.options.match_lex && !match_skipped && self.want.is_none())
1518 .then(|| {
1519 self.options
1520 .match_values
1521 .values()
1522 .find_map(|matcher| match &matcher.matcher {
1523 MatchTokenMatcher::Callback(callback) => callback(remaining)
1524 .filter(|result| {
1525 !result.source.is_empty() && remaining.starts_with(&result.source)
1526 })
1527 .map(|result| (result.source, result.value)),
1528 MatchTokenMatcher::Regex(regex) => {
1529 let captures = regex.captures(remaining)?;
1530 let found = captures
1531 .get(0)
1532 .filter(|found| found.start() == 0 && !found.as_str().is_empty())?;
1533 let source = found.as_str().to_string();
1534 let value = matcher.transform.as_ref().map_or_else(
1535 || {
1536 matcher
1537 .val
1538 .clone()
1539 .unwrap_or_else(|| Value::String(source.clone()))
1540 },
1541 |transform| {
1542 let groups = captures
1543 .iter()
1544 .map(|capture| {
1545 capture.map_or_else(String::new, |value| {
1546 value.as_str().into()
1547 })
1548 })
1549 .collect::<Vec<_>>();
1550 transform(&groups)
1551 },
1552 );
1553 Some((source, value))
1554 }
1555 })
1556 })
1557 .flatten();
1558 if let Some((source, value)) = custom_value {
1559 for _ in source.chars() {
1560 self.advance();
1561 }
1562 return Ok(Token::new("#VL", TIN_VL, value, source, pnt));
1563 }
1564
1565 let remaining = self.tail_at_byte(self.byte_position());
1566 // With no custom matcher there is nothing for the band to do: both
1567 // passes walk an empty table and yield nothing, and `fix_len` is
1568 // read only by that walk. Most grammars register none, and every
1569 // token fetch of theirs paid the eager pass's scan of the fixed
1570 // table (one closure call per fixed literal) to arrive at the
1571 // `None` this guard now hands over directly. TS `makeMatchMatcher`
1572 // returns null on an empty table (ts/src/lexer.ts) and the band is
1573 // never installed; Go reaches the same place by defaulting
1574 // `MatchLex` off unless `Options.Match` is set. Rust defaults
1575 // `match_lex` true as TS does, so the guard is the parity.
1576 let custom = (self.options.match_lex
1577 && !match_skipped
1578 && !self.options.match_tokens.is_empty())
1579 .then(|| {
1580 // Two passes, position-expected before eager, as go/lexer.go
1581 // matchMatch and ts/src/lexer.ts makeMatchMatcher both make.
1582 // One tin-ordered pass in which eagerness merely bypassed the
1583 // slot gate let an eager matcher EARLIER in tin order win over
1584 // an expected one later: with `p = %x31-39` beside
1585 // `d = %x30-39`, the `2` of `12` lexed as the narrower class
1586 // the `*d` loop never asked for. Eagerness is for firing where
1587 // the slot's list is narrower than the grammar, never for
1588 // outbidding what the slot names.
1589 //
1590 // Under a want the alternate's own tin list is the sharper
1591 // gate, so one filtered pass is the whole search. With no
1592 // expected list at all (a standalone lexer, no rule) nothing
1593 // constrains the caller and every matcher is eligible in the
1594 // first pass.
1595 // The longest FIXED literal this slot expects that matches
1596 // here, or 0. Only the eager pass consults it: there, a
1597 // literal the slot names beats an eager-only matcher that
1598 // cuts no further than it does. Without this, a character
1599 // class that CONTAINS a literal the grammar also uses
1600 // swallows it wherever the class is eager (`num = "0" /
1601 // posdigit *digit` beside `digit = %x30-39` rejected
1602 // `0.0.0`). LENGTH decides, not mere existence, so a keyword
1603 // literal cannot truncate a longer word: ties go to the
1604 // literal, and an eager matcher that cuts further still
1605 // wins. TS and Go do the same, in makeMatchMatcher and
1606 // matchMatch.
1607 //
1608 // Computed once per fetch and only when a regex matcher in the
1609 // eager pass has something to weigh against it, as TS
1610 // `expectedFixedLen` does (`fixLen = -1` until asked). The
1611 // scan is the whole fixed table against the slot's list; an
1612 // expected matcher that wins in pass 0, or a fetch under a
1613 // want, never needs it. Nothing the scan reads changes
1614 // between the two passes, so lazy equals eager.
1615 let mut fix_len: Option<usize> = None;
1616 let compute_fix_len = || {
1617 if self.want.is_none() && self.options.fixed.lex {
1618 expected_match_tins.map_or(0, |expected| {
1619 self.options
1620 .fixed
1621 .tokens
1622 .values()
1623 .filter(|token| {
1624 !token.source.is_empty()
1625 && expected.contains(&token.tin)
1626 && remaining.starts_with(&token.source)
1627 })
1628 .map(|token| token.source.len())
1629 .max()
1630 .unwrap_or(0)
1631 })
1632 } else {
1633 0
1634 }
1635 };
1636 let passes = if self.want.is_some() { 1 } else { 2 };
1637 (0..passes).find_map(|pass| {
1638 self.options.match_tokens.values().find_map(|matcher| {
1639 if !self.wants(matcher.tin) {
1640 return None;
1641 }
1642 if self.want.is_none() {
1643 let expected = expected_match_tins
1644 .is_none_or(|expected| expected.contains(&matcher.tin));
1645 if pass == 0 {
1646 if !expected {
1647 return None;
1648 }
1649 } else if expected || !matcher.eager {
1650 return None;
1651 }
1652 }
1653 let result = match &matcher.matcher {
1654 MatchTokenMatcher::Regex(regex) => regex
1655 .find(remaining)
1656 .filter(|found| found.start() == 0)
1657 // The eager pass yields to an expected
1658 // literal it cannot out-cut; the fixed
1659 // matcher (2e6) runs next and takes it. See
1660 // `fix_len` above.
1661 .filter(|found| {
1662 pass == 0 || {
1663 let fix_len = *fix_len.get_or_insert_with(compute_fix_len);
1664 fix_len == 0 || found.len() > fix_len
1665 }
1666 })
1667 .map(|found| {
1668 let source = found.as_str().to_string();
1669 (source.clone(), Value::String(source))
1670 }),
1671 MatchTokenMatcher::Callback(callback) => callback(remaining)
1672 .filter(|result| {
1673 !result.source.is_empty() && remaining.starts_with(&result.source)
1674 })
1675 .map(|result| (result.source, result.value)),
1676 };
1677 result.map(|(source, value)| (matcher.name.clone(), matcher.tin, source, value))
1678 })
1679 })
1680 });
1681 if let Some(Some((name, tin, matched, value))) = custom {
1682 for _ in matched.chars() {
1683 self.advance();
1684 }
1685 return Ok(Token::new(name, tin, value, matched, pnt));
1686 }
1687
1688 if let Some(token) = self.run_custom_matchers(&mut custom_index, 2_000_000.0, &mut plugin) {
1689 return Ok(token);
1690 }
1691 self.resync(&mut pnt, &mut c);
1692
1693 // Fixed literals occupy the 2e6 band and use longest-match wins.
1694 let fixed_skipped = if self.options.fixed.lex {
1695 match self.run_check(self.options.fixed.check.clone(), pnt) {
1696 CheckFlow::Continue => false,
1697 CheckFlow::Skip => true,
1698 CheckFlow::Token(token) => return Ok(*token),
1699 }
1700 } else {
1701 false
1702 };
1703 self.resync(&mut pnt, &mut c);
1704 let fixed_skipped = fixed_skipped
1705 || !self.listed(entry, first, self.options.fixed.check.is_some(), |ch| {
1706 self.options
1707 .fixed
1708 .tokens
1709 .values()
1710 .any(|token| token.source.starts_with(ch))
1711 });
1712 let remaining = self.tail_at_byte(self.byte_position());
1713 // The winner is carried out of the table as its position, not as a
1714 // copy of its text. `Token::new` takes the name and the source text
1715 // by reference and stores both inline, so the only owned copy the
1716 // token needs is the one inside `Value::String`. Naming the match
1717 // as three owned values cost three `String` allocations per fixed
1718 // token, two of them freed again before the token was built.
1719 // The first byte decides almost every entry. Asking `wants` and then
1720 // `starts_with` of each fixed token in turn ran a tin lookup and a
1721 // `memcmp` per token in the grammar per token in the input, and a
1722 // grammar with fifty fixed tokens pays fifty of each to reject
1723 // forty-nine. One byte answers the same question, and an empty
1724 // source is kept out by its own check, which only entries that
1725 // already matched the byte ever reach.
1726 let first_byte = remaining.as_bytes().first().copied();
1727 if self.options.fixed.lex && !fixed_skipped && self.window_open() {
1728 // A literal longer than what is left of a window, which what is
1729 // left starts, may yet be there once the window has grown: the
1730 // read meets the window's end, whatever this matcher decides,
1731 // so that a later matcher's fill cannot let the cut stand on it
1732 // (#326).
1733 if self.options.fixed.tokens.values().any(|token| {
1734 remaining.len() < token.source.len() && token.source.starts_with(remaining)
1735 }) {
1736 self.touch_end();
1737 }
1738 }
1739 let fixed = (self.options.fixed.lex && !fixed_skipped)
1740 .then(|| {
1741 self.options
1742 .fixed
1743 .tokens
1744 .values()
1745 .enumerate()
1746 .filter(|(_, token)| {
1747 token.source.as_bytes().first().copied() == first_byte
1748 && !token.source.is_empty()
1749 && self.wants(token.tin)
1750 && remaining.starts_with(&token.source)
1751 })
1752 .max_by_key(|(_, token)| token.source.len())
1753 .map(|(index, token)| (index, token.source.chars().count()))
1754 })
1755 .flatten();
1756 if let Some((index, source_chars)) = fixed {
1757 for _ in 0..source_chars {
1758 self.advance();
1759 }
1760 let (_, token) = self
1761 .options
1762 .fixed
1763 .tokens
1764 .get_index(index)
1765 .expect("index came from this table, which nothing writes to mid-parse");
1766 // A fixed token has no value, as in TypeScript and Go: its
1767 // text is its `src`.
1768 return Ok(Token::new(
1769 &token.name,
1770 token.tin,
1771 Value::Undefined,
1772 token.source.as_str(),
1773 pnt,
1774 ));
1775 }
1776
1777 if let Some(token) = self.run_custom_matchers(&mut custom_index, 3_000_000.0, &mut plugin) {
1778 return Ok(token);
1779 }
1780 self.resync(&mut pnt, &mut c);
1781
1782 // 1. Whitespace
1783 let space_skipped = if self.options.space.lex && self.wants(TIN_SP) {
1784 match self.run_check(self.options.space.check.clone(), pnt) {
1785 CheckFlow::Continue => false,
1786 CheckFlow::Skip => true,
1787 CheckFlow::Token(token) => return Ok(*token),
1788 }
1789 } else {
1790 false
1791 };
1792 self.resync(&mut pnt, &mut c);
1793 if self.options.space.lex
1794 && !space_skipped
1795 && self.wants(TIN_SP)
1796 && c.is_some_and(|ch| self.char_sets.space.contains(ch))
1797 && self.listed(entry, first, self.options.space.check.is_some(), |ch| {
1798 self.char_sets.space.contains(ch)
1799 })
1800 {
1801 let mut src = String::new();
1802 while let Some(ch) = self.peek() {
1803 if self.char_sets.space.contains(ch) {
1804 src.push(ch);
1805 self.advance();
1806 } else {
1807 break;
1808 }
1809 }
1810 return Ok(Token::new(
1811 "#SP",
1812 TIN_SP,
1813 Value::String(src.clone()),
1814 src,
1815 pnt,
1816 ));
1817 }
1818
1819 if let Some(token) = self.run_custom_matchers(&mut custom_index, 4_000_000.0, &mut plugin) {
1820 return Ok(token);
1821 }
1822 self.resync(&mut pnt, &mut c);
1823
1824 // 2. Line ending
1825 let line_skipped = if self.options.line.lex && self.wants(TIN_LN) {
1826 match self.run_check(self.options.line.check.clone(), pnt) {
1827 CheckFlow::Continue => false,
1828 CheckFlow::Skip => true,
1829 CheckFlow::Token(token) => return Ok(*token),
1830 }
1831 } else {
1832 false
1833 };
1834 self.resync(&mut pnt, &mut c);
1835 let line_skipped = line_skipped
1836 || !self.listed(entry, first, self.options.line.check.is_some(), |ch| {
1837 self.char_sets.line_ends.contains(ch)
1838 });
1839 if self.options.line.lex
1840 && !line_skipped
1841 && self.wants(TIN_LN)
1842 && c.is_some_and(|ch| self.char_sets.line_ends.contains(ch))
1843 {
1844 let mut src = String::new();
1845 let mut seen = std::collections::HashSet::new();
1846 while let Some(ch) = self.peek() {
1847 if !self.char_sets.line_ends.contains(ch) {
1848 break;
1849 }
1850 if self.options.line.single && !seen.insert(ch) {
1851 break;
1852 }
1853 src.push(self.advance().expect("peeked character must advance"));
1854 }
1855 self.ci = 1;
1856 return Ok(Token::new(
1857 "#LN",
1858 TIN_LN,
1859 Value::String(src.clone()),
1860 src,
1861 pnt,
1862 ));
1863 }
1864
1865 if let Some(bad_char @ ('\u{2028}' | '\u{2029}')) = c {
1866 if self.options.line.lex && !line_skipped && self.wants(TIN_LN) {
1867 self.advance();
1868 let err = TabnasError::new(
1869 "unexpected",
1870 bad_char.to_string(),
1871 "",
1872 pnt.site.pos,
1873 pnt.site.ri,
1874 pnt.site.ci,
1875 );
1876 self.err = Some(err.clone());
1877 return Err(Box::new(err));
1878 }
1879 }
1880
1881 if let Some(token) = self.run_custom_matchers(&mut custom_index, 5_000_000.0, &mut plugin) {
1882 return Ok(token);
1883 }
1884 self.resync(&mut pnt, &mut c);
1885
1886 // 3. Quoted strings. These precede comments in the canonical matcher
1887 // order, so an overlapping quote/comment opener is a string unless
1888 // string matching explicitly abandons the malformed candidate.
1889 let string_skipped = if self.options.string.lex && self.wants(TIN_ST) {
1890 match self.run_check(self.options.string.check.clone(), pnt) {
1891 CheckFlow::Continue => false,
1892 CheckFlow::Skip => true,
1893 CheckFlow::Token(token) => return Ok(*token),
1894 }
1895 } else {
1896 false
1897 };
1898 self.resync(&mut pnt, &mut c);
1899 let quote = c.filter(|ch| self.char_sets.string.contains(*ch));
1900 if let Some(quote) = quote.filter(|_| {
1901 self.options.string.lex
1902 && !string_skipped
1903 && self.wants(TIN_ST)
1904 && self.listed(entry, first, self.options.string.check.is_some(), |ch| {
1905 self.char_sets.string.contains(ch)
1906 })
1907 }) {
1908 let start = (self.idx, self.ri, self.ci);
1909 match self.match_string(quote, pnt) {
1910 result @ Ok(_) => return result,
1911 Err(error) if !self.options.string.abandon => return Err(error),
1912 Err(_) => {
1913 (self.idx, self.ri, self.ci) = start;
1914 self.err = None;
1915 }
1916 }
1917 }
1918
1919 if let Some(token) = self.run_custom_matchers(&mut custom_index, 6_000_000.0, &mut plugin) {
1920 return Ok(token);
1921 }
1922 self.resync(&mut pnt, &mut c);
1923
1924 // 4. Comments (longest opening marker wins; ties sort by name).
1925 let comment_skipped = if self.options.comment.lex && self.wants(TIN_CM) {
1926 match self.run_check(self.options.comment.check.clone(), pnt) {
1927 CheckFlow::Continue => false,
1928 CheckFlow::Skip => true,
1929 CheckFlow::Token(token) => return Ok(*token),
1930 }
1931 } else {
1932 false
1933 };
1934 self.resync(&mut pnt, &mut c);
1935 if self.options.comment.lex
1936 && !comment_skipped
1937 && self.wants(TIN_CM)
1938 && self.listed(entry, first, self.options.comment.check.is_some(), |ch| {
1939 self.options
1940 .comment
1941 .definitions
1942 .values()
1943 .any(|definition| definition.start.starts_with(ch))
1944 })
1945 {
1946 if let Some(token) = self.match_comment(pnt)? {
1947 return Ok(token);
1948 }
1949 }
1950
1951 if let Some(token) = self.run_custom_matchers(&mut custom_index, 7_000_000.0, &mut plugin) {
1952 return Ok(token);
1953 }
1954 self.resync(&mut pnt, &mut c);
1955
1956 // 5. Numbers
1957 let number_skipped = if self.options.number.lex && self.wants(TIN_NR) {
1958 match self.run_check(self.options.number.check.clone(), pnt) {
1959 CheckFlow::Continue => false,
1960 CheckFlow::Skip => true,
1961 CheckFlow::Token(token) => return Ok(*token),
1962 }
1963 } else {
1964 false
1965 };
1966 self.resync(&mut pnt, &mut c);
1967 let could_start_number =
1968 |ch: char| ch == '-' || ch == '+' || ch == '.' || ch.is_ascii_digit();
1969 if self.options.number.lex
1970 && !number_skipped
1971 && self.wants(TIN_NR)
1972 && c.is_some_and(could_start_number)
1973 && self.listed(
1974 entry,
1975 first,
1976 self.options.number.check.is_some(),
1977 could_start_number,
1978 )
1979 {
1980 if let Some(tkn) = self.match_number(pnt)? {
1981 return Ok(tkn);
1982 }
1983 }
1984
1985 if let Some(token) = self.run_custom_matchers(&mut custom_index, 8_000_000.0, &mut plugin) {
1986 return Ok(token);
1987 }
1988 self.resync(&mut pnt, &mut c);
1989
1990 // 6. Text and named/regex values share the same delimited run.
1991 // Negotiated lexing gates this combined family by its primary token
1992 // identity (#TX), matching the TypeScript and Go dispatchers. Once
1993 // entered, an exact or regexp value definition may still produce
1994 // #VL; the caller rejects and rolls that cut back when #VL was not
1995 // requested.
1996 let text_matcher_wanted = self.wants(TIN_TX);
1997 let value_lex = self.options.value.lex && text_matcher_wanted;
1998 let text_lex = self.options.text.lex && text_matcher_wanted;
1999 let text_skipped = if text_lex || value_lex {
2000 match self.run_check(self.options.text.check.clone(), pnt) {
2001 CheckFlow::Continue => false,
2002 CheckFlow::Skip => true,
2003 CheckFlow::Token(token) => return Ok(*token),
2004 }
2005 } else {
2006 false
2007 };
2008 self.resync(&mut pnt, &mut c);
2009 if (text_lex || value_lex) && !text_skipped && !self.is_text_delimiter_here() {
2010 let start = (self.idx, self.ri, self.ci);
2011 // Only a `value` definition declaring `consume` looks at the
2012 // rest of the document, and the JSON grammar has none: the
2013 // tail is sliced where one asks for it, from the offset the
2014 // token started at, never copied.
2015 let start_byte = self.byte_position();
2016 let mut src = String::new();
2017 while let Some(ch) = self.peek() {
2018 if self.is_text_delimiter_here() {
2019 break;
2020 }
2021 src.push(ch);
2022 self.advance();
2023 }
2024
2025 let mut output = None;
2026 if value_lex {
2027 if let Some(definition) = self
2028 .options
2029 .value
2030 .definitions
2031 .get(&src)
2032 .filter(|definition| definition.matcher.is_none())
2033 .cloned()
2034 {
2035 output = Some(Token::new(
2036 "#VL",
2037 TIN_VL,
2038 definition
2039 .val
2040 .clone()
2041 .unwrap_or_else(|| Value::String(src.clone())),
2042 src.clone(),
2043 pnt,
2044 ));
2045 }
2046
2047 if output.is_none() {
2048 let mut definitions: Vec<_> = self
2049 .options
2050 .value
2051 .definitions
2052 .iter()
2053 .filter(|(_, definition)| definition.matcher.is_some())
2054 .map(|(name, definition)| (name.clone(), definition.clone()))
2055 .collect();
2056 definitions.sort_by(|(name_a, _), (name_b, _)| name_a.cmp(name_b));
2057 for (_, definition) in definitions {
2058 let regex = definition.matcher.as_ref().expect("filtered matcher");
2059 let target: &str = if definition.consume {
2060 self.tail_at_byte(start_byte)
2061 } else {
2062 &src
2063 };
2064 let Some(captures) = regex.captures(target) else {
2065 continue;
2066 };
2067 let Some(found) = captures.get(0).filter(|found| found.start() == 0) else {
2068 continue;
2069 };
2070 if !definition.consume && found.end() != target.len() {
2071 continue;
2072 }
2073 let matched = found.as_str().to_string();
2074 let value = definition.transform.as_ref().map_or_else(
2075 || {
2076 definition
2077 .val
2078 .clone()
2079 .unwrap_or_else(|| Value::String(matched.clone()))
2080 },
2081 |transform| {
2082 let groups = captures
2083 .iter()
2084 .map(|capture| {
2085 capture
2086 .map_or_else(String::new, |value| value.as_str().into())
2087 })
2088 .collect::<Vec<_>>();
2089 transform(&groups)
2090 },
2091 );
2092 if definition.consume {
2093 (self.idx, self.ri, self.ci) = start;
2094 for _ in matched.chars() {
2095 self.advance();
2096 }
2097 }
2098 output = Some(Token::new("#VL", TIN_VL, value, matched, pnt));
2099 break;
2100 }
2101 }
2102 }
2103
2104 if output.is_none() && (!text_lex || text_skipped) {
2105 (self.idx, self.ri, self.ci) = start;
2106 } else if output.is_none() {
2107 output = Some(Token::new(
2108 "#TX",
2109 TIN_TX,
2110 Value::String(src.clone()),
2111 src,
2112 pnt,
2113 ));
2114 }
2115
2116 if let Some(mut token) = output {
2117 let value = std::mem::replace(&mut token.val, Value::Undefined);
2118 token.val = self.modify_text_value(value, &mut plugin);
2119 return Ok(token);
2120 }
2121 }
2122
2123 if let Some(token) = self.run_custom_matchers(&mut custom_index, f64::INFINITY, &mut plugin)
2124 {
2125 return Ok(token);
2126 }
2127 self.resync(&mut pnt, &mut c);
2128
2129 // 7. Unclaimed character -> Error: unexpected, raised where the
2130 // cursor stands, as TypeScript's `Lex.next` builds its #BD at the
2131 // live `pnt` and Go's `nextUnfiltered2` at `l.pnt`. A matcher that
2132 // stepped over the last character and declined leaves no character
2133 // to name: the error then names none, at the end of the source, as
2134 // it does in both. This took a character unconditionally, and so
2135 // panicked on a lone byte-order mark under the xml plugin, whose
2136 // matcher steps over a mark at the start of the source.
2137 let bad_source = self.advance().map_or_else(String::new, String::from);
2138 let err = TabnasError::new(
2139 "unexpected",
2140 bad_source,
2141 "",
2142 pnt.site.pos,
2143 pnt.site.ri,
2144 pnt.site.ci,
2145 );
2146 self.err = Some(err.clone());
2147 Err(Box::new(err))
2148 }
2149
2150 fn match_comment(&mut self, pnt: Point) -> LexResult<Option<Token>> {
2151 let remaining = self.tail_at_byte(self.byte_position());
2152 if self.window_open()
2153 && self.options.comment.definitions.values().any(|definition| {
2154 definition.lex
2155 && remaining.len() < definition.start.len()
2156 && definition.start.starts_with(remaining)
2157 })
2158 {
2159 // A marker longer than what is left of a window, which what is
2160 // left starts, may yet start a comment once the window has
2161 // grown (#326).
2162 self.touch_end();
2163 }
2164 let mut definitions: Vec<_> = self
2165 .options
2166 .comment
2167 .definitions
2168 .iter()
2169 .filter(|(_, definition)| {
2170 // Same first-byte test as the fixed-token scan above.
2171 definition.start.as_bytes().first().copied()
2172 == remaining.as_bytes().first().copied()
2173 && !definition.start.is_empty()
2174 && definition.lex
2175 && remaining.starts_with(&definition.start)
2176 })
2177 .collect();
2178 definitions.sort_by(|(name_a, a), (name_b, b)| {
2179 b.start
2180 .len()
2181 .cmp(&a.start.len())
2182 .then_with(|| name_a.cmp(name_b))
2183 });
2184 let Some((_, definition)) = definitions.first() else {
2185 return Ok(None);
2186 };
2187 let definition = (*definition).clone();
2188 let mut src = String::new();
2189 for _ in definition.start.chars() {
2190 src.push(self.advance().expect("comment marker must advance"));
2191 }
2192
2193 let mut terminated_by_suffix = false;
2194 let mut closed = definition.line;
2195 loop {
2196 let at = self.byte_position();
2197 let remainder = self.tail_at_byte(at);
2198 let suffix = definition
2199 .suffixes
2200 .iter()
2201 .filter(|suffix| !suffix.is_empty() && remainder.starts_with(*suffix))
2202 .max_by_key(|suffix| suffix.len())
2203 .cloned();
2204 let suffix = match (suffix, definition.suffix_matcher.as_ref()) {
2205 (Some(suffix), _) => Some(suffix),
2206 (None, None) => None,
2207 (None, Some(matcher)) => match matcher.run(self.tail_at_byte(at)) {
2208 Some(effect) => Some(effect),
2209 None => {
2210 let saved = self.state();
2211 let wanted = self.want.clone();
2212 let token = matcher.run_imperative(self);
2213 self.restore(saved);
2214 self.want = wanted;
2215 token.map(|token| token.src.to_string())
2216 }
2217 },
2218 };
2219 let remainder = self.tail_at_byte(self.byte_position());
2220 let suffix =
2221 suffix.filter(|suffix| !suffix.is_empty() && remainder.starts_with(suffix));
2222 if let Some(suffix) = suffix {
2223 for _ in suffix.chars() {
2224 src.push(self.advance().expect("comment suffix must advance"));
2225 }
2226 terminated_by_suffix = true;
2227 closed = true;
2228 break;
2229 }
2230 if !definition.line
2231 && !definition.end.is_empty()
2232 && remainder.starts_with(&definition.end)
2233 {
2234 for _ in definition.end.chars() {
2235 src.push(self.advance().expect("comment end must advance"));
2236 }
2237 closed = true;
2238 break;
2239 }
2240 let Some(ch) = self.peek() else {
2241 break;
2242 };
2243 if definition.line && (self.char_sets.line_ends.contains(ch)) {
2244 break;
2245 }
2246 src.push(self.advance().expect("comment body must advance"));
2247 }
2248
2249 if !closed {
2250 let err = TabnasError::new(
2251 "unterminated_comment",
2252 src,
2253 "",
2254 pnt.site.pos,
2255 pnt.site.ri,
2256 pnt.site.ci,
2257 );
2258 self.err = Some(err.clone());
2259 return Err(Box::new(err));
2260 }
2261
2262 if definition.eat_line && !terminated_by_suffix {
2263 while let Some(ch) = self.peek() {
2264 if !self.char_sets.line_ends.contains(ch) {
2265 break;
2266 }
2267 src.push(self.advance().expect("comment line tail must advance"));
2268 }
2269 }
2270
2271 Ok(Some(Token::new(
2272 "#CM",
2273 TIN_CM,
2274 Value::String(src.clone()),
2275 src,
2276 pnt,
2277 )))
2278 }
2279
2280 fn match_number(&mut self, pnt: Point) -> LexResult<Option<Token>> {
2281 let start_idx = self.idx;
2282 let mut src = String::new();
2283
2284 // Optional sign.
2285 if matches!(self.peek(), Some('-' | '+')) {
2286 src.push(self.advance().unwrap());
2287 }
2288
2289 // Base-prefixed integers are complete at the final valid digit.
2290 if self.peek() == Some('0') {
2291 if let Some(prefix) = self.peek_at(1) {
2292 let radix = match prefix {
2293 'x' | 'X' if self.options.number.hex => Some(16),
2294 'o' | 'O' if self.options.number.oct => Some(8),
2295 'b' | 'B' if self.options.number.bin => Some(2),
2296 _ => None,
2297 };
2298 if let Some(radix) = radix {
2299 src.push(self.advance().expect("peeked zero"));
2300 src.push(self.advance().expect("peeked base prefix"));
2301 let mut saw_digit = false;
2302 while let Some(ch) = self.peek() {
2303 if ch.is_digit(radix) {
2304 saw_digit = true;
2305 src.push(self.advance().expect("peeked base digit"));
2306 } else if self
2307 .options
2308 .number
2309 .sep
2310 .as_ref()
2311 .is_some_and(|separator| separator.contains(ch))
2312 {
2313 src.push(self.advance().expect("peeked base digit"));
2314 } else {
2315 break;
2316 }
2317 }
2318 if saw_digit && self.is_text_delimiter_here() {
2319 if self
2320 .exclude_regex
2321 .as_ref()
2322 .is_some_and(|regex| regex.is_match(&src))
2323 {
2324 self.reset_number(start_idx, pnt);
2325 return Ok(None);
2326 }
2327 if self.options.value.lex {
2328 if let Some(definition) = self
2329 .options
2330 .value
2331 .definitions
2332 .get(&src)
2333 .filter(|definition| definition.matcher.is_none())
2334 {
2335 return Ok(Some(Token::new(
2336 "#VL",
2337 TIN_VL,
2338 definition
2339 .val
2340 .clone()
2341 .unwrap_or_else(|| Value::String(src.clone())),
2342 src,
2343 pnt,
2344 )));
2345 }
2346 }
2347 // The digits of the literal, prefix, sign and any
2348 // separators removed, folded as they are read. The
2349 // fold keeps a bounded head, a digit count and a
2350 // sticky bit, so a literal of any length costs the
2351 // same handful of bytes: buffering the digits
2352 // instead would let one long token multiply the
2353 // memory the source already holds.
2354 let mut fold = DigitFold::new(radix.trailing_zeros());
2355 for ch in src.chars().skip_while(|ch| matches!(ch, '-' | '+')).skip(2) {
2356 if self
2357 .options
2358 .number
2359 .sep
2360 .as_ref()
2361 .is_some_and(|separator| separator.contains(ch))
2362 {
2363 continue;
2364 }
2365 fold.push(ch.to_digit(radix).expect("validated base digit"));
2366 }
2367 let mut value = fold.finish();
2368 if src.starts_with('-') {
2369 value = -value;
2370 }
2371 return Ok(Some(Token::new(
2372 "#NR",
2373 TIN_NR,
2374 Value::Number(value),
2375 src,
2376 pnt,
2377 )));
2378 }
2379 self.reset_number(start_idx, pnt);
2380 return Ok(None);
2381 }
2382 }
2383 }
2384
2385 let Some(ch) = self.peek() else {
2386 self.reset_number(start_idx, pnt);
2387 return Ok(None);
2388 };
2389 if ch == '.' {
2390 if !self.peek_at(1).is_some_and(|next| next.is_ascii_digit()) {
2391 self.reset_number(start_idx, pnt);
2392 return Ok(None);
2393 }
2394 src.push(self.advance().expect("peeked leading decimal point"));
2395 } else if !ch.is_ascii_digit() {
2396 self.reset_number(start_idx, pnt);
2397 return Ok(None);
2398 }
2399
2400 let (has_digits, edge_separator) = self.scan_number_digits(&mut src);
2401 if !has_digits || edge_separator {
2402 self.reset_number(start_idx, pnt);
2403 return Ok(None);
2404 }
2405
2406 // The canonical regexp admits a trailing decimal point and an
2407 // exponent after it (`2.e3`), but declines `0.a` as one text run.
2408 if self.peek() == Some('.') {
2409 let next = self.peek_at(1);
2410 let exponent_after_dot = matches!(next, Some('e' | 'E'))
2411 && match self.peek_at(2) {
2412 Some('+' | '-') => self.peek_at(3).is_some_and(|ch| ch.is_ascii_digit()),
2413 Some(ch) => ch.is_ascii_digit(),
2414 None => false,
2415 };
2416 if next.is_some_and(|ch| ch.is_ascii_digit()) {
2417 src.push(self.advance().expect("peeked decimal point"));
2418 let (_, edge_separator) = self.scan_number_digits(&mut src);
2419 if edge_separator {
2420 self.reset_number(start_idx, pnt);
2421 return Ok(None);
2422 }
2423 } else if next.is_some()
2424 && !self.is_text_delimiter_at(self.idx + 1)
2425 && next != Some('.')
2426 && !exponent_after_dot
2427 {
2428 self.reset_number(start_idx, pnt);
2429 return Ok(None);
2430 } else {
2431 src.push(self.advance().expect("peeked trailing decimal point"));
2432 }
2433 }
2434
2435 if matches!(self.peek(), Some('e' | 'E')) {
2436 let exponent_start = self.idx;
2437 let source_len = src.len();
2438 src.push(self.advance().expect("peeked exponent marker"));
2439 if matches!(self.peek(), Some('+' | '-')) {
2440 src.push(self.advance().expect("peeked exponent sign"));
2441 }
2442 let (has_exponent_digits, edge_separator) = self.scan_number_digits(&mut src);
2443 if edge_separator {
2444 self.reset_number(start_idx, pnt);
2445 return Ok(None);
2446 }
2447 if !has_exponent_digits {
2448 self.idx = exponent_start;
2449 src.truncate(source_len);
2450 }
2451 }
2452
2453 if !self.is_text_delimiter_here() {
2454 self.reset_number(start_idx, pnt);
2455 return Ok(None);
2456 }
2457
2458 // Check exclusion regex (e.g. ^00+)
2459 if let Some(ref re) = self.exclude_regex {
2460 if re.is_match(&src) {
2461 // Number is excluded, backtrack
2462 self.reset_number(start_idx, pnt);
2463 return Ok(None);
2464 }
2465 }
2466
2467 if self.options.value.lex {
2468 if let Some(definition) = self
2469 .options
2470 .value
2471 .definitions
2472 .get(&src)
2473 .filter(|definition| definition.matcher.is_none())
2474 {
2475 return Ok(Some(Token::new(
2476 "#VL",
2477 TIN_VL,
2478 definition
2479 .val
2480 .clone()
2481 .unwrap_or_else(|| Value::String(src.clone())),
2482 src,
2483 pnt,
2484 )));
2485 }
2486 }
2487
2488 // Parse float
2489 let parse_src = self.options.number.sep.as_ref().map_or_else(
2490 || src.clone(),
2491 |separator| src.chars().filter(|ch| !separator.contains(*ch)).collect(),
2492 );
2493 match parse_src.parse::<f64>() {
2494 Ok(num) => Ok(Some(Token::new(
2495 "#NR",
2496 TIN_NR,
2497 Value::Number(num),
2498 src,
2499 pnt,
2500 ))),
2501 Err(_) => {
2502 self.reset_number(start_idx, pnt);
2503 Ok(None)
2504 }
2505 }
2506 }
2507
2508 fn reset_number(&mut self, start_idx: usize, pnt: Point) {
2509 self.idx = start_idx;
2510 self.ri = pnt.site.ri;
2511 self.ci = pnt.site.ci;
2512 }
2513
2514 /// Consume a decimal digit/separator run. Separators are legal only
2515 /// between digits; a leading or trailing separator makes the whole run
2516 /// fall through to text, matching the TypeScript regexp and Go scanner.
2517 fn scan_number_digits(&mut self, src: &mut String) -> (bool, bool) {
2518 // The run is measured before any of it is consumed. Advancing
2519 // as it goes would hold `&mut self` across a read of
2520 // `self.options.number.sep`, and the way that used to be settled
2521 // was to clone the separator — an allocation and a free for
2522 // every number in the input, for a value that cannot change
2523 // while one number is being scanned.
2524 let run_start = self.idx;
2525 let mut saw_digit = false;
2526 let mut last_was_separator = false;
2527 let mut end = run_start;
2528 {
2529 let separator = self.options.number.sep.as_deref();
2530 while let Some(ch) = self.ch_get(end) {
2531 if ch.is_ascii_digit() {
2532 saw_digit = true;
2533 last_was_separator = false;
2534 } else if separator.is_some_and(|separator| separator.contains(ch)) {
2535 last_was_separator = true;
2536 } else {
2537 break;
2538 }
2539 end += 1;
2540 }
2541 }
2542 while self.idx < end {
2543 src.push(self.advance().expect("scanned number character"));
2544 }
2545 let starts_with_separator = self.idx > run_start
2546 && self.options.number.sep.as_deref().is_some_and(|separator| {
2547 self.chars[run_start - self.low..self.idx - self.low]
2548 .first()
2549 .is_some_and(|ch| separator.contains(*ch))
2550 });
2551 (saw_digit, starts_with_separator || last_was_separator)
2552 }
2553
2554 fn match_string(&mut self, quote: char, pnt: Point) -> LexResult<Token> {
2555 let quote_char = self.advance().unwrap();
2556 let mut out_str = String::new();
2557 let mut raw_src = String::new();
2558 raw_src.push(quote_char);
2559
2560 let mut pending_high_surrogate: Option<u16> = None;
2561
2562 // The body classes of TypeScript's `buildStringBodySpec`
2563 // (ts/src/lexer.ts), which Go's `BuildStringBodySpec` ports: a
2564 // line character is LINE or LINE+ROW only inside a multi-line
2565 // string, where it resets the column and, in `line.rowChars`,
2566 // counts a row; anywhere else in a body it is plain content,
2567 // counted as a column, unless it is a control character, which
2568 // stops the body as `unprintable`. This loop used to count a row
2569 // for any row character it stepped over, string body or not, and
2570 // to refuse any line character inside a single-line string: with
2571 // json5's U+2028 and U+2029 as row characters, `y` in
2572 // `"a<U+2028>b" y` sat on row 2 here and on row 1 in TypeScript
2573 // and Go, and with the two in `line.chars` as well the string was
2574 // `unprintable` (tabnas/parser#263).
2575 let multi_line = self.options.string.multi_chars.contains(quote);
2576
2577 while let Some(c) = self.peek() {
2578 if c == quote {
2579 raw_src.push(self.advance().unwrap());
2580 // Rust strings cannot represent a lone UTF-16 surrogate, so
2581 // preserve the Go-port behavior and fold it to U+FFFD.
2582 self.flush_surrogate(&mut pending_high_surrogate, &mut out_str);
2583 return Ok(Token::new(
2584 "#ST",
2585 TIN_ST,
2586 Value::String(out_str),
2587 raw_src,
2588 pnt,
2589 ));
2590 }
2591
2592 if let Some(replacement) = self.options.string.replace.get(&c).cloned() {
2593 raw_src.push(self.advance().expect("peeked character must advance"));
2594 self.flush_surrogate(&mut pending_high_surrogate, &mut out_str);
2595 out_str.push_str(&replacement);
2596 continue;
2597 }
2598
2599 if multi_line && self.char_sets.line.contains(c) {
2600 raw_src.push(
2601 self.advance_string_line()
2602 .expect("peeked character must advance"),
2603 );
2604 out_str.push(c);
2605 continue;
2606 }
2607
2608 // A control character stops the body: a line character
2609 // always, since a single-line string cannot hold one, and any
2610 // other unless `string.allowControl` admits it. Sited ON the
2611 // character, as TypeScript does (`pnt.sI = sI; pnt.cI = cI`
2612 // before its `bad()` call, ts/src/lexer.ts). `pnt` is the
2613 // opening quote, and reporting that put every embedded
2614 // newline at the start of its string.
2615 if (c as u32) < 32
2616 && (self.char_sets.line.contains(c) || !self.options.string.allow_control)
2617 {
2618 let site = self.current_point().site;
2619 let err =
2620 TabnasError::new("unprintable", c.to_string(), "", site.pos, site.ri, site.ci);
2621 self.err = Some(err.clone());
2622 return Err(Box::new(err));
2623 }
2624
2625 if c == self.options.string.escape_char {
2626 raw_src.push(self.advance().unwrap());
2627 let esc_point = self.current_point();
2628 if let Some(esc) = self.advance() {
2629 raw_src.push(esc);
2630 if let Some(replacement) = self.options.string.escape.get(&esc).cloned() {
2631 self.flush_surrogate(&mut pending_high_surrogate, &mut out_str);
2632 out_str.push_str(&replacement);
2633 continue;
2634 }
2635 match esc {
2636 'u' => {
2637 // Unicode escape: \uXXXX or \u{X...}. An
2638 // invalid one is reported on the backslash
2639 // with the span TypeScript cuts: six source
2640 // characters for the fixed-width form, four
2641 // for `\x`, and through the closing brace
2642 // (or to the end of the source) for the
2643 // braced form -- clipped, never padded, so a
2644 // truncated escape at end of input reports
2645 // exactly the characters that are there.
2646 if self.peek() == Some('{') && !self.options.string.escape_strict {
2647 raw_src.push(self.advance().unwrap()); // '{'
2648 let mut hex = String::new();
2649 let mut closed = false;
2650 while let Some(h) = self.peek() {
2651 if h == '}' {
2652 raw_src.push(self.advance().unwrap());
2653 closed = true;
2654 break;
2655 }
2656 raw_src.push(self.advance().unwrap());
2657 hex.push(h);
2658 }
2659
2660 if !closed
2661 || hex.is_empty()
2662 || hex.len() > 6
2663 || !hex.chars().all(|ch| ch.is_ascii_hexdigit())
2664 {
2665 let err = TabnasError::new(
2666 "invalid_unicode",
2667 self.source_span(esc_point.site.pos - 1, self.idx),
2668 "",
2669 esc_point.site.pos - 1,
2670 esc_point.site.ri,
2671 esc_point.site.ci - 1,
2672 );
2673 self.err = Some(err.clone());
2674 return Err(Box::new(err));
2675 }
2676
2677 let cp = match u32::from_str_radix(&hex, 16) {
2678 Ok(val) if val <= 0x10FFFF => val,
2679 _ => {
2680 let err = TabnasError::new(
2681 "invalid_unicode",
2682 self.source_span(esc_point.site.pos - 1, self.idx),
2683 "",
2684 esc_point.site.pos - 1,
2685 esc_point.site.ri,
2686 esc_point.site.ci - 1,
2687 );
2688 self.err = Some(err.clone());
2689 return Err(Box::new(err));
2690 }
2691 };
2692
2693 self.emit_unicode_escape(
2694 cp,
2695 &mut pending_high_surrogate,
2696 &mut out_str,
2697 );
2698 } else {
2699 // Exactly 4 hex digits: \uXXXX
2700 let mut hex = String::new();
2701 for _ in 0..4 {
2702 if let Some(h) = self.peek() {
2703 if h.is_ascii_hexdigit() {
2704 raw_src.push(self.advance().unwrap());
2705 hex.push(h);
2706 } else {
2707 break;
2708 }
2709 } else {
2710 break;
2711 }
2712 }
2713
2714 if hex.len() != 4 {
2715 let err = TabnasError::new(
2716 "invalid_unicode",
2717 self.source_span(
2718 esc_point.site.pos - 1,
2719 esc_point.site.pos + 5,
2720 ),
2721 "",
2722 esc_point.site.pos - 1,
2723 esc_point.site.ri,
2724 esc_point.site.ci - 1,
2725 );
2726 self.err = Some(err.clone());
2727 return Err(Box::new(err));
2728 }
2729
2730 let cp = u16::from_str_radix(&hex, 16).map_err(|_| {
2731 let err = TabnasError::new(
2732 "invalid_unicode",
2733 self.source_span(
2734 esc_point.site.pos - 1,
2735 esc_point.site.pos + 5,
2736 ),
2737 "",
2738 esc_point.site.pos - 1,
2739 esc_point.site.ri,
2740 esc_point.site.ci - 1,
2741 );
2742 self.err = Some(err.clone());
2743 err
2744 })?;
2745
2746 self.emit_unicode_escape(
2747 u32::from(cp),
2748 &mut pending_high_surrogate,
2749 &mut out_str,
2750 );
2751 }
2752 }
2753 'x' if !self.options.string.escape_strict => {
2754 let mut hex = String::new();
2755 for _ in 0..2 {
2756 if let Some(h) = self.peek() {
2757 if h.is_ascii_hexdigit() {
2758 raw_src.push(
2759 self.advance().expect("peeked character must advance"),
2760 );
2761 hex.push(h);
2762 }
2763 }
2764 }
2765 if hex.len() != 2 {
2766 let err = TabnasError::new(
2767 "invalid_ascii",
2768 self.source_span(
2769 esc_point.site.pos - 1,
2770 esc_point.site.pos + 3,
2771 ),
2772 "",
2773 esc_point.site.pos - 1,
2774 esc_point.site.ri,
2775 esc_point.site.ci - 1,
2776 );
2777 self.err = Some(err.clone());
2778 return Err(Box::new(err));
2779 }
2780 let byte = u8::from_str_radix(&hex, 16).expect("validated ASCII hex");
2781 self.flush_surrogate(&mut pending_high_surrogate, &mut out_str);
2782 out_str.push(char::from(byte));
2783 }
2784 other => {
2785 if !self.options.string.allow_unknown {
2786 // Sited on the escape CHARACTER with a
2787 // one-character span, as TypeScript
2788 // (`pnt.sI = sI; pnt.cI = cI; lex.bad(
2789 // S.unexpected, sI, sI + 1)`) and Go do;
2790 // the other escape errors sit on the
2791 // backslash and span the construct.
2792 let err = TabnasError::new(
2793 "unexpected",
2794 other.to_string(),
2795 "",
2796 esc_point.site.pos,
2797 esc_point.site.ri,
2798 esc_point.site.ci,
2799 );
2800 self.err = Some(err.clone());
2801 return Err(Box::new(err));
2802 }
2803 self.flush_surrogate(&mut pending_high_surrogate, &mut out_str);
2804 out_str.push(other);
2805 }
2806 }
2807 } else {
2808 let err = TabnasError::new(
2809 "unterminated_string",
2810 raw_src,
2811 "",
2812 pnt.site.pos,
2813 pnt.site.ri,
2814 pnt.site.ci,
2815 );
2816 self.err = Some(err.clone());
2817 return Err(Box::new(err));
2818 }
2819 } else {
2820 self.flush_surrogate(&mut pending_high_surrogate, &mut out_str);
2821 raw_src.push(self.advance_body().expect("peeked character must advance"));
2822 out_str.push(c);
2823 }
2824 }
2825
2826 let err = TabnasError::new(
2827 "unterminated_string",
2828 raw_src,
2829 "",
2830 pnt.site.pos,
2831 pnt.site.ri,
2832 pnt.site.ci,
2833 );
2834 self.err = Some(err.clone());
2835 Err(Box::new(err))
2836 }
2837
2838 fn flush_surrogate(&self, pending: &mut Option<u16>, out: &mut String) {
2839 if pending.take().is_some() {
2840 out.push('\u{FFFD}');
2841 }
2842 }
2843
2844 /// Emit one decoded Unicode escape while pairing UTF-16 surrogate code
2845 /// units across both `\\uXXXX` and `\\u{...}` spellings.
2846 fn emit_unicode_escape(&self, cp: u32, pending: &mut Option<u16>, out: &mut String) {
2847 if (0xD800..=0xDBFF).contains(&cp) {
2848 self.flush_surrogate(pending, out);
2849 *pending = Some(cp as u16);
2850 } else if (0xDC00..=0xDFFF).contains(&cp) {
2851 if let Some(high) = pending.take() {
2852 let scalar = 0x10000 + (((u32::from(high)) - 0xD800) << 10) + (cp - 0xDC00);
2853 out.push(char::from_u32(scalar).expect("paired surrogates form a Unicode scalar"));
2854 } else {
2855 out.push('\u{FFFD}');
2856 }
2857 } else {
2858 self.flush_surrogate(pending, out);
2859 out.push(char::from_u32(cp).expect("validated escape is a Unicode scalar"));
2860 }
2861 }
2862}
2863
2864// ---------------------------------------------------------------------------
2865// Base-prefixed integer literals.
2866//
2867// A `0x`, `0o` or `0b` literal is read as an EXACT integer and rounded to
2868// a double ONCE. The obvious fold -- `value = value * radix + digit` in
2869// `f64` -- rounds at every digit, and past the 53-bit exact integer range
2870// those roundings accumulate: `0Xa6f2f78f4f9bf44` came out as
2871// `43a4de5ef1e9f37e` where canonical TypeScript and the Go port both
2872// answer `43a4de5ef1e9f37f`, one unit in the last place low. That is
2873// silently altered data, not a formatting difference.
2874//
2875// TypeScript coerces the literal with unary `+`, whose StringNumericValue
2876// is the exact mathematical value of the digits rounded once, half to
2877// even; Go reads it through `big.Int` and `big.Float.Float64()`, which is
2878// the same rule. These reproduce it. Only the VALUE is affected: which
2879// literals are accepted, and the token they become, are settled by
2880// `match_number` before any of this runs.
2881//
2882// The decimal path needs none of it -- `str::parse::<f64>` is correctly
2883// rounded for a digit string of any length.
2884// ---------------------------------------------------------------------------
2885
2886/// `2^k` for a non-negative `k`, exactly, saturating to infinity above the
2887/// double range. A repeated multiply would round on the way up.
2888fn pow2(k: i64) -> f64 {
2889 debug_assert!(k >= 0, "only non-negative exponents arise here");
2890 if k > 1023 {
2891 f64::INFINITY
2892 } else {
2893 f64::from_bits(((k + 1023) as u64) << 52)
2894 }
2895}
2896
2897/// Folds the digits of a base-prefixed literal into the NEAREST double,
2898/// rounding half to even, without holding the digits.
2899///
2900/// `bits` is the width of one digit, so the base is a power of two: 1 for
2901/// binary, 3 for octal, 4 for hexadecimal. Those are the only bases
2902/// `match_number` reads, which is what lets a `u128` head plus a sticky
2903/// bit stand in for arbitrary-precision arithmetic.
2904///
2905/// Only three things about a literal can change the answer: the top
2906/// `128 / bits` significant digits, how many digits follow them, and
2907/// whether any of those is non-zero. This keeps exactly those, so the
2908/// space it costs does not grow with the literal, however long an
2909/// untrusted document makes one.
2910struct DigitFold {
2911 bits: u32,
2912 /// The significant digits packed so far, at most `head_len` of them.
2913 head: u128,
2914 /// How many significant digits have been pushed, head and tail alike.
2915 len: usize,
2916 /// Whether any digit past the head was non-zero.
2917 sticky: bool,
2918 /// Whether a non-zero digit has been seen. Leading zeros carry no
2919 /// value, and dropping them is what makes the head wider than the 54
2920 /// significant bits the rounding needs.
2921 started: bool,
2922}
2923
2924impl DigitFold {
2925 fn new(bits: u32) -> Self {
2926 debug_assert!(
2927 (1..=4).contains(&bits),
2928 "only the power-of-two bases the lexer reads"
2929 );
2930 DigitFold {
2931 bits,
2932 head: 0,
2933 len: 0,
2934 sticky: false,
2935 started: false,
2936 }
2937 }
2938
2939 /// A `u128` holds exactly this many digits of the base.
2940 fn head_len(&self) -> usize {
2941 (128 / self.bits) as usize
2942 }
2943
2944 fn push(&mut self, digit: u32) {
2945 if !self.started {
2946 if 0 == digit {
2947 return;
2948 }
2949 self.started = true;
2950 }
2951 if self.len < self.head_len() {
2952 self.head = (self.head << self.bits) | u128::from(digit);
2953 } else if 0 != digit {
2954 self.sticky = true;
2955 }
2956 self.len += 1;
2957 }
2958
2959 fn finish(&self) -> f64 {
2960 if !self.started {
2961 return 0.0;
2962 }
2963 let head_len = self.head_len();
2964 if self.len <= head_len {
2965 // A `u128` to `f64` cast rounds to nearest, ties to even, which
2966 // is the rule the canonical runtime follows.
2967 return self.head as f64;
2968 }
2969
2970 // Longer than a u128: the head holds the top `head_len` digits and
2971 // `sticky` remembers whether anything below them was set. Those two
2972 // are all the rounding can depend on. The leading digit is
2973 // non-zero, so the head is at least 121 bits wide in every base
2974 // here and `shift` is comfortably positive.
2975 let dropped = i64::from(self.bits) * (self.len - head_len) as i64;
2976 let shift = 128 - self.head.leading_zeros() - 53;
2977
2978 let mut mantissa = (self.head >> shift) as u64;
2979 let half = (self.head >> (shift - 1)) & 1 == 1;
2980 let sticky = self.head & ((1u128 << (shift - 1)) - 1) != 0 || self.sticky;
2981 if half && (sticky || mantissa & 1 == 1) {
2982 // At most 2^53, which is still an exact double.
2983 mantissa += 1;
2984 }
2985 // The mantissa carries at most 53 significant bits, so the scaling
2986 // is exact inside the double range and overflows to infinity
2987 // outside it.
2988 mantissa as f64 * pow2(dropped + i64::from(shift))
2989 }
2990}
2991
2992#[cfg(test)]
2993mod window_tests {
2994 use super::{WindowSizes, WINDOW_PEAK};
2995
2996 /// A parse from a reader holds no more of the source than its window:
2997 /// a block behind the cursor and a margin or two ahead of it, however
2998 /// long the source, flat from 2,000 records to 20,000.
2999 #[test]
3000 fn a_parse_from_a_reader_holds_a_bounded_window() {
3001 let mut parser = crate::Tabnas::make_json();
3002 parser
3003 .set_options(|options| options.lex.window = true)
3004 .expect("lex.window sets");
3005 let sizes = WindowSizes {
3006 margin: 1024,
3007 block: 1024,
3008 chunk: 1024,
3009 };
3010 let mut peaks = Vec::new();
3011 for count in [2_000, 20_000] {
3012 let records: Vec<String> = (0..count)
3013 .map(|i| format!("{{\"id\":{i},\"name\":\"n{i}\",\"tags\":[\"a\",\"b\"]}}"))
3014 .collect();
3015 let text = format!("[{}]", records.join(","));
3016 WINDOW_PEAK.with(|peak| peak.set(0));
3017 parser
3018 .parse_reader_with_sizes(text.as_bytes(), sizes)
3019 .expect("the reader parses");
3020 let peak = WINDOW_PEAK.with(std::cell::Cell::get);
3021 assert!(peak <= 4 * 1024, "{count} records held {peak} characters");
3022 peaks.push(peak);
3023 }
3024 // Ten times the records hold no more than a few records' more:
3025 // where the blocks fall against the records moves the peak a
3026 // little, the length not at all.
3027 assert!(peaks[1] <= peaks[0] + 256, "{peaks:?}");
3028 }
3029}