antlr4_runtime/
lexer.rs

1use std::collections::{BTreeSet, HashMap};
2use std::hash::BuildHasherDefault;
3use std::rc::Rc;
4
5use crate::char_stream::{CharStream, TextInterval};
6use crate::int_stream::EOF;
7use crate::prediction::PredictionFxHasher;
8use crate::recognizer::{Recognizer, RecognizerData};
9use crate::token::{CommonToken, CommonTokenFactory, TokenFactory, TokenSourceError, TokenSpec};
10
11#[allow(clippy::disallowed_types)]
12type FxHashMap<K, V> = HashMap<K, V, BuildHasherDefault<PredictionFxHasher>>;
13
14pub const SKIP: i32 = -3;
15pub const MORE: i32 = -2;
16pub const DEFAULT_MODE: i32 = 0;
17
18#[derive(Clone, Copy, Debug, Eq, PartialEq)]
19pub struct LexerMode(pub i32);
20
21/// Grammar-specific lexer action reached on the accepted ATN path.
22///
23/// ANTLR serializes embedded lexer actions as `(rule_index, action_index)`
24/// pairs. The runtime also records the input position where the action was
25/// reached so generated code can evaluate templates such as `Text()` at the
26/// same point as a generated ANTLR lexer, not only at the token end.
27#[derive(Clone, Copy, Debug, Eq, PartialEq)]
28pub struct LexerCustomAction {
29    rule_index: i32,
30    action_index: i32,
31    position: usize,
32}
33
34impl LexerCustomAction {
35    /// Creates a custom lexer action event from serialized ATN metadata.
36    pub const fn new(rule_index: i32, action_index: i32, position: usize) -> Self {
37        Self {
38            rule_index,
39            action_index,
40            position,
41        }
42    }
43
44    /// Lexer rule index that owns the embedded action.
45    pub const fn rule_index(self) -> i32 {
46        self.rule_index
47    }
48
49    /// Per-rule action index assigned by ANTLR serialization.
50    pub const fn action_index(self) -> i32 {
51        self.action_index
52    }
53
54    /// Character-stream position at which the action transition was reached.
55    pub const fn position(self) -> usize {
56        self.position
57    }
58}
59
60/// Grammar-specific lexer predicate reached while exploring an ATN path.
61#[derive(Clone, Copy, Debug, Eq, PartialEq)]
62pub struct LexerPredicate {
63    rule_index: usize,
64    pred_index: usize,
65    position: usize,
66}
67
68impl LexerPredicate {
69    /// Creates a lexer predicate event from serialized ATN metadata.
70    pub const fn new(rule_index: usize, pred_index: usize, position: usize) -> Self {
71        Self {
72            rule_index,
73            pred_index,
74            position,
75        }
76    }
77
78    /// Lexer rule index that owns the predicate transition.
79    pub const fn rule_index(self) -> usize {
80        self.rule_index
81    }
82
83    /// Per-rule predicate index assigned by ANTLR serialization.
84    pub const fn pred_index(self) -> usize {
85        self.pred_index
86    }
87
88    /// Character-stream position at which the predicate is evaluated.
89    pub const fn position(self) -> usize {
90        self.position
91    }
92}
93
94pub trait Lexer: Recognizer {
95    fn mode(&self) -> i32;
96    fn set_mode(&mut self, mode: i32);
97    fn push_mode(&mut self, mode: i32);
98    fn pop_mode(&mut self) -> Option<i32>;
99}
100
101#[derive(Clone, Debug)]
102pub struct BaseLexer<I, F = CommonTokenFactory> {
103    input: I,
104    data: RecognizerData,
105    factory: F,
106    mode: i32,
107    mode_stack: Vec<i32>,
108    token_start: usize,
109    token_start_line: usize,
110    token_start_column: usize,
111    line: usize,
112    column: usize,
113    hit_eof: bool,
114    errors: Vec<TokenSourceError>,
115    lexer_dfa: LexerDfaTrace,
116}
117
118/// Compact observation log for the default-mode lexer DFA printed by `showDFA`
119/// runtime-suite descriptors.
120#[derive(Clone, Debug, Default)]
121struct LexerDfaTrace {
122    state_numbers: FxHashMap<LexerDfaKey, usize>,
123    accept_predictions: FxHashMap<usize, i32>,
124    edges: BTreeSet<LexerDfaEdge>,
125    cached_states: FxHashMap<usize, Rc<LexerDfaCachedState>>,
126    transitions: FxHashMap<(usize, i32), LexerDfaCachedTransition>,
127    mode_starts: FxHashMap<i32, usize>,
128}
129
130impl LexerDfaTrace {
131    fn new() -> Self {
132        Self {
133            state_numbers: FxHashMap::default(),
134            accept_predictions: FxHashMap::default(),
135            edges: BTreeSet::new(),
136            cached_states: FxHashMap::default(),
137            transitions: FxHashMap::default(),
138            mode_starts: FxHashMap::default(),
139        }
140    }
141}
142
143/// Normalized lexer ATN config-set identity used for observed DFA traces.
144#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
145pub(crate) struct LexerDfaKey {
146    configs: Vec<LexerDfaConfigKey>,
147}
148
149impl LexerDfaKey {
150    pub(crate) fn new(mut configs: Vec<LexerDfaConfigKey>) -> Self {
151        configs.sort_unstable();
152        Self { configs }
153    }
154}
155
156/// One lexer ATN config identity with the absolute input position removed.
157#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
158pub(crate) struct LexerDfaConfigKey {
159    pub(crate) state: usize,
160    pub(crate) alt_rule_index: Option<usize>,
161    pub(crate) consumed_eof: bool,
162    pub(crate) passed_non_greedy: bool,
163    pub(crate) stack: Vec<usize>,
164    pub(crate) actions: Vec<LexerDfaActionKey>,
165}
166
167#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
168pub(crate) struct LexerDfaActionKey {
169    pub(crate) action_index: usize,
170    pub(crate) position_delta: usize,
171    pub(crate) rule_index: usize,
172}
173
174impl LexerDfaConfigKey {
175    pub(crate) const fn new(
176        state: usize,
177        alt_rule_index: Option<usize>,
178        consumed_eof: bool,
179        passed_non_greedy: bool,
180        stack: Vec<usize>,
181        actions: Vec<LexerDfaActionKey>,
182    ) -> Self {
183        Self {
184            state,
185            alt_rule_index,
186            consumed_eof,
187            passed_non_greedy,
188            stack,
189            actions,
190        }
191    }
192}
193
194#[derive(Clone, Debug)]
195pub(crate) struct LexerDfaCachedTransition {
196    pub(crate) target_state: usize,
197    pub(crate) position_delta: usize,
198}
199
200#[derive(Clone, Debug)]
201pub(crate) struct LexerDfaCachedAccept {
202    pub(crate) position_delta: usize,
203    pub(crate) rule_index: usize,
204    pub(crate) consumed_eof: bool,
205    pub(crate) actions: Vec<LexerDfaActionKey>,
206}
207
208#[derive(Clone, Debug)]
209pub(crate) struct LexerDfaCachedState {
210    pub(crate) has_semantic_context: bool,
211    pub(crate) configs: Vec<LexerDfaConfigKey>,
212    pub(crate) accept: Option<LexerDfaCachedAccept>,
213}
214
215/// One printable lexer DFA edge keyed so repeated matches keep deterministic
216/// output order.
217#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
218struct LexerDfaEdge {
219    from: usize,
220    symbol: i32,
221    to: usize,
222}
223
224impl<I> BaseLexer<I>
225where
226    I: CharStream,
227{
228    /// Creates a lexer base using `CommonTokenFactory`.
229    pub fn new(input: I, data: RecognizerData) -> Self {
230        Self::with_factory(input, data, CommonTokenFactory)
231    }
232}
233
234impl<I, F> BaseLexer<I, F>
235where
236    I: CharStream,
237    F: TokenFactory,
238{
239    /// Creates a lexer base with a custom token factory.
240    pub fn with_factory(input: I, data: RecognizerData, factory: F) -> Self {
241        Self {
242            input,
243            data,
244            factory,
245            mode: DEFAULT_MODE,
246            mode_stack: Vec::new(),
247            token_start: 0,
248            token_start_line: 1,
249            token_start_column: 0,
250            line: 1,
251            column: 0,
252            hit_eof: false,
253            errors: Vec::new(),
254            lexer_dfa: LexerDfaTrace::new(),
255        }
256    }
257
258    pub const fn input(&self) -> &I {
259        &self.input
260    }
261
262    pub const fn input_mut(&mut self) -> &mut I {
263        &mut self.input
264    }
265
266    /// Captures the input index and source position for the token currently
267    /// being matched.
268    pub fn begin_token(&mut self) {
269        self.token_start = self.input.index();
270        self.token_start_line = self.line;
271        self.token_start_column = self.column;
272    }
273
274    /// Returns the absolute character index where the current token began.
275    pub const fn token_start(&self) -> usize {
276        self.token_start
277    }
278
279    /// Returns the source line captured at the start of the current token.
280    pub const fn token_start_line(&self) -> usize {
281        self.token_start_line
282    }
283
284    /// Returns the source column captured at the start of the current token.
285    pub const fn token_start_column(&self) -> usize {
286        self.token_start_column
287    }
288
289    /// Consumes one character from the input stream and updates lexer line and
290    /// column counters.
291    ///
292    /// The input stream is indexed by Unicode scalar values. Newline handling
293    /// follows ANTLR's default convention of incrementing the line and resetting
294    /// the column after `\n`.
295    pub fn consume_char(&mut self) {
296        let la = self.input.la(1);
297        if la == EOF {
298            return;
299        }
300        self.input.consume();
301        if char::from_u32(la.cast_unsigned()) == Some('\n') {
302            self.line += 1;
303            self.column = 0;
304        } else {
305            self.column += 1;
306        }
307    }
308
309    /// Rewinds or advances the input cursor to a token accept boundary.
310    ///
311    /// Some generated lexers intentionally accept a longer path to disambiguate
312    /// a token, then emit only the prefix and leave the suffix for the next
313    /// token. Recomputing line/column from `token_start` keeps the visible lexer
314    /// position consistent after moving the cursor backwards.
315    pub fn reset_accept_position(&mut self, index: usize) {
316        let target = index.max(self.token_start);
317        self.input.seek(self.token_start);
318        self.line = self.token_start_line;
319        self.column = self.token_start_column;
320        while self.input.index() < target && self.input.la(1) != EOF {
321            self.consume_char();
322        }
323    }
324
325    /// Builds a token spanning from the current token start to the character
326    /// before the input cursor.
327    ///
328    /// When generated or interpreted lexer code does not supply explicit text,
329    /// the base lexer captures the matched source interval so downstream token
330    /// streams and parse trees can render token text without retaining a source
331    /// pair object.
332    pub fn emit(&self, token_type: i32, channel: i32, text: Option<String>) -> CommonToken {
333        let stop = self.input.index().checked_sub(1).unwrap_or(usize::MAX);
334        self.emit_with_stop(token_type, channel, stop, text)
335    }
336
337    /// Builds a token with an explicit stop index.
338    ///
339    /// EOF-matching lexer rules do not consume a Unicode scalar value, so their
340    /// stop index can be one before the current input index. The caller passes
341    /// `usize::MAX` to represent ANTLR's `-1` stop index at empty input.
342    pub fn emit_with_stop(
343        &self,
344        token_type: i32,
345        channel: i32,
346        stop: usize,
347        text: Option<String>,
348    ) -> CommonToken {
349        let text = text.or_else(|| {
350            if stop == usize::MAX {
351                Some("<EOF>".to_owned())
352            } else {
353                None
354            }
355        });
356        let source_text = if text.is_none() && stop != usize::MAX {
357            self.input
358                .text_source_interval(TextInterval::new(self.token_start, stop))
359                .and_then(|(input, start_byte, stop_byte)| {
360                    Some(crate::token::TokenSourceText {
361                        input,
362                        start_byte: u32::try_from(start_byte).ok()?,
363                        stop_byte: u32::try_from(stop_byte).ok()?,
364                    })
365                })
366        } else {
367            None
368        };
369        let text = text.or_else(|| {
370            source_text
371                .is_none()
372                .then(|| self.input.text(TextInterval::new(self.token_start, stop)))
373        });
374        self.factory.create(TokenSpec {
375            token_type,
376            channel,
377            start: self.token_start,
378            stop,
379            line: self.token_start_line,
380            column: self.token_start_column,
381            text,
382            source_text,
383            source_name: self.input.source_name(),
384        })
385    }
386
387    /// Returns the current token text from the token start through the input
388    /// cursor.
389    pub fn token_text(&self) -> String {
390        self.token_text_until(self.input.index())
391    }
392
393    /// Returns the current token text from the token start through
394    /// `stop_exclusive`.
395    ///
396    /// Lexer custom actions can occur before the accepted token is complete.
397    /// The action event records the position where the transition fired, and
398    /// generated action code uses this helper to render ANTLR's `Text()`
399    /// template at that exact point.
400    pub fn token_text_until(&self, stop_exclusive: usize) -> String {
401        if stop_exclusive <= self.token_start {
402            return String::new();
403        }
404        self.input
405            .text(TextInterval::new(self.token_start, stop_exclusive - 1))
406    }
407
408    /// Computes the zero-based source column at an absolute input position
409    /// reached during prediction of the current token.
410    pub fn column_at(&self, position: usize) -> usize {
411        let mut column = self.token_start_column;
412        if position <= self.token_start {
413            return column;
414        }
415        for ch in self
416            .input
417            .text(TextInterval::new(self.token_start, position - 1))
418            .chars()
419        {
420            if ch == '\n' {
421                column = 0;
422            } else {
423                column += 1;
424            }
425        }
426        column
427    }
428
429    /// Builds the synthetic EOF token at the current input cursor.
430    pub fn eof_token(&self) -> CommonToken {
431        CommonToken::eof(
432            self.input.source_name(),
433            self.input.index(),
434            self.line,
435            self.column,
436        )
437    }
438}
439
440impl<I, F> Recognizer for BaseLexer<I, F>
441where
442    I: CharStream,
443    F: TokenFactory,
444{
445    fn data(&self) -> &RecognizerData {
446        &self.data
447    }
448
449    fn data_mut(&mut self) -> &mut RecognizerData {
450        &mut self.data
451    }
452}
453
454impl<I, F> Lexer for BaseLexer<I, F>
455where
456    I: CharStream,
457    F: TokenFactory,
458{
459    fn mode(&self) -> i32 {
460        self.mode
461    }
462
463    fn set_mode(&mut self, mode: i32) {
464        self.mode = mode;
465    }
466
467    fn push_mode(&mut self, mode: i32) {
468        self.mode_stack.push(self.mode);
469        self.mode = mode;
470    }
471
472    fn pop_mode(&mut self) -> Option<i32> {
473        let mode = self.mode_stack.pop()?;
474        self.mode = mode;
475        Some(mode)
476    }
477}
478
479impl<I, F> BaseLexer<I, F>
480where
481    I: CharStream,
482    F: TokenFactory,
483{
484    pub const fn line(&self) -> usize {
485        self.line
486    }
487
488    pub const fn column(&self) -> usize {
489        self.column
490    }
491
492    pub fn source_name(&self) -> &str {
493        self.input.source_name()
494    }
495
496    pub const fn hit_eof(&self) -> bool {
497        self.hit_eof
498    }
499
500    pub const fn set_hit_eof(&mut self, hit_eof: bool) {
501        self.hit_eof = hit_eof;
502    }
503
504    /// Buffers a lexer diagnostic until the token stream consumer is ready to
505    /// emit errors in parser-compatible order.
506    pub fn record_error(&mut self, line: usize, column: usize, message: impl Into<String>) {
507        self.errors
508            .push(TokenSourceError::new(line, column, message));
509    }
510
511    /// Returns and clears lexer diagnostics produced while fetching tokens.
512    pub fn drain_errors(&mut self) -> Vec<TokenSourceError> {
513        std::mem::take(&mut self.errors)
514    }
515
516    /// Returns the stable state number for a normalized lexer DFA config set,
517    /// creating one if this input path has not reached it before.
518    pub(crate) fn lexer_dfa_state(
519        &mut self,
520        key: LexerDfaKey,
521        accept_prediction: Option<i32>,
522    ) -> usize {
523        let next = self.lexer_dfa.state_numbers.len();
524        let state = *self.lexer_dfa.state_numbers.entry(key).or_insert(next);
525        if let Some(prediction) = accept_prediction {
526            self.lexer_dfa.accept_predictions.insert(state, prediction);
527        }
528        state
529    }
530
531    /// Records a visible lexer DFA edge unless it was already observed.
532    pub fn record_lexer_dfa_edge(&mut self, from: usize, symbol: i32, to: usize) {
533        self.lexer_dfa
534            .edges
535            .insert(LexerDfaEdge { from, symbol, to });
536    }
537
538    pub(crate) fn cached_lexer_dfa_transition(
539        &self,
540        state: usize,
541        symbol: i32,
542    ) -> Option<LexerDfaCachedTransition> {
543        self.lexer_dfa.transitions.get(&(state, symbol)).cloned()
544    }
545
546    pub(crate) fn cache_lexer_dfa_transition(
547        &mut self,
548        state: usize,
549        symbol: i32,
550        transition: LexerDfaCachedTransition,
551    ) {
552        self.lexer_dfa
553            .transitions
554            .entry((state, symbol))
555            .or_insert(transition);
556    }
557
558    pub(crate) fn cached_lexer_dfa_state(&self, state: usize) -> Option<Rc<LexerDfaCachedState>> {
559        self.lexer_dfa.cached_states.get(&state).cloned()
560    }
561
562    pub(crate) fn cache_lexer_dfa_state(
563        &mut self,
564        state: usize,
565        cached_state: LexerDfaCachedState,
566    ) {
567        self.lexer_dfa
568            .cached_states
569            .entry(state)
570            .or_insert_with(|| Rc::new(cached_state));
571    }
572
573    pub(crate) fn cached_lexer_mode_start(&self, mode: i32) -> Option<usize> {
574        self.lexer_dfa.mode_starts.get(&mode).copied()
575    }
576
577    pub(crate) fn cache_lexer_mode_start(&mut self, mode: i32, state: usize) {
578        self.lexer_dfa.mode_starts.entry(mode).or_insert(state);
579    }
580
581    /// Serializes the observed default-mode lexer DFA in ANTLR's text shape.
582    pub fn lexer_dfa_string(&self) -> String {
583        let mut out = String::new();
584        for edge in &self.lexer_dfa.edges {
585            let Some(label) = lexer_dfa_edge_label(edge.symbol) else {
586                continue;
587            };
588            out.push_str(&self.lexer_dfa_state_string(edge.from));
589            out.push('-');
590            out.push_str(&label);
591            out.push_str("->");
592            out.push_str(&self.lexer_dfa_state_string(edge.to));
593            out.push('\n');
594        }
595        out
596    }
597
598    fn lexer_dfa_state_string(&self, state: usize) -> String {
599        self.lexer_dfa.accept_predictions.get(&state).map_or_else(
600            || format!("s{state}"),
601            |prediction| format!(":s{state}=>{prediction}"),
602        )
603    }
604}
605
606fn lexer_dfa_edge_label(symbol: i32) -> Option<String> {
607    char::from_u32(symbol.cast_unsigned()).map(|ch| format!("'{ch}'"))
608}
antlr4_runtime/lexer.rs

antlr4_runtime/
lexer.rs
