antlr4_runtime/
token.rs

1use crate::char_stream::TextInterval;
2use std::fmt;
3use std::ops::Range;
4use std::rc::Rc;
5
6pub const TOKEN_EOF: i32 = -1;
7pub const INVALID_TOKEN_TYPE: i32 = 0;
8pub const DEFAULT_CHANNEL: i32 = 0;
9pub const HIDDEN_CHANNEL: i32 = 1;
10
11#[derive(Clone, Copy, Debug, Eq, PartialEq)]
12pub enum TokenChannel {
13    Default,
14    Hidden,
15    Custom(i32),
16}
17
18impl TokenChannel {
19    pub const fn value(self) -> i32 {
20        match self {
21            Self::Default => DEFAULT_CHANNEL,
22            Self::Hidden => HIDDEN_CHANNEL,
23            Self::Custom(channel) => channel,
24        }
25    }
26}
27
28impl From<i32> for TokenChannel {
29    fn from(value: i32) -> Self {
30        match value {
31            DEFAULT_CHANNEL => Self::Default,
32            HIDDEN_CHANNEL => Self::Hidden,
33            other => Self::Custom(other),
34        }
35    }
36}
37
38pub trait Token: fmt::Debug {
39    fn token_type(&self) -> i32;
40    fn channel(&self) -> i32;
41    /// Zero-based absolute start index measured in Unicode scalar values.
42    fn start(&self) -> usize;
43    /// Zero-based absolute inclusive stop index measured in Unicode scalar
44    /// values.
45    fn stop(&self) -> usize;
46    fn token_index(&self) -> isize;
47    /// One-based source line where the token starts.
48    fn line(&self) -> usize;
49    /// Zero-based source column where the token starts, measured in Unicode
50    /// scalar values from the start of `line`.
51    fn column(&self) -> usize;
52    fn text(&self) -> Option<&str>;
53    fn source_name(&self) -> &str;
54
55    fn interval(&self) -> TextInterval {
56        TextInterval::new(self.start(), self.stop())
57    }
58
59    /// Zero-based absolute start offset measured in UTF-8 bytes.
60    ///
61    /// The default implementation treats the character index as a byte offset,
62    /// which is exact for ASCII and preserves compatibility for token
63    /// implementations that do not expose source byte bounds.
64    fn start_byte(&self) -> usize {
65        self.start()
66    }
67
68    /// Zero-based exclusive end offset measured in UTF-8 bytes.
69    ///
70    /// Unlike [`Self::stop`], this is exclusive so
71    /// `token.start_byte()..token.stop_byte()` can slice the original UTF-8
72    /// source when the token carries source byte bounds. The default
73    /// implementation treats character indices as byte offsets.
74    fn stop_byte(&self) -> usize {
75        default_stop_byte(self.start(), self.stop())
76    }
77
78    /// Zero-based UTF-8 byte span for the token text.
79    fn byte_span(&self) -> Range<usize> {
80        self.start_byte()..self.stop_byte()
81    }
82}
83
84#[derive(Clone, Debug, Eq, PartialEq)]
85pub struct CommonToken {
86    token_type: i32,
87    channel: i32,
88    start: usize,
89    stop: usize,
90    token_index: isize,
91    line: usize,
92    column: usize,
93    text: Option<TokenText>,
94    byte_span: Option<TokenByteSpan>,
95    source_name: Rc<str>,
96}
97
98#[derive(Clone, Copy, Debug, Eq, PartialEq)]
99struct TokenByteSpan {
100    start_byte: u32,
101    stop_byte: u32,
102}
103
104#[derive(Clone, Debug, Eq, PartialEq)]
105enum TokenText {
106    Explicit(Rc<str>),
107    Source {
108        input: Rc<str>,
109        start_byte: u32,
110        stop_byte: u32,
111    },
112}
113
114impl TokenText {
115    fn as_str(&self) -> &str {
116        match self {
117            Self::Explicit(text) => text.as_ref(),
118            Self::Source {
119                input,
120                start_byte,
121                stop_byte,
122            } => &input[*start_byte as usize..*stop_byte as usize],
123        }
124    }
125}
126
127#[derive(Clone, Debug, Eq, PartialEq)]
128pub struct TokenSourceText {
129    pub input: Rc<str>,
130    pub start_byte: u32,
131    pub stop_byte: u32,
132}
133
134#[derive(Debug)]
135pub struct TokenSpec<'a> {
136    pub token_type: i32,
137    pub channel: i32,
138    pub start: usize,
139    pub stop: usize,
140    pub line: usize,
141    pub column: usize,
142    pub text: Option<String>,
143    pub source_text: Option<TokenSourceText>,
144    pub source_name: &'a str,
145}
146
147impl CommonToken {
148    pub fn new(token_type: i32) -> Self {
149        Self {
150            token_type,
151            channel: DEFAULT_CHANNEL,
152            start: 0,
153            stop: 0,
154            token_index: -1,
155            line: 1,
156            column: 0,
157            text: None,
158            byte_span: None,
159            source_name: Rc::from(""),
160        }
161    }
162
163    pub fn eof(source_name: impl Into<Rc<str>>, index: usize, line: usize, column: usize) -> Self {
164        Self {
165            token_type: TOKEN_EOF,
166            channel: DEFAULT_CHANNEL,
167            start: index,
168            stop: index.checked_sub(1).unwrap_or(usize::MAX),
169            token_index: -1,
170            line,
171            column,
172            text: Some(TokenText::Explicit(Rc::from("<EOF>"))),
173            byte_span: None,
174            source_name: source_name.into(),
175        }
176    }
177
178    #[must_use]
179    pub fn with_text(mut self, text: impl Into<Rc<str>>) -> Self {
180        self.text = Some(TokenText::Explicit(text.into()));
181        self
182    }
183
184    #[must_use]
185    pub fn with_source_text(mut self, input: Rc<str>, start_byte: u32, stop_byte: u32) -> Self {
186        debug_assert!(
187            start_byte <= stop_byte && stop_byte as usize <= input.len(),
188            "invalid token source-text bounds: start={start_byte}, stop={stop_byte}, len={}",
189            input.len()
190        );
191        self.text = Some(TokenText::Source {
192            input,
193            start_byte,
194            stop_byte,
195        });
196        self.byte_span = Some(TokenByteSpan {
197            start_byte,
198            stop_byte,
199        });
200        self
201    }
202
203    #[must_use]
204    pub(crate) fn with_byte_span(mut self, start_byte: u32, stop_byte: u32) -> Self {
205        debug_assert!(
206            start_byte <= stop_byte,
207            "invalid token byte span: start={start_byte}, stop={stop_byte}"
208        );
209        self.byte_span = Some(TokenByteSpan {
210            start_byte,
211            stop_byte,
212        });
213        self
214    }
215
216    #[must_use]
217    pub const fn with_span(mut self, start: usize, stop: usize) -> Self {
218        self.start = start;
219        self.stop = stop;
220        self
221    }
222
223    #[must_use]
224    pub const fn with_position(mut self, line: usize, column: usize) -> Self {
225        self.line = line;
226        self.column = column;
227        self
228    }
229
230    #[must_use]
231    pub const fn with_channel(mut self, channel: i32) -> Self {
232        self.channel = channel;
233        self
234    }
235
236    #[must_use]
237    pub fn with_source_name(mut self, source_name: impl Into<Rc<str>>) -> Self {
238        self.source_name = source_name.into();
239        self
240    }
241
242    pub const fn set_token_index(&mut self, token_index: isize) {
243        self.token_index = token_index;
244    }
245
246    const fn source_byte_span(&self) -> Option<Range<usize>> {
247        match self.byte_span {
248            Some(TokenByteSpan {
249                start_byte,
250                stop_byte,
251            }) => Some(start_byte as usize..stop_byte as usize),
252            None => None,
253        }
254    }
255}
256
257impl Token for CommonToken {
258    fn token_type(&self) -> i32 {
259        self.token_type
260    }
261
262    fn channel(&self) -> i32 {
263        self.channel
264    }
265
266    fn start(&self) -> usize {
267        self.start
268    }
269
270    fn stop(&self) -> usize {
271        self.stop
272    }
273
274    fn token_index(&self) -> isize {
275        self.token_index
276    }
277
278    fn line(&self) -> usize {
279        self.line
280    }
281
282    fn column(&self) -> usize {
283        self.column
284    }
285
286    fn text(&self) -> Option<&str> {
287        self.text.as_ref().map(TokenText::as_str)
288    }
289
290    fn source_name(&self) -> &str {
291        self.source_name.as_ref()
292    }
293
294    fn start_byte(&self) -> usize {
295        self.source_byte_span()
296            .map_or(self.start, |byte_span| byte_span.start)
297    }
298
299    fn stop_byte(&self) -> usize {
300        self.source_byte_span()
301            .map_or_else(|| default_stop_byte(self.start, self.stop), |span| span.end)
302    }
303}
304
305impl CommonToken {
306    /// The token's text, empty when unset — ANTLR's `getText()` shape, which
307    /// generated test actions print directly (`token.text()` in `{}`).
308    ///
309    /// This inherent method intentionally shadows the Option-returning
310    /// [`Token::text`] trait method on concrete `CommonToken` values; generic
311    /// code keeps the trait signature.
312    #[must_use]
313    pub fn text(&self) -> &str {
314        self.text.as_ref().map_or("", TokenText::as_str)
315    }
316}
317
318impl fmt::Display for CommonToken {
319    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
320        let text = Self::text(self);
321        let channel = if self.channel() == DEFAULT_CHANNEL {
322            String::new()
323        } else {
324            format!(",channel={}", self.channel())
325        };
326        write!(
327            f,
328            "[@{},{}:{}='{}',<{}>{},{}:{}]",
329            self.token_index(),
330            display_token_boundary(self.start()),
331            display_token_boundary(self.stop()),
332            display_text(text),
333            self.token_type(),
334            channel,
335            self.line(),
336            self.column()
337        )
338    }
339}
340
341/// Formats synthetic-token boundaries with ANTLR's `-1` sentinel.
342fn display_token_boundary(value: usize) -> String {
343    if value == usize::MAX {
344        "-1".to_owned()
345    } else {
346        value.to_string()
347    }
348}
349
350const fn default_stop_byte(start: usize, stop: usize) -> usize {
351    match stop.checked_add(1) {
352        Some(end) if end >= start => end,
353        Some(_) | None => start,
354    }
355}
356
357/// Escapes token text the way ANTLR's token display format expects.
358///
359/// Debug escaping is close but not identical: ANTLR leaves ordinary
360/// backslashes and quotes unescaped, and only normalizes control characters
361/// that would otherwise disrupt the one-line token representation.
362fn display_text(text: &str) -> String {
363    let mut out = String::new();
364    for ch in text.chars() {
365        match ch {
366            '\n' => out.push_str("\\n"),
367            '\r' => out.push_str("\\r"),
368            '\t' => out.push_str("\\t"),
369            other => out.push(other),
370        }
371    }
372    out
373}
374
375pub type TokenRef = Rc<CommonToken>;
376
377pub trait TokenFactory {
378    fn create(&self, spec: TokenSpec<'_>) -> CommonToken;
379}
380
381#[derive(Clone, Debug, Default)]
382pub struct CommonTokenFactory;
383
384impl TokenFactory for CommonTokenFactory {
385    fn create(&self, spec: TokenSpec<'_>) -> CommonToken {
386        let mut token = CommonToken::new(spec.token_type)
387            .with_channel(spec.channel)
388            .with_span(spec.start, spec.stop)
389            .with_position(spec.line, spec.column)
390            .with_source_name(spec.source_name);
391        if let Some(text) = spec.text {
392            token = token.with_text(text);
393        } else if let Some(source_text) = spec.source_text {
394            token = token.with_source_text(
395                source_text.input,
396                source_text.start_byte,
397                source_text.stop_byte,
398            );
399        }
400        token
401    }
402}
403
404/// A diagnostic buffered by a token source while it was producing tokens.
405#[derive(Clone, Debug, Eq, PartialEq)]
406pub struct TokenSourceError {
407    /// One-based input line where the diagnostic starts.
408    pub line: usize,
409    /// Zero-based column within `line` where the diagnostic starts.
410    pub column: usize,
411    /// ANTLR-compatible diagnostic message without the leading line/column.
412    pub message: String,
413}
414
415impl TokenSourceError {
416    /// Creates a token-source diagnostic at the given input position.
417    pub fn new(line: usize, column: usize, message: impl Into<String>) -> Self {
418        Self {
419            line,
420            column,
421            message: message.into(),
422        }
423    }
424}
425
426pub trait TokenSource {
427    fn next_token(&mut self) -> CommonToken;
428    fn line(&self) -> usize;
429    fn column(&self) -> usize;
430    fn source_name(&self) -> &str;
431    /// Returns and clears diagnostics emitted while fetching tokens.
432    fn drain_errors(&mut self) -> Vec<TokenSourceError> {
433        Vec::new()
434    }
435
436    /// Serializes lexer DFA cache state when the token source exposes one.
437    fn lexer_dfa_string(&self) -> String {
438        String::new()
439    }
440}
441
442#[cfg(test)]
443mod tests {
444    use super::*;
445
446    #[test]
447    fn common_token_display_matches_antlr_shape() {
448        let mut token = CommonToken::new(7)
449            .with_text("abc")
450            .with_span(2, 4)
451            .with_position(3, 9);
452        token.set_token_index(5);
453        assert_eq!(token.to_string(), "[@5,2:4='abc',<7>,3:9]");
454    }
455
456    #[test]
457    fn common_token_display_matches_antlr_escaping() {
458        let quote = CommonToken::new(1).with_text("\"");
459        assert_eq!(quote.to_string(), "[@-1,0:0='\"',<1>,1:0]");
460
461        let newline = CommonToken::new(1).with_text("\n");
462        assert_eq!(newline.to_string(), "[@-1,0:0='\\n',<1>,1:0]");
463
464        let backslash = CommonToken::new(1).with_text("\\");
465        assert_eq!(backslash.to_string(), "[@-1,0:0='\\',<1>,1:0]");
466    }
467
468    #[test]
469    fn common_token_display_includes_non_default_channel() {
470        let token = CommonToken::new(2).with_text("b").with_channel(2);
471        assert_eq!(token.to_string(), "[@-1,0:0='b',<2>,channel=2,1:0]");
472    }
473
474    #[test]
475    fn eof_display_uses_antlr_empty_input_stop_index() {
476        let token = CommonToken::eof("", 0, 1, 0);
477        assert_eq!(token.to_string(), "[@-1,0:-1='<EOF>',<-1>,1:0]");
478    }
479
480    #[test]
481    fn source_backed_token_exposes_utf8_byte_span() {
482        let source: Rc<str> = Rc::from("éβz");
483        let token = CommonToken::new(1)
484            .with_span(1, 1)
485            .with_source_text(source, 2, 4);
486
487        assert_eq!(token.start(), 1);
488        assert_eq!(token.stop(), 1);
489        assert_eq!(token.start_byte(), 2);
490        assert_eq!(token.stop_byte(), 4);
491        assert_eq!(token.byte_span(), 2..4);
492        assert_eq!(token.text(), "β");
493    }
494
495    #[test]
496    fn explicit_text_byte_span_falls_back_to_character_span() {
497        let token = CommonToken::new(1).with_text("β").with_span(3, 3);
498
499        assert_eq!(token.byte_span(), 3..4);
500    }
501
502    #[test]
503    fn explicit_text_can_carry_utf8_byte_span() {
504        let token = CommonToken::new(TOKEN_EOF)
505            .with_text("<EOF>")
506            .with_span(1, 0)
507            .with_byte_span(2, 2);
508
509        assert_eq!(token.text(), "<EOF>");
510        assert_eq!(token.byte_span(), 2..2);
511    }
512}
antlr4_runtime/token.rs

antlr4_runtime/
token.rs
