antlr4_runtime/
token.rs

1use crate::char_stream::TextInterval;
2use std::fmt;
3use std::rc::Rc;
4
5pub const TOKEN_EOF: i32 = -1;
6pub const INVALID_TOKEN_TYPE: i32 = 0;
7pub const DEFAULT_CHANNEL: i32 = 0;
8pub const HIDDEN_CHANNEL: i32 = 1;
9
10#[derive(Clone, Copy, Debug, Eq, PartialEq)]
11pub enum TokenChannel {
12    Default,
13    Hidden,
14    Custom(i32),
15}
16
17impl TokenChannel {
18    pub const fn value(self) -> i32 {
19        match self {
20            Self::Default => DEFAULT_CHANNEL,
21            Self::Hidden => HIDDEN_CHANNEL,
22            Self::Custom(channel) => channel,
23        }
24    }
25}
26
27impl From<i32> for TokenChannel {
28    fn from(value: i32) -> Self {
29        match value {
30            DEFAULT_CHANNEL => Self::Default,
31            HIDDEN_CHANNEL => Self::Hidden,
32            other => Self::Custom(other),
33        }
34    }
35}
36
37pub trait Token: fmt::Debug {
38    fn token_type(&self) -> i32;
39    fn channel(&self) -> i32;
40    fn start(&self) -> usize;
41    fn stop(&self) -> usize;
42    fn token_index(&self) -> isize;
43    fn line(&self) -> usize;
44    fn column(&self) -> usize;
45    fn text(&self) -> Option<&str>;
46    fn source_name(&self) -> &str;
47
48    fn interval(&self) -> TextInterval {
49        TextInterval::new(self.start(), self.stop())
50    }
51}
52
53#[derive(Clone, Debug, Eq, PartialEq)]
54pub struct CommonToken {
55    token_type: i32,
56    channel: i32,
57    start: usize,
58    stop: usize,
59    token_index: isize,
60    line: usize,
61    column: usize,
62    text: Option<TokenText>,
63    source_name: Rc<str>,
64}
65
66#[derive(Clone, Debug, Eq, PartialEq)]
67enum TokenText {
68    Explicit(Rc<str>),
69    Source {
70        input: Rc<str>,
71        start_byte: u32,
72        stop_byte: u32,
73    },
74}
75
76impl TokenText {
77    fn as_str(&self) -> &str {
78        match self {
79            Self::Explicit(text) => text.as_ref(),
80            Self::Source {
81                input,
82                start_byte,
83                stop_byte,
84            } => &input[*start_byte as usize..*stop_byte as usize],
85        }
86    }
87}
88
89#[derive(Clone, Debug, Eq, PartialEq)]
90pub struct TokenSourceText {
91    pub input: Rc<str>,
92    pub start_byte: u32,
93    pub stop_byte: u32,
94}
95
96#[derive(Debug)]
97pub struct TokenSpec<'a> {
98    pub token_type: i32,
99    pub channel: i32,
100    pub start: usize,
101    pub stop: usize,
102    pub line: usize,
103    pub column: usize,
104    pub text: Option<String>,
105    pub source_text: Option<TokenSourceText>,
106    pub source_name: &'a str,
107}
108
109impl CommonToken {
110    pub fn new(token_type: i32) -> Self {
111        Self {
112            token_type,
113            channel: DEFAULT_CHANNEL,
114            start: 0,
115            stop: 0,
116            token_index: -1,
117            line: 1,
118            column: 0,
119            text: None,
120            source_name: Rc::from(""),
121        }
122    }
123
124    pub fn eof(source_name: impl Into<Rc<str>>, index: usize, line: usize, column: usize) -> Self {
125        Self {
126            token_type: TOKEN_EOF,
127            channel: DEFAULT_CHANNEL,
128            start: index,
129            stop: index.checked_sub(1).unwrap_or(usize::MAX),
130            token_index: -1,
131            line,
132            column,
133            text: Some(TokenText::Explicit(Rc::from("<EOF>"))),
134            source_name: source_name.into(),
135        }
136    }
137
138    #[must_use]
139    pub fn with_text(mut self, text: impl Into<Rc<str>>) -> Self {
140        self.text = Some(TokenText::Explicit(text.into()));
141        self
142    }
143
144    #[must_use]
145    pub fn with_source_text(mut self, input: Rc<str>, start_byte: u32, stop_byte: u32) -> Self {
146        debug_assert!(
147            start_byte <= stop_byte && stop_byte as usize <= input.len(),
148            "invalid token source-text bounds: start={start_byte}, stop={stop_byte}, len={}",
149            input.len()
150        );
151        self.text = Some(TokenText::Source {
152            input,
153            start_byte,
154            stop_byte,
155        });
156        self
157    }
158
159    #[must_use]
160    pub const fn with_span(mut self, start: usize, stop: usize) -> Self {
161        self.start = start;
162        self.stop = stop;
163        self
164    }
165
166    #[must_use]
167    pub const fn with_position(mut self, line: usize, column: usize) -> Self {
168        self.line = line;
169        self.column = column;
170        self
171    }
172
173    #[must_use]
174    pub const fn with_channel(mut self, channel: i32) -> Self {
175        self.channel = channel;
176        self
177    }
178
179    #[must_use]
180    pub fn with_source_name(mut self, source_name: impl Into<Rc<str>>) -> Self {
181        self.source_name = source_name.into();
182        self
183    }
184
185    pub const fn set_token_index(&mut self, token_index: isize) {
186        self.token_index = token_index;
187    }
188}
189
190impl Token for CommonToken {
191    fn token_type(&self) -> i32 {
192        self.token_type
193    }
194
195    fn channel(&self) -> i32 {
196        self.channel
197    }
198
199    fn start(&self) -> usize {
200        self.start
201    }
202
203    fn stop(&self) -> usize {
204        self.stop
205    }
206
207    fn token_index(&self) -> isize {
208        self.token_index
209    }
210
211    fn line(&self) -> usize {
212        self.line
213    }
214
215    fn column(&self) -> usize {
216        self.column
217    }
218
219    fn text(&self) -> Option<&str> {
220        self.text.as_ref().map(TokenText::as_str)
221    }
222
223    fn source_name(&self) -> &str {
224        self.source_name.as_ref()
225    }
226}
227
228impl fmt::Display for CommonToken {
229    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
230        let text = self.text().unwrap_or("");
231        let channel = if self.channel() == DEFAULT_CHANNEL {
232            String::new()
233        } else {
234            format!(",channel={}", self.channel())
235        };
236        write!(
237            f,
238            "[@{},{}:{}='{}',<{}>{},{}:{}]",
239            self.token_index(),
240            display_token_boundary(self.start()),
241            display_token_boundary(self.stop()),
242            display_text(text),
243            self.token_type(),
244            channel,
245            self.line(),
246            self.column()
247        )
248    }
249}
250
251/// Formats synthetic-token boundaries with ANTLR's `-1` sentinel.
252fn display_token_boundary(value: usize) -> String {
253    if value == usize::MAX {
254        "-1".to_owned()
255    } else {
256        value.to_string()
257    }
258}
259
260/// Escapes token text the way ANTLR's token display format expects.
261///
262/// Debug escaping is close but not identical: ANTLR leaves ordinary
263/// backslashes and quotes unescaped, and only normalizes control characters
264/// that would otherwise disrupt the one-line token representation.
265fn display_text(text: &str) -> String {
266    let mut out = String::new();
267    for ch in text.chars() {
268        match ch {
269            '\n' => out.push_str("\\n"),
270            '\r' => out.push_str("\\r"),
271            '\t' => out.push_str("\\t"),
272            other => out.push(other),
273        }
274    }
275    out
276}
277
278pub type TokenRef = Rc<CommonToken>;
279
280pub trait TokenFactory {
281    fn create(&self, spec: TokenSpec<'_>) -> CommonToken;
282}
283
284#[derive(Clone, Debug, Default)]
285pub struct CommonTokenFactory;
286
287impl TokenFactory for CommonTokenFactory {
288    fn create(&self, spec: TokenSpec<'_>) -> CommonToken {
289        let mut token = CommonToken::new(spec.token_type)
290            .with_channel(spec.channel)
291            .with_span(spec.start, spec.stop)
292            .with_position(spec.line, spec.column)
293            .with_source_name(spec.source_name);
294        if let Some(text) = spec.text {
295            token = token.with_text(text);
296        } else if let Some(source_text) = spec.source_text {
297            token = token.with_source_text(
298                source_text.input,
299                source_text.start_byte,
300                source_text.stop_byte,
301            );
302        }
303        token
304    }
305}
306
307/// A diagnostic buffered by a token source while it was producing tokens.
308#[derive(Clone, Debug, Eq, PartialEq)]
309pub struct TokenSourceError {
310    /// One-based input line where the diagnostic starts.
311    pub line: usize,
312    /// Zero-based column within `line` where the diagnostic starts.
313    pub column: usize,
314    /// ANTLR-compatible diagnostic message without the leading line/column.
315    pub message: String,
316}
317
318impl TokenSourceError {
319    /// Creates a token-source diagnostic at the given input position.
320    pub fn new(line: usize, column: usize, message: impl Into<String>) -> Self {
321        Self {
322            line,
323            column,
324            message: message.into(),
325        }
326    }
327}
328
329pub trait TokenSource {
330    fn next_token(&mut self) -> CommonToken;
331    fn line(&self) -> usize;
332    fn column(&self) -> usize;
333    fn source_name(&self) -> &str;
334    /// Returns and clears diagnostics emitted while fetching tokens.
335    fn drain_errors(&mut self) -> Vec<TokenSourceError> {
336        Vec::new()
337    }
338
339    /// Serializes lexer DFA cache state when the token source exposes one.
340    fn lexer_dfa_string(&self) -> String {
341        String::new()
342    }
343}
344
345#[cfg(test)]
346mod tests {
347    use super::*;
348
349    #[test]
350    fn common_token_display_matches_antlr_shape() {
351        let mut token = CommonToken::new(7)
352            .with_text("abc")
353            .with_span(2, 4)
354            .with_position(3, 9);
355        token.set_token_index(5);
356        assert_eq!(token.to_string(), "[@5,2:4='abc',<7>,3:9]");
357    }
358
359    #[test]
360    fn common_token_display_matches_antlr_escaping() {
361        let quote = CommonToken::new(1).with_text("\"");
362        assert_eq!(quote.to_string(), "[@-1,0:0='\"',<1>,1:0]");
363
364        let newline = CommonToken::new(1).with_text("\n");
365        assert_eq!(newline.to_string(), "[@-1,0:0='\\n',<1>,1:0]");
366
367        let backslash = CommonToken::new(1).with_text("\\");
368        assert_eq!(backslash.to_string(), "[@-1,0:0='\\',<1>,1:0]");
369    }
370
371    #[test]
372    fn common_token_display_includes_non_default_channel() {
373        let token = CommonToken::new(2).with_text("b").with_channel(2);
374        assert_eq!(token.to_string(), "[@-1,0:0='b',<2>,channel=2,1:0]");
375    }
376
377    #[test]
378    fn eof_display_uses_antlr_empty_input_stop_index() {
379        let token = CommonToken::eof("", 0, 1, 0);
380        assert_eq!(token.to_string(), "[@-1,0:-1='<EOF>',<-1>,1:0]");
381    }
382}
antlr4_runtime/token.rs

antlr4_runtime/
token.rs
