antlr4_runtime/
token.rs

1use crate::char_stream::TextInterval;
2use std::fmt;
3use std::ops::Range;
4use std::rc::Rc;
5
6pub const TOKEN_EOF: i32 = -1;
7pub const INVALID_TOKEN_TYPE: i32 = 0;
8pub const DEFAULT_CHANNEL: i32 = 0;
9pub const HIDDEN_CHANNEL: i32 = 1;
10
11/// Largest source or location offset accepted by the compact token store.
12///
13/// `u32::MAX` is reserved for ANTLR's synthetic `-1` source boundary.
14pub const MAX_TOKEN_OFFSET: usize = (u32::MAX - 1) as usize;
15
16#[repr(transparent)]
17#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
18pub struct TokenId(u32);
19
20impl TokenId {
21    #[must_use]
22    pub const fn index(self) -> usize {
23        self.0 as usize
24    }
25}
26
27impl TryFrom<usize> for TokenId {
28    type Error = TokenStoreError;
29
30    fn try_from(value: usize) -> Result<Self, Self::Error> {
31        u32::try_from(value)
32            .map(Self)
33            .map_err(|_| TokenStoreError::overflow("index", value, u32::MAX as usize))
34    }
35}
36
37#[derive(Clone, Copy, Debug, Eq, PartialEq)]
38pub enum TokenChannel {
39    Default,
40    Hidden,
41    Custom(i32),
42}
43
44impl TokenChannel {
45    pub const fn value(self) -> i32 {
46        match self {
47            Self::Default => DEFAULT_CHANNEL,
48            Self::Hidden => HIDDEN_CHANNEL,
49            Self::Custom(channel) => channel,
50        }
51    }
52}
53
54impl From<i32> for TokenChannel {
55    fn from(value: i32) -> Self {
56        match value {
57            DEFAULT_CHANNEL => Self::Default,
58            HIDDEN_CHANNEL => Self::Hidden,
59            other => Self::Custom(other),
60        }
61    }
62}
63
64pub trait Token: fmt::Debug {
65    fn token_id(&self) -> TokenId;
66    fn token_type(&self) -> i32;
67    fn channel(&self) -> i32;
68    /// Zero-based absolute start index measured in Unicode scalar values.
69    fn start(&self) -> usize;
70    /// Zero-based absolute inclusive stop index measured in Unicode scalar
71    /// values.
72    fn stop(&self) -> usize;
73    /// One-based source line where the token starts.
74    fn line(&self) -> usize;
75    /// Zero-based source column where the token starts, measured in Unicode
76    /// scalar values from the start of `line`.
77    fn column(&self) -> usize;
78    fn text(&self) -> Option<&str>;
79    fn source_name(&self) -> &str;
80
81    fn interval(&self) -> TextInterval {
82        TextInterval::new(self.start(), self.stop())
83    }
84
85    /// Zero-based absolute start offset measured in UTF-8 bytes.
86    fn start_byte(&self) -> usize;
87
88    /// Zero-based exclusive end offset measured in UTF-8 bytes.
89    fn stop_byte(&self) -> usize;
90
91    /// Zero-based UTF-8 byte span for the token text.
92    fn byte_span(&self) -> Range<usize> {
93        self.start_byte()..self.stop_byte()
94    }
95}
96
97impl<T: Token + ?Sized> Token for &T {
98    fn token_id(&self) -> TokenId {
99        (**self).token_id()
100    }
101
102    fn token_type(&self) -> i32 {
103        (**self).token_type()
104    }
105
106    fn channel(&self) -> i32 {
107        (**self).channel()
108    }
109
110    fn start(&self) -> usize {
111        (**self).start()
112    }
113
114    fn stop(&self) -> usize {
115        (**self).stop()
116    }
117
118    fn line(&self) -> usize {
119        (**self).line()
120    }
121
122    fn column(&self) -> usize {
123        (**self).column()
124    }
125
126    fn text(&self) -> Option<&str> {
127        (**self).text()
128    }
129
130    fn source_name(&self) -> &str {
131        (**self).source_name()
132    }
133
134    fn start_byte(&self) -> usize {
135        (**self).start_byte()
136    }
137
138    fn stop_byte(&self) -> usize {
139        (**self).stop_byte()
140    }
141}
142
143/// The fields emitted for one token.
144///
145/// This is transient sink input, not an owned token representation. Source
146/// text and the source name live once in [`TokenStore`].
147#[derive(Clone, Debug)]
148pub struct TokenSpec {
149    pub token_type: i32,
150    pub channel: i32,
151    pub start: usize,
152    pub stop: usize,
153    pub start_byte: usize,
154    pub stop_byte: usize,
155    pub line: usize,
156    pub column: usize,
157    pub text: Option<String>,
158    pub source_backed: bool,
159}
160
161impl TokenSpec {
162    #[must_use]
163    pub fn explicit(token_type: i32, text: impl Into<String>) -> Self {
164        Self {
165            token_type,
166            channel: DEFAULT_CHANNEL,
167            start: 0,
168            stop: 0,
169            start_byte: 0,
170            stop_byte: 1,
171            line: 1,
172            column: 0,
173            text: Some(text.into()),
174            source_backed: false,
175        }
176    }
177
178    #[must_use]
179    pub fn eof(index: usize, byte_offset: usize, line: usize, column: usize) -> Self {
180        Self {
181            token_type: TOKEN_EOF,
182            channel: DEFAULT_CHANNEL,
183            start: index,
184            stop: index.checked_sub(1).unwrap_or(usize::MAX),
185            start_byte: byte_offset,
186            stop_byte: byte_offset,
187            line,
188            column,
189            text: Some("<EOF>".to_owned()),
190            source_backed: false,
191        }
192    }
193
194    #[must_use]
195    pub const fn with_channel(mut self, channel: i32) -> Self {
196        self.channel = channel;
197        self
198    }
199
200    #[must_use]
201    pub const fn with_span(mut self, start: usize, stop: usize) -> Self {
202        self.start = start;
203        self.stop = stop;
204        self.start_byte = start;
205        self.stop_byte = default_stop_byte(start, stop);
206        self
207    }
208
209    #[must_use]
210    pub const fn with_byte_span(mut self, start_byte: usize, stop_byte: usize) -> Self {
211        self.start_byte = start_byte;
212        self.stop_byte = stop_byte;
213        self
214    }
215
216    #[must_use]
217    pub const fn with_position(mut self, line: usize, column: usize) -> Self {
218        self.line = line;
219        self.column = column;
220        self
221    }
222}
223
224#[derive(Clone, Debug, Eq, PartialEq)]
225pub struct TokenStoreError(TokenStoreErrorKind);
226
227impl TokenStoreError {
228    const fn overflow(field: &'static str, value: usize, limit: usize) -> Self {
229        Self(TokenStoreErrorKind::Overflow {
230            field,
231            value,
232            limit,
233        })
234    }
235
236    const fn invalid_source_boundary(offset: usize, source_len: usize) -> Self {
237        Self(TokenStoreErrorKind::InvalidSourceBoundary { offset, source_len })
238    }
239
240    pub(crate) const fn invalid_source_output(
241        expected_id: usize,
242        returned_id: usize,
243        appended: usize,
244    ) -> Self {
245        Self(TokenStoreErrorKind::InvalidSourceOutput {
246            expected_id,
247            returned_id,
248            appended,
249        })
250    }
251}
252
253impl fmt::Display for TokenStoreError {
254    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
255        match self.0 {
256            TokenStoreErrorKind::Overflow {
257                field,
258                value,
259                limit,
260            } => write!(
261                f,
262                "token {field} {value} exceeds the supported limit {limit}"
263            ),
264            TokenStoreErrorKind::InvalidSourceBoundary { offset, source_len } => write!(
265                f,
266                "token source byte offset {offset} is not a UTF-8 character boundary \
267                 for source length {source_len}"
268            ),
269            TokenStoreErrorKind::InvalidSourceOutput {
270                expected_id,
271                returned_id,
272                appended,
273            } => write!(
274                f,
275                "token source must append exactly one token and return ID {expected_id}, \
276                 but appended {appended} and returned ID {returned_id}"
277            ),
278        }
279    }
280}
281
282impl std::error::Error for TokenStoreError {}
283
284#[derive(Clone, Debug, Eq, PartialEq)]
285enum TokenStoreErrorKind {
286    Overflow {
287        field: &'static str,
288        value: usize,
289        limit: usize,
290    },
291    InvalidSourceBoundary {
292        offset: usize,
293        source_len: usize,
294    },
295    InvalidSourceOutput {
296        expected_id: usize,
297        returned_id: usize,
298        appended: usize,
299    },
300}
301
302/// Canonical compact storage for every token associated with one token stream.
303#[derive(Debug)]
304pub struct TokenStore {
305    source: Option<Rc<str>>,
306    source_name: Rc<str>,
307    token_types: Vec<i32>,
308    channels: Vec<i32>,
309    scalar_starts: Vec<u32>,
310    scalar_stops: Vec<u32>,
311    byte_starts: Vec<u32>,
312    byte_stops: Vec<u32>,
313    lines: Vec<u32>,
314    columns: Vec<u32>,
315    source_backed: Vec<bool>,
316    explicit_text: Vec<(TokenId, Rc<str>)>,
317}
318
319impl TokenStore {
320    pub(crate) fn new(source: Option<Rc<str>>, source_name: impl Into<Rc<str>>) -> Self {
321        Self {
322            source,
323            source_name: source_name.into(),
324            token_types: Vec::new(),
325            channels: Vec::new(),
326            scalar_starts: Vec::new(),
327            scalar_stops: Vec::new(),
328            byte_starts: Vec::new(),
329            byte_stops: Vec::new(),
330            lines: Vec::new(),
331            columns: Vec::new(),
332            source_backed: Vec::new(),
333            explicit_text: Vec::new(),
334        }
335    }
336
337    #[must_use]
338    pub const fn len(&self) -> usize {
339        self.token_types.len()
340    }
341
342    #[must_use]
343    pub const fn is_empty(&self) -> bool {
344        self.token_types.is_empty()
345    }
346
347    pub(crate) fn push(&mut self, spec: TokenSpec) -> Result<TokenId, TokenStoreError> {
348        let raw_id = u32::try_from(self.len())
349            .map_err(|_| TokenStoreError::overflow("count", self.len(), u32::MAX as usize))?;
350        let id = TokenId(raw_id);
351        let scalar_start = compact_boundary("start offset", spec.start)?;
352        let scalar_stop = compact_boundary("stop offset", spec.stop)?;
353        let byte_start = compact_offset("start byte", spec.start_byte)?;
354        let byte_stop = compact_offset("stop byte", spec.stop_byte)?;
355        let line = compact_offset("line", spec.line)?;
356        let column = compact_offset("column", spec.column)?;
357
358        if spec.source_backed {
359            let Some(source) = self.source.as_ref() else {
360                return Err(TokenStoreError::overflow("source text", 1, 0));
361            };
362            if spec.start_byte > spec.stop_byte || spec.stop_byte > source.len() {
363                return Err(TokenStoreError::overflow(
364                    "source byte span",
365                    spec.stop_byte,
366                    source.len(),
367                ));
368            }
369            if !source.is_char_boundary(spec.start_byte) {
370                return Err(TokenStoreError::invalid_source_boundary(
371                    spec.start_byte,
372                    source.len(),
373                ));
374            }
375            if !source.is_char_boundary(spec.stop_byte) {
376                return Err(TokenStoreError::invalid_source_boundary(
377                    spec.stop_byte,
378                    source.len(),
379                ));
380            }
381        }
382
383        self.token_types.push(spec.token_type);
384        self.channels.push(spec.channel);
385        self.scalar_starts.push(scalar_start);
386        self.scalar_stops.push(scalar_stop);
387        self.byte_starts.push(byte_start);
388        self.byte_stops.push(byte_stop);
389        self.lines.push(line);
390        self.columns.push(column);
391        self.source_backed.push(spec.source_backed);
392        if let Some(text) = spec.text {
393            self.explicit_text.push((id, Rc::from(text)));
394        }
395        Ok(id)
396    }
397
398    const fn contains(&self, id: TokenId) -> bool {
399        id.index() < self.len()
400    }
401
402    /// Returns a borrowing view of one token record.
403    #[must_use]
404    pub fn view(&self, id: TokenId) -> Option<TokenView<'_>> {
405        self.contains(id).then_some(TokenView { store: self, id })
406    }
407
408    /// Returns the token type for `id`.
409    #[must_use]
410    pub fn token_type(&self, id: TokenId) -> Option<i32> {
411        self.token_types.get(id.index()).copied()
412    }
413
414    /// Returns the token channel for `id`.
415    #[must_use]
416    pub fn channel(&self, id: TokenId) -> Option<i32> {
417        self.channels.get(id.index()).copied()
418    }
419
420    /// Returns the token's zero-based scalar start offset.
421    #[must_use]
422    pub fn start(&self, id: TokenId) -> Option<usize> {
423        self.scalar_starts
424            .get(id.index())
425            .copied()
426            .map(expand_boundary)
427    }
428
429    /// Returns the token's zero-based inclusive scalar stop offset.
430    #[must_use]
431    pub fn stop(&self, id: TokenId) -> Option<usize> {
432        self.scalar_stops
433            .get(id.index())
434            .copied()
435            .map(expand_boundary)
436    }
437
438    /// Returns the token's one-based source line.
439    #[must_use]
440    pub fn line(&self, id: TokenId) -> Option<usize> {
441        self.lines.get(id.index()).map(|line| *line as usize)
442    }
443
444    /// Returns the token's zero-based source column.
445    #[must_use]
446    pub fn column(&self, id: TokenId) -> Option<usize> {
447        self.columns.get(id.index()).map(|column| *column as usize)
448    }
449
450    /// Returns the token's zero-based UTF-8 byte start offset.
451    #[must_use]
452    pub fn start_byte(&self, id: TokenId) -> Option<usize> {
453        self.byte_starts
454            .get(id.index())
455            .map(|offset| *offset as usize)
456    }
457
458    /// Returns the token's zero-based exclusive UTF-8 byte stop offset.
459    #[must_use]
460    pub fn stop_byte(&self, id: TokenId) -> Option<usize> {
461        self.byte_stops
462            .get(id.index())
463            .map(|offset| *offset as usize)
464    }
465
466    fn explicit_text(&self, id: TokenId) -> Option<&str> {
467        self.explicit_text
468            .binary_search_by_key(&id, |(token_id, _)| *token_id)
469            .ok()
470            .map(|index| self.explicit_text[index].1.as_ref())
471    }
472
473    /// Returns explicit or source-backed text for `id`.
474    #[must_use]
475    pub fn text(&self, id: TokenId) -> Option<&str> {
476        if let Some(text) = self.explicit_text(id) {
477            return Some(text);
478        }
479        if !self.source_backed.get(id.index()).copied().unwrap_or(false) {
480            return None;
481        }
482        let source = self.source.as_deref()?;
483        let start = self.byte_starts[id.index()] as usize;
484        let stop = self.byte_stops[id.index()] as usize;
485        source.get(start..stop)
486    }
487}
488
489const fn default_stop_byte(start: usize, stop: usize) -> usize {
490    match stop.checked_add(1) {
491        Some(end) if end >= start => end,
492        Some(_) | None => start,
493    }
494}
495
496const fn compact_boundary(field: &'static str, value: usize) -> Result<u32, TokenStoreError> {
497    if value == usize::MAX {
498        return Ok(u32::MAX);
499    }
500    compact_offset(field, value)
501}
502
503const fn compact_offset(field: &'static str, value: usize) -> Result<u32, TokenStoreError> {
504    if value > MAX_TOKEN_OFFSET {
505        return Err(TokenStoreError::overflow(field, value, MAX_TOKEN_OFFSET));
506    }
507    Ok(value as u32)
508}
509
510/// Borrowing public view of one canonical token-store record.
511#[derive(Clone, Copy)]
512pub struct TokenView<'a> {
513    store: &'a TokenStore,
514    id: TokenId,
515}
516
517impl<'a> TokenView<'a> {
518    /// The token's text, empty when no explicit or source-backed text exists.
519    #[must_use]
520    #[allow(clippy::trivially_copy_pass_by_ref)]
521    pub fn text(&self) -> &'a str {
522        self.store.text(self.id).unwrap_or("")
523    }
524}
525
526impl fmt::Debug for TokenView<'_> {
527    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
528        f.debug_struct("TokenView")
529            .field("id", &self.id)
530            .field("token_type", &self.token_type())
531            .field("channel", &self.channel())
532            .field("text", &self.text())
533            .finish()
534    }
535}
536
537impl PartialEq for TokenView<'_> {
538    fn eq(&self, other: &Self) -> bool {
539        self.id == other.id
540            && self.token_type() == other.token_type()
541            && self.channel() == other.channel()
542            && self.start() == other.start()
543            && self.stop() == other.stop()
544            && self.line() == other.line()
545            && self.column() == other.column()
546            && self.text() == other.text()
547            && self.source_name() == other.source_name()
548    }
549}
550
551impl Eq for TokenView<'_> {}
552
553impl Token for TokenView<'_> {
554    fn token_id(&self) -> TokenId {
555        self.id
556    }
557
558    fn token_type(&self) -> i32 {
559        self.store.token_types[self.id.index()]
560    }
561
562    fn channel(&self) -> i32 {
563        self.store.channels[self.id.index()]
564    }
565
566    fn start(&self) -> usize {
567        expand_boundary(self.store.scalar_starts[self.id.index()])
568    }
569
570    fn stop(&self) -> usize {
571        expand_boundary(self.store.scalar_stops[self.id.index()])
572    }
573
574    fn line(&self) -> usize {
575        self.store.lines[self.id.index()] as usize
576    }
577
578    fn column(&self) -> usize {
579        self.store.columns[self.id.index()] as usize
580    }
581
582    fn text(&self) -> Option<&str> {
583        self.store.text(self.id)
584    }
585
586    fn source_name(&self) -> &str {
587        self.store.source_name.as_ref()
588    }
589
590    fn start_byte(&self) -> usize {
591        self.store.byte_starts[self.id.index()] as usize
592    }
593
594    fn stop_byte(&self) -> usize {
595        self.store.byte_stops[self.id.index()] as usize
596    }
597}
598
599impl fmt::Display for TokenView<'_> {
600    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
601        let channel = if self.channel() == DEFAULT_CHANNEL {
602            String::new()
603        } else {
604            format!(",channel={}", self.channel())
605        };
606        write!(
607            f,
608            "[@{},{}:{}='{}',<{}>{},{}:{}]",
609            display_token_index(self),
610            display_token_boundary(self.start()),
611            display_token_boundary(self.stop()),
612            display_text(self.text()),
613            self.token_type(),
614            channel,
615            self.line(),
616            self.column()
617        )
618    }
619}
620
621impl AsRef<str> for TokenView<'_> {
622    fn as_ref(&self) -> &str {
623        self.text()
624    }
625}
626
627const fn expand_boundary(value: u32) -> usize {
628    if value == u32::MAX {
629        usize::MAX
630    } else {
631        value as usize
632    }
633}
634
635/// Mutable append-only view used by a token source.
636#[derive(Debug)]
637pub struct TokenSink<'a> {
638    store: &'a mut TokenStore,
639}
640
641impl<'a> TokenSink<'a> {
642    pub(crate) const fn new(store: &'a mut TokenStore) -> Self {
643        Self { store }
644    }
645
646    pub fn push(&mut self, spec: TokenSpec) -> Result<TokenId, TokenStoreError> {
647        self.store.push(spec)
648    }
649
650    pub fn view(&self, id: TokenId) -> Option<TokenView<'_>> {
651        self.store.view(id)
652    }
653
654    pub(crate) const fn token_count(&self) -> usize {
655        self.store.len()
656    }
657}
658
659/// A diagnostic buffered by a token source while it was producing tokens.
660#[derive(Clone, Debug, Eq, PartialEq)]
661pub struct TokenSourceError {
662    /// One-based input line where the diagnostic starts.
663    pub line: usize,
664    /// Zero-based column within `line` where the diagnostic starts.
665    pub column: usize,
666    /// ANTLR-compatible diagnostic message without the leading line/column.
667    pub message: String,
668}
669
670impl TokenSourceError {
671    /// Creates a token-source diagnostic at the given input position.
672    pub fn new(line: usize, column: usize, message: impl Into<String>) -> Self {
673        Self {
674            line,
675            column,
676            message: message.into(),
677        }
678    }
679}
680
681pub trait TokenSource {
682    fn next_token(&mut self, sink: &mut TokenSink<'_>) -> Result<TokenId, TokenStoreError>;
683    fn line(&self) -> usize;
684    fn column(&self) -> usize;
685    fn source_name(&self) -> &str;
686
687    /// Returns the source buffer once for ownership by the token store.
688    fn source_text(&self) -> Option<Rc<str>> {
689        None
690    }
691
692    /// Returns and clears diagnostics emitted while fetching tokens.
693    fn drain_errors(&mut self) -> Vec<TokenSourceError> {
694        Vec::new()
695    }
696
697    /// Serializes lexer DFA cache state when the token source exposes one.
698    fn lexer_dfa_string(&self) -> String {
699        String::new()
700    }
701}
702
703fn display_token_index(token: &impl Token) -> String {
704    if token.start() == usize::MAX && token.stop() == usize::MAX {
705        "-1".to_owned()
706    } else {
707        token.token_id().index().to_string()
708    }
709}
710
711/// Formats synthetic-token boundaries with ANTLR's `-1` sentinel.
712fn display_token_boundary(value: usize) -> String {
713    if value == usize::MAX {
714        "-1".to_owned()
715    } else {
716        value.to_string()
717    }
718}
719
720/// Escapes token text the way ANTLR's token display format expects.
721fn display_text(text: &str) -> String {
722    let mut out = String::new();
723    for ch in text.chars() {
724        match ch {
725            '\n' => out.push_str("\\n"),
726            '\r' => out.push_str("\\r"),
727            '\t' => out.push_str("\\t"),
728            other => out.push(other),
729        }
730    }
731    out
732}
733
734#[cfg(test)]
735mod tests {
736    use super::*;
737
738    fn one_token(spec: TokenSpec) -> TokenStore {
739        let mut store = TokenStore::new(None, "");
740        store.push(spec).expect("test token should fit");
741        store
742    }
743
744    #[test]
745    fn token_view_display_matches_antlr_shape() {
746        let store = one_token(
747            TokenSpec::explicit(7, "abc")
748                .with_span(2, 4)
749                .with_position(3, 9),
750        );
751        assert_eq!(
752            store.view(TokenId(0)).expect("token").to_string(),
753            "[@0,2:4='abc',<7>,3:9]"
754        );
755    }
756
757    #[test]
758    fn synthetic_token_display_uses_antlr_negative_index() {
759        let store = one_token(
760            TokenSpec::explicit(7, "<missing X>")
761                .with_span(usize::MAX, usize::MAX)
762                .with_byte_span(0, 0)
763                .with_position(3, 9),
764        );
765        assert_eq!(
766            store.view(TokenId(0)).expect("token").to_string(),
767            "[@-1,-1:-1='<missing X>',<7>,3:9]"
768        );
769    }
770
771    #[test]
772    fn source_backed_token_exposes_utf8_byte_span() {
773        let mut store = TokenStore::new(Some(Rc::from("éβz")), "");
774        let id = store
775            .push(TokenSpec {
776                token_type: 1,
777                channel: DEFAULT_CHANNEL,
778                start: 1,
779                stop: 1,
780                start_byte: 2,
781                stop_byte: 4,
782                line: 1,
783                column: 1,
784                text: None,
785                source_backed: true,
786            })
787            .expect("token should fit");
788        let token = TokenView { store: &store, id };
789
790        assert_eq!(token.start(), 1);
791        assert_eq!(token.stop(), 1);
792        assert_eq!(token.byte_span(), 2..4);
793        assert_eq!(token.text(), "β");
794    }
795
796    #[test]
797    fn source_backed_token_rejects_non_utf8_boundaries() {
798        for (start_byte, stop_byte) in [(1, 2), (0, 1)] {
799            let mut store = TokenStore::new(Some(Rc::from("éz")), "");
800            let error = store
801                .push(TokenSpec {
802                    token_type: 1,
803                    channel: DEFAULT_CHANNEL,
804                    start: 0,
805                    stop: 0,
806                    start_byte,
807                    stop_byte,
808                    line: 1,
809                    column: 0,
810                    text: None,
811                    source_backed: true,
812                })
813                .expect_err("spans that split UTF-8 code points must fail");
814
815            assert!(error.to_string().contains("UTF-8 character boundary"));
816            assert!(store.is_empty());
817        }
818    }
819
820    #[test]
821    fn overlarge_offset_is_rejected() {
822        let mut store = TokenStore::new(None, "");
823        let error = store
824            .push(TokenSpec::explicit(1, "x").with_span(MAX_TOKEN_OFFSET + 1, 0))
825            .expect_err("overlarge offsets must fail");
826        assert!(error.to_string().contains("supported limit"));
827    }
828}
antlr4_runtime/token.rs

antlr4_runtime/
token.rs
