antlr4_runtime/
token.rs

1use crate::char_stream::TextInterval;
2use std::fmt;
3use std::ops::Range;
4use std::rc::Rc;
5
6pub const TOKEN_EOF: i32 = -1;
7pub const INVALID_TOKEN_TYPE: i32 = 0;
8pub const DEFAULT_CHANNEL: i32 = 0;
9pub const HIDDEN_CHANNEL: i32 = 1;
10
11/// Largest source or location offset accepted by the compact token store.
12///
13/// `u32::MAX` is reserved for ANTLR's synthetic `-1` source boundary.
14pub const MAX_TOKEN_OFFSET: usize = (u32::MAX - 1) as usize;
15
16#[repr(transparent)]
17#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
18pub struct TokenId(u32);
19
20impl TokenId {
21    #[must_use]
22    pub const fn index(self) -> usize {
23        self.0 as usize
24    }
25}
26
27impl TryFrom<usize> for TokenId {
28    type Error = TokenStoreError;
29
30    fn try_from(value: usize) -> Result<Self, Self::Error> {
31        u32::try_from(value)
32            .map(Self)
33            .map_err(|_| TokenStoreError::overflow("index", value, u32::MAX as usize))
34    }
35}
36
37#[derive(Clone, Copy, Debug, Eq, PartialEq)]
38pub enum TokenChannel {
39    Default,
40    Hidden,
41    Custom(i32),
42}
43
44impl TokenChannel {
45    pub const fn value(self) -> i32 {
46        match self {
47            Self::Default => DEFAULT_CHANNEL,
48            Self::Hidden => HIDDEN_CHANNEL,
49            Self::Custom(channel) => channel,
50        }
51    }
52}
53
54impl From<i32> for TokenChannel {
55    fn from(value: i32) -> Self {
56        match value {
57            DEFAULT_CHANNEL => Self::Default,
58            HIDDEN_CHANNEL => Self::Hidden,
59            other => Self::Custom(other),
60        }
61    }
62}
63
64pub trait Token: fmt::Debug {
65    fn token_id(&self) -> TokenId;
66    fn token_type(&self) -> i32;
67    fn channel(&self) -> i32;
68    /// Zero-based absolute start index measured in Unicode scalar values.
69    fn start(&self) -> usize;
70    /// Zero-based absolute inclusive stop index measured in Unicode scalar
71    /// values.
72    fn stop(&self) -> usize;
73    /// One-based source line where the token starts.
74    fn line(&self) -> usize;
75    /// Zero-based source column where the token starts, measured in Unicode
76    /// scalar values from the start of `line`.
77    fn column(&self) -> usize;
78    /// Returns the token's explicit or source-backed text.
79    ///
80    /// [`TokenView::text`] has the same return type and semantics. Use
81    /// [`TokenView::text_or_empty`] only when missing text should intentionally
82    /// be treated as an empty string.
83    fn text(&self) -> Option<&str>;
84    fn source_name(&self) -> &str;
85
86    fn interval(&self) -> TextInterval {
87        TextInterval::new(self.start(), self.stop())
88    }
89
90    /// Zero-based absolute start offset measured in UTF-8 bytes.
91    fn start_byte(&self) -> usize;
92
93    /// Zero-based exclusive end offset measured in UTF-8 bytes.
94    fn stop_byte(&self) -> usize;
95
96    /// Zero-based UTF-8 byte span for the token text.
97    fn byte_span(&self) -> Range<usize> {
98        self.start_byte()..self.stop_byte()
99    }
100}
101
102impl<T: Token + ?Sized> Token for &T {
103    fn token_id(&self) -> TokenId {
104        (**self).token_id()
105    }
106
107    fn token_type(&self) -> i32 {
108        (**self).token_type()
109    }
110
111    fn channel(&self) -> i32 {
112        (**self).channel()
113    }
114
115    fn start(&self) -> usize {
116        (**self).start()
117    }
118
119    fn stop(&self) -> usize {
120        (**self).stop()
121    }
122
123    fn line(&self) -> usize {
124        (**self).line()
125    }
126
127    fn column(&self) -> usize {
128        (**self).column()
129    }
130
131    fn text(&self) -> Option<&str> {
132        (**self).text()
133    }
134
135    fn source_name(&self) -> &str {
136        (**self).source_name()
137    }
138
139    fn start_byte(&self) -> usize {
140        (**self).start_byte()
141    }
142
143    fn stop_byte(&self) -> usize {
144        (**self).stop_byte()
145    }
146}
147
148/// The fields emitted for one token.
149///
150/// This is transient sink input, not an owned token representation. Source
151/// text and the source name live once in [`TokenStore`].
152#[derive(Clone, Debug)]
153pub struct TokenSpec {
154    pub token_type: i32,
155    pub channel: i32,
156    pub start: usize,
157    pub stop: usize,
158    pub start_byte: usize,
159    pub stop_byte: usize,
160    pub line: usize,
161    pub column: usize,
162    pub text: Option<String>,
163    pub source_backed: bool,
164}
165
166impl TokenSpec {
167    #[must_use]
168    pub fn explicit(token_type: i32, text: impl Into<String>) -> Self {
169        Self {
170            token_type,
171            channel: DEFAULT_CHANNEL,
172            start: 0,
173            stop: 0,
174            start_byte: 0,
175            stop_byte: 1,
176            line: 1,
177            column: 0,
178            text: Some(text.into()),
179            source_backed: false,
180        }
181    }
182
183    #[must_use]
184    pub fn eof(index: usize, byte_offset: usize, line: usize, column: usize) -> Self {
185        Self {
186            token_type: TOKEN_EOF,
187            channel: DEFAULT_CHANNEL,
188            start: index,
189            stop: index.checked_sub(1).unwrap_or(usize::MAX),
190            start_byte: byte_offset,
191            stop_byte: byte_offset,
192            line,
193            column,
194            text: Some("<EOF>".to_owned()),
195            source_backed: false,
196        }
197    }
198
199    #[must_use]
200    pub const fn with_channel(mut self, channel: i32) -> Self {
201        self.channel = channel;
202        self
203    }
204
205    #[must_use]
206    pub const fn with_span(mut self, start: usize, stop: usize) -> Self {
207        self.start = start;
208        self.stop = stop;
209        self.start_byte = start;
210        self.stop_byte = default_stop_byte(start, stop);
211        self
212    }
213
214    #[must_use]
215    pub const fn with_byte_span(mut self, start_byte: usize, stop_byte: usize) -> Self {
216        self.start_byte = start_byte;
217        self.stop_byte = stop_byte;
218        self
219    }
220
221    #[must_use]
222    pub const fn with_position(mut self, line: usize, column: usize) -> Self {
223        self.line = line;
224        self.column = column;
225        self
226    }
227}
228
229#[derive(Clone, Debug, Eq, PartialEq)]
230pub struct TokenStoreError(TokenStoreErrorKind);
231
232impl TokenStoreError {
233    const fn overflow(field: &'static str, value: usize, limit: usize) -> Self {
234        Self(TokenStoreErrorKind::Overflow {
235            field,
236            value,
237            limit,
238        })
239    }
240
241    const fn invalid_source_boundary(offset: usize, source_len: usize) -> Self {
242        Self(TokenStoreErrorKind::InvalidSourceBoundary { offset, source_len })
243    }
244
245    pub(crate) const fn invalid_source_output(
246        expected_id: usize,
247        returned_id: usize,
248        appended: usize,
249    ) -> Self {
250        Self(TokenStoreErrorKind::InvalidSourceOutput {
251            expected_id,
252            returned_id,
253            appended,
254        })
255    }
256}
257
258impl fmt::Display for TokenStoreError {
259    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
260        match self.0 {
261            TokenStoreErrorKind::Overflow {
262                field,
263                value,
264                limit,
265            } => write!(
266                f,
267                "token {field} {value} exceeds the supported limit {limit}"
268            ),
269            TokenStoreErrorKind::InvalidSourceBoundary { offset, source_len } => write!(
270                f,
271                "token source byte offset {offset} is not a UTF-8 character boundary \
272                 for source length {source_len}"
273            ),
274            TokenStoreErrorKind::InvalidSourceOutput {
275                expected_id,
276                returned_id,
277                appended,
278            } => write!(
279                f,
280                "token source must append exactly one token and return ID {expected_id}, \
281                 but appended {appended} and returned ID {returned_id}"
282            ),
283        }
284    }
285}
286
287impl std::error::Error for TokenStoreError {}
288
289#[derive(Clone, Debug, Eq, PartialEq)]
290enum TokenStoreErrorKind {
291    Overflow {
292        field: &'static str,
293        value: usize,
294        limit: usize,
295    },
296    InvalidSourceBoundary {
297        offset: usize,
298        source_len: usize,
299    },
300    InvalidSourceOutput {
301        expected_id: usize,
302        returned_id: usize,
303        appended: usize,
304    },
305}
306
307/// Canonical compact storage for every token associated with one token stream.
308#[derive(Debug)]
309pub struct TokenStore {
310    source: Option<Rc<str>>,
311    source_name: Rc<str>,
312    token_types: Vec<i32>,
313    channels: Vec<i32>,
314    scalar_starts: Vec<u32>,
315    scalar_stops: Vec<u32>,
316    byte_starts: Vec<u32>,
317    byte_stops: Vec<u32>,
318    lines: Vec<u32>,
319    columns: Vec<u32>,
320    source_backed: Vec<bool>,
321    explicit_text: Vec<(TokenId, Rc<str>)>,
322}
323
324impl TokenStore {
325    pub(crate) fn new(source: Option<Rc<str>>, source_name: impl Into<Rc<str>>) -> Self {
326        Self {
327            source,
328            source_name: source_name.into(),
329            token_types: Vec::new(),
330            channels: Vec::new(),
331            scalar_starts: Vec::new(),
332            scalar_stops: Vec::new(),
333            byte_starts: Vec::new(),
334            byte_stops: Vec::new(),
335            lines: Vec::new(),
336            columns: Vec::new(),
337            source_backed: Vec::new(),
338            explicit_text: Vec::new(),
339        }
340    }
341
342    #[must_use]
343    pub const fn len(&self) -> usize {
344        self.token_types.len()
345    }
346
347    #[must_use]
348    pub const fn is_empty(&self) -> bool {
349        self.token_types.is_empty()
350    }
351
352    /// Iterates borrowing views of every stored token in [`TokenId`] order.
353    pub fn iter(&self) -> TokenIter<'_> {
354        self.iter_prefix(self.len())
355    }
356
357    pub(crate) fn iter_prefix(&self, stop: usize) -> TokenIter<'_> {
358        assert!(
359            stop <= self.len(),
360            "token iterator prefix exceeds store length"
361        );
362        TokenIter {
363            store: self,
364            next: 0,
365            stop,
366        }
367    }
368
369    pub(crate) fn push(&mut self, spec: TokenSpec) -> Result<TokenId, TokenStoreError> {
370        let raw_id = u32::try_from(self.len())
371            .map_err(|_| TokenStoreError::overflow("count", self.len(), u32::MAX as usize))?;
372        let id = TokenId(raw_id);
373        let scalar_start = compact_boundary("start offset", spec.start)?;
374        let scalar_stop = compact_boundary("stop offset", spec.stop)?;
375        let byte_start = compact_offset("start byte", spec.start_byte)?;
376        let byte_stop = compact_offset("stop byte", spec.stop_byte)?;
377        let line = compact_offset("line", spec.line)?;
378        let column = compact_offset("column", spec.column)?;
379
380        if spec.source_backed {
381            let Some(source) = self.source.as_ref() else {
382                return Err(TokenStoreError::overflow("source text", 1, 0));
383            };
384            if spec.start_byte > spec.stop_byte || spec.stop_byte > source.len() {
385                return Err(TokenStoreError::overflow(
386                    "source byte span",
387                    spec.stop_byte,
388                    source.len(),
389                ));
390            }
391            if !source.is_char_boundary(spec.start_byte) {
392                return Err(TokenStoreError::invalid_source_boundary(
393                    spec.start_byte,
394                    source.len(),
395                ));
396            }
397            if !source.is_char_boundary(spec.stop_byte) {
398                return Err(TokenStoreError::invalid_source_boundary(
399                    spec.stop_byte,
400                    source.len(),
401                ));
402            }
403        }
404
405        self.token_types.push(spec.token_type);
406        self.channels.push(spec.channel);
407        self.scalar_starts.push(scalar_start);
408        self.scalar_stops.push(scalar_stop);
409        self.byte_starts.push(byte_start);
410        self.byte_stops.push(byte_stop);
411        self.lines.push(line);
412        self.columns.push(column);
413        self.source_backed.push(spec.source_backed);
414        if let Some(text) = spec.text {
415            self.explicit_text.push((id, Rc::from(text)));
416        }
417        Ok(id)
418    }
419
420    const fn contains(&self, id: TokenId) -> bool {
421        id.index() < self.len()
422    }
423
424    /// Returns a borrowing view of one token record.
425    #[must_use]
426    pub fn view(&self, id: TokenId) -> Option<TokenView<'_>> {
427        self.contains(id).then_some(TokenView { store: self, id })
428    }
429
430    /// Returns the token type for `id`.
431    #[must_use]
432    pub fn token_type(&self, id: TokenId) -> Option<i32> {
433        self.token_types.get(id.index()).copied()
434    }
435
436    /// Returns the token channel for `id`.
437    #[must_use]
438    pub fn channel(&self, id: TokenId) -> Option<i32> {
439        self.channels.get(id.index()).copied()
440    }
441
442    /// Returns the token's zero-based scalar start offset.
443    #[must_use]
444    pub fn start(&self, id: TokenId) -> Option<usize> {
445        self.scalar_starts
446            .get(id.index())
447            .copied()
448            .map(expand_boundary)
449    }
450
451    /// Returns the token's zero-based inclusive scalar stop offset.
452    #[must_use]
453    pub fn stop(&self, id: TokenId) -> Option<usize> {
454        self.scalar_stops
455            .get(id.index())
456            .copied()
457            .map(expand_boundary)
458    }
459
460    /// Returns the token's one-based source line.
461    #[must_use]
462    pub fn line(&self, id: TokenId) -> Option<usize> {
463        self.lines.get(id.index()).map(|line| *line as usize)
464    }
465
466    /// Returns the token's zero-based source column.
467    #[must_use]
468    pub fn column(&self, id: TokenId) -> Option<usize> {
469        self.columns.get(id.index()).map(|column| *column as usize)
470    }
471
472    /// Returns the token's zero-based UTF-8 byte start offset.
473    #[must_use]
474    pub fn start_byte(&self, id: TokenId) -> Option<usize> {
475        self.byte_starts
476            .get(id.index())
477            .map(|offset| *offset as usize)
478    }
479
480    /// Returns the token's zero-based exclusive UTF-8 byte stop offset.
481    #[must_use]
482    pub fn stop_byte(&self, id: TokenId) -> Option<usize> {
483        self.byte_stops
484            .get(id.index())
485            .map(|offset| *offset as usize)
486    }
487
488    fn explicit_text(&self, id: TokenId) -> Option<&str> {
489        self.explicit_text
490            .binary_search_by_key(&id, |(token_id, _)| *token_id)
491            .ok()
492            .map(|index| self.explicit_text[index].1.as_ref())
493    }
494
495    /// Returns explicit or source-backed text for `id`.
496    #[must_use]
497    pub fn text(&self, id: TokenId) -> Option<&str> {
498        if let Some(text) = self.explicit_text(id) {
499            return Some(text);
500        }
501        if !self.source_backed.get(id.index()).copied().unwrap_or(false) {
502            return None;
503        }
504        let source = self.source.as_deref()?;
505        let start = self.byte_starts[id.index()] as usize;
506        let stop = self.byte_stops[id.index()] as usize;
507        source.get(start..stop)
508    }
509}
510
511impl<'a> IntoIterator for &'a TokenStore {
512    type Item = TokenView<'a>;
513    type IntoIter = TokenIter<'a>;
514
515    fn into_iter(self) -> Self::IntoIter {
516        self.iter()
517    }
518}
519
520/// Iterator over borrowing views of a token store.
521#[derive(Debug)]
522pub struct TokenIter<'a> {
523    store: &'a TokenStore,
524    next: usize,
525    stop: usize,
526}
527
528impl<'a> Iterator for TokenIter<'a> {
529    type Item = TokenView<'a>;
530
531    fn next(&mut self) -> Option<Self::Item> {
532        if self.next >= self.stop {
533            return None;
534        }
535        let id = TokenId::try_from(self.next).ok()?;
536        self.next += 1;
537        self.store.view(id)
538    }
539
540    fn size_hint(&self) -> (usize, Option<usize>) {
541        let remaining = self.stop - self.next;
542        (remaining, Some(remaining))
543    }
544}
545
546impl DoubleEndedIterator for TokenIter<'_> {
547    fn next_back(&mut self) -> Option<Self::Item> {
548        if self.next >= self.stop {
549            return None;
550        }
551        self.stop -= 1;
552        let id = TokenId::try_from(self.stop).ok()?;
553        self.store.view(id)
554    }
555}
556
557impl ExactSizeIterator for TokenIter<'_> {}
558
559const fn default_stop_byte(start: usize, stop: usize) -> usize {
560    match stop.checked_add(1) {
561        Some(end) if end >= start => end,
562        Some(_) | None => start,
563    }
564}
565
566const fn compact_boundary(field: &'static str, value: usize) -> Result<u32, TokenStoreError> {
567    if value == usize::MAX {
568        return Ok(u32::MAX);
569    }
570    compact_offset(field, value)
571}
572
573const fn compact_offset(field: &'static str, value: usize) -> Result<u32, TokenStoreError> {
574    if value > MAX_TOKEN_OFFSET {
575        return Err(TokenStoreError::overflow(field, value, MAX_TOKEN_OFFSET));
576    }
577    Ok(value as u32)
578}
579
580/// Borrowing public view of one canonical token-store record.
581#[derive(Clone, Copy)]
582pub struct TokenView<'a> {
583    store: &'a TokenStore,
584    id: TokenId,
585}
586
587impl<'a> TokenView<'a> {
588    /// Returns the token's explicit or source-backed text.
589    #[must_use]
590    #[allow(clippy::trivially_copy_pass_by_ref)]
591    pub fn text(&self) -> Option<&'a str> {
592        self.store.text(self.id)
593    }
594
595    /// Returns the token's text, or an empty string when no text exists.
596    #[must_use]
597    #[allow(clippy::trivially_copy_pass_by_ref)]
598    pub fn text_or_empty(&self) -> &'a str {
599        self.text().unwrap_or("")
600    }
601}
602
603impl fmt::Debug for TokenView<'_> {
604    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
605        f.debug_struct("TokenView")
606            .field("id", &self.id)
607            .field("token_type", &self.token_type())
608            .field("channel", &self.channel())
609            .field("text", &self.text())
610            .finish()
611    }
612}
613
614impl PartialEq for TokenView<'_> {
615    fn eq(&self, other: &Self) -> bool {
616        self.id == other.id
617            && self.token_type() == other.token_type()
618            && self.channel() == other.channel()
619            && self.start() == other.start()
620            && self.stop() == other.stop()
621            && self.line() == other.line()
622            && self.column() == other.column()
623            && self.text() == other.text()
624            && self.source_name() == other.source_name()
625    }
626}
627
628impl Eq for TokenView<'_> {}
629
630impl Token for TokenView<'_> {
631    fn token_id(&self) -> TokenId {
632        self.id
633    }
634
635    fn token_type(&self) -> i32 {
636        self.store.token_types[self.id.index()]
637    }
638
639    fn channel(&self) -> i32 {
640        self.store.channels[self.id.index()]
641    }
642
643    fn start(&self) -> usize {
644        expand_boundary(self.store.scalar_starts[self.id.index()])
645    }
646
647    fn stop(&self) -> usize {
648        expand_boundary(self.store.scalar_stops[self.id.index()])
649    }
650
651    fn line(&self) -> usize {
652        self.store.lines[self.id.index()] as usize
653    }
654
655    fn column(&self) -> usize {
656        self.store.columns[self.id.index()] as usize
657    }
658
659    fn text(&self) -> Option<&str> {
660        self.store.text(self.id)
661    }
662
663    fn source_name(&self) -> &str {
664        self.store.source_name.as_ref()
665    }
666
667    fn start_byte(&self) -> usize {
668        self.store.byte_starts[self.id.index()] as usize
669    }
670
671    fn stop_byte(&self) -> usize {
672        self.store.byte_stops[self.id.index()] as usize
673    }
674}
675
676impl fmt::Display for TokenView<'_> {
677    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
678        let channel = if self.channel() == DEFAULT_CHANNEL {
679            String::new()
680        } else {
681            format!(",channel={}", self.channel())
682        };
683        write!(
684            f,
685            "[@{},{}:{}='{}',<{}>{},{}:{}]",
686            display_token_index(self),
687            display_token_boundary(self.start()),
688            display_token_boundary(self.stop()),
689            display_text(self.text_or_empty()),
690            self.token_type(),
691            channel,
692            self.line(),
693            self.column()
694        )
695    }
696}
697
698impl AsRef<str> for TokenView<'_> {
699    fn as_ref(&self) -> &str {
700        self.text_or_empty()
701    }
702}
703
704const fn expand_boundary(value: u32) -> usize {
705    if value == u32::MAX {
706        usize::MAX
707    } else {
708        value as usize
709    }
710}
711
712/// Mutable append-only view used by a token source.
713#[derive(Debug)]
714pub struct TokenSink<'a> {
715    store: &'a mut TokenStore,
716}
717
718impl<'a> TokenSink<'a> {
719    pub(crate) const fn new(store: &'a mut TokenStore) -> Self {
720        Self { store }
721    }
722
723    pub fn push(&mut self, spec: TokenSpec) -> Result<TokenId, TokenStoreError> {
724        self.store.push(spec)
725    }
726
727    pub fn view(&self, id: TokenId) -> Option<TokenView<'_>> {
728        self.store.view(id)
729    }
730
731    pub(crate) const fn token_count(&self) -> usize {
732        self.store.len()
733    }
734}
735
736/// A diagnostic buffered by a token source while it was producing tokens.
737#[derive(Clone, Debug, Eq, PartialEq)]
738pub struct TokenSourceError {
739    /// One-based input line where the diagnostic starts.
740    pub line: usize,
741    /// Zero-based column within `line` where the diagnostic starts.
742    pub column: usize,
743    /// ANTLR-compatible diagnostic message without the leading line/column.
744    pub message: String,
745}
746
747impl TokenSourceError {
748    /// Creates a token-source diagnostic at the given input position.
749    pub fn new(line: usize, column: usize, message: impl Into<String>) -> Self {
750        Self {
751            line,
752            column,
753            message: message.into(),
754        }
755    }
756}
757
758pub trait TokenSource {
759    fn next_token(&mut self, sink: &mut TokenSink<'_>) -> Result<TokenId, TokenStoreError>;
760    fn line(&self) -> usize;
761    fn column(&self) -> usize;
762    fn source_name(&self) -> &str;
763
764    /// Returns the source buffer once for ownership by the token store.
765    fn source_text(&self) -> Option<Rc<str>> {
766        None
767    }
768
769    /// Returns and clears diagnostics emitted while fetching tokens.
770    fn drain_errors(&mut self) -> Vec<TokenSourceError> {
771        Vec::new()
772    }
773
774    /// Reports a buffered diagnostic through source-owned listeners.
775    ///
776    /// Returns `true` when the source owns diagnostic reporting. The parser
777    /// uses its own listeners as a fallback for token sources that return
778    /// `false`.
779    fn report_error(&self, _error: &TokenSourceError) -> bool {
780        false
781    }
782
783    /// Serializes lexer DFA cache state when the token source exposes one.
784    fn lexer_dfa_string(&self) -> String {
785        String::new()
786    }
787}
788
789fn display_token_index(token: &impl Token) -> String {
790    if token.start() == usize::MAX && token.stop() == usize::MAX {
791        "-1".to_owned()
792    } else {
793        token.token_id().index().to_string()
794    }
795}
796
797/// Formats synthetic-token boundaries with ANTLR's `-1` sentinel.
798fn display_token_boundary(value: usize) -> String {
799    if value == usize::MAX {
800        "-1".to_owned()
801    } else {
802        value.to_string()
803    }
804}
805
806/// Escapes token text the way ANTLR's token display format expects.
807fn display_text(text: &str) -> String {
808    let mut out = String::new();
809    for ch in text.chars() {
810        match ch {
811            '\n' => out.push_str("\\n"),
812            '\r' => out.push_str("\\r"),
813            '\t' => out.push_str("\\t"),
814            other => out.push(other),
815        }
816    }
817    out
818}
819
820#[cfg(test)]
821mod tests {
822    use super::*;
823
824    fn one_token(spec: TokenSpec) -> TokenStore {
825        let mut store = TokenStore::new(None, "");
826        store.push(spec).expect("test token should fit");
827        store
828    }
829
830    #[test]
831    fn token_view_display_matches_antlr_shape() {
832        let store = one_token(
833            TokenSpec::explicit(7, "abc")
834                .with_span(2, 4)
835                .with_position(3, 9),
836        );
837        assert_eq!(
838            store.view(TokenId(0)).expect("token").to_string(),
839            "[@0,2:4='abc',<7>,3:9]"
840        );
841    }
842
843    #[test]
844    fn synthetic_token_display_uses_antlr_negative_index() {
845        let store = one_token(
846            TokenSpec::explicit(7, "<missing X>")
847                .with_span(usize::MAX, usize::MAX)
848                .with_byte_span(0, 0)
849                .with_position(3, 9),
850        );
851        assert_eq!(
852            store.view(TokenId(0)).expect("token").to_string(),
853            "[@-1,-1:-1='<missing X>',<7>,3:9]"
854        );
855    }
856
857    #[test]
858    fn source_backed_token_exposes_utf8_byte_span() {
859        let mut store = TokenStore::new(Some(Rc::from("éβz")), "");
860        let id = store
861            .push(TokenSpec {
862                token_type: 1,
863                channel: DEFAULT_CHANNEL,
864                start: 1,
865                stop: 1,
866                start_byte: 2,
867                stop_byte: 4,
868                line: 1,
869                column: 1,
870                text: None,
871                source_backed: true,
872            })
873            .expect("token should fit");
874        let token = TokenView { store: &store, id };
875
876        assert_eq!(token.start(), 1);
877        assert_eq!(token.stop(), 1);
878        assert_eq!(token.byte_span(), 2..4);
879        assert_eq!(token.text(), Some("β"));
880    }
881
882    #[test]
883    fn source_backed_token_rejects_non_utf8_boundaries() {
884        for (start_byte, stop_byte) in [(1, 2), (0, 1)] {
885            let mut store = TokenStore::new(Some(Rc::from("éz")), "");
886            let error = store
887                .push(TokenSpec {
888                    token_type: 1,
889                    channel: DEFAULT_CHANNEL,
890                    start: 0,
891                    stop: 0,
892                    start_byte,
893                    stop_byte,
894                    line: 1,
895                    column: 0,
896                    text: None,
897                    source_backed: true,
898                })
899                .expect_err("spans that split UTF-8 code points must fail");
900
901            assert!(error.to_string().contains("UTF-8 character boundary"));
902            assert!(store.is_empty());
903        }
904    }
905
906    #[test]
907    fn overlarge_offset_is_rejected() {
908        let mut store = TokenStore::new(None, "");
909        let error = store
910            .push(TokenSpec::explicit(1, "x").with_span(MAX_TOKEN_OFFSET + 1, 0))
911            .expect_err("overlarge offsets must fail");
912        assert!(error.to_string().contains("supported limit"));
913    }
914
915    #[test]
916    fn token_store_iterates_all_records_in_id_order() {
917        let mut store = TokenStore::new(None, "iterator-test");
918        for spec in [
919            TokenSpec::explicit(1, "a"),
920            TokenSpec::explicit(2, " comment").with_channel(HIDDEN_CHANNEL),
921            TokenSpec::eof(9, 9, 1, 9),
922        ] {
923            store.push(spec).expect("test token should fit");
924        }
925
926        let mut iter = store.iter();
927        assert_eq!(iter.len(), 3);
928        assert_eq!(iter.next().and_then(|token| token.text()), Some("a"));
929        assert_eq!(
930            iter.next_back().map(|token| token.token_type()),
931            Some(TOKEN_EOF)
932        );
933        assert_eq!(iter.len(), 1);
934
935        assert_eq!(
936            (&store)
937                .into_iter()
938                .map(|token| (token.token_id().index(), token.channel()))
939                .collect::<Vec<_>>(),
940            [
941                (0, DEFAULT_CHANNEL),
942                (1, HIDDEN_CHANNEL),
943                (2, DEFAULT_CHANNEL)
944            ]
945        );
946    }
947
948    #[test]
949    fn token_view_text_matches_token_trait_semantics() {
950        fn generic_text(token: &impl Token) -> Option<&str> {
951            token.text()
952        }
953
954        let store = one_token(TokenSpec {
955            token_type: 1,
956            channel: DEFAULT_CHANNEL,
957            start: 0,
958            stop: 0,
959            start_byte: 0,
960            stop_byte: 0,
961            line: 1,
962            column: 0,
963            text: None,
964            source_backed: false,
965        });
966        let token = store.view(TokenId(0)).expect("token");
967
968        assert_eq!(token.text(), None);
969        assert_eq!(generic_text(&token), None);
970        assert_eq!(token.text_or_empty(), "");
971    }
972}
antlr4_runtime/token.rs

antlr4_runtime/
token.rs
