antlr4_runtime/
char_stream.rs

1use crate::int_stream::{EOF, IntStream, UNKNOWN_SOURCE_NAME};
2use std::rc::Rc;
3
4#[derive(Clone, Copy, Debug, Eq, PartialEq)]
5pub struct TextInterval {
6    pub start: usize,
7    pub stop: usize,
8}
9
10impl TextInterval {
11    pub const fn new(start: usize, stop: usize) -> Self {
12        Self { start, stop }
13    }
14
15    pub const fn empty() -> Self {
16        Self { start: 1, stop: 0 }
17    }
18
19    pub const fn is_empty(self) -> bool {
20        self.start > self.stop
21    }
22}
23
24pub trait CharStream: IntStream {
25    fn text(&self, interval: TextInterval) -> String;
26
27    fn text_source_interval(&self, _interval: TextInterval) -> Option<(Rc<str>, usize, usize)> {
28        None
29    }
30}
31
32#[derive(Clone, Debug)]
33pub struct InputStream {
34    source: Rc<str>,
35    data: InputData,
36    cursor: usize,
37    source_name: String,
38}
39
40#[derive(Clone, Debug)]
41enum InputData {
42    Ascii,
43    Unicode {
44        chars: Vec<char>,
45        byte_offsets: Vec<usize>,
46    },
47}
48
49impl InputData {
50    fn new(input: &str) -> Self {
51        if input.is_ascii() {
52            Self::Ascii
53        } else {
54            Self::Unicode {
55                chars: input.chars().collect(),
56                byte_offsets: input.char_indices().map(|(index, _)| index).collect(),
57            }
58        }
59    }
60
61    const fn len(&self, source: &str) -> usize {
62        match self {
63            Self::Ascii => source.len(),
64            Self::Unicode { chars, .. } => chars.len(),
65        }
66    }
67
68    fn get(&self, source: &str, index: usize) -> Option<char> {
69        match self {
70            Self::Ascii => source.as_bytes().get(index).map(|byte| char::from(*byte)),
71            Self::Unicode { chars, .. } => chars.get(index).copied(),
72        }
73    }
74
75    fn byte_bounds(&self, source: &str, start: usize, stop: usize) -> Option<(usize, usize)> {
76        match self {
77            Self::Ascii => Some((start, stop + 1)),
78            Self::Unicode { byte_offsets, .. } => {
79                let start_byte = *byte_offsets.get(start)?;
80                let stop_byte = byte_offsets.get(stop + 1).copied().unwrap_or(source.len());
81                Some((start_byte, stop_byte))
82            }
83        }
84    }
85}
86
87impl InputStream {
88    /// Creates a character stream from UTF-8 text using ANTLR's unknown source
89    /// name placeholder.
90    pub fn new(input: impl AsRef<str>) -> Self {
91        Self::with_source_name(input, UNKNOWN_SOURCE_NAME)
92    }
93
94    /// Creates a character stream with an explicit source name for tokens and
95    /// diagnostics.
96    pub fn with_source_name(input: impl AsRef<str>, source_name: impl Into<String>) -> Self {
97        let input = input.as_ref();
98        Self {
99            source: Rc::from(input),
100            data: InputData::new(input),
101            cursor: 0,
102            source_name: source_name.into(),
103        }
104    }
105
106    /// Returns true when the cursor has reached or passed the end of input.
107    pub fn is_eof(&self) -> bool {
108        self.cursor >= self.data.len(&self.source)
109    }
110}
111
112impl IntStream for InputStream {
113    fn consume(&mut self) {
114        if !self.is_eof() {
115            self.cursor += 1;
116        }
117    }
118
119    fn la(&mut self, offset: isize) -> i32 {
120        if offset == 0 {
121            return 0;
122        }
123
124        let absolute = if offset > 0 {
125            self.cursor.checked_add((offset - 1).cast_unsigned())
126        } else {
127            offset
128                .checked_neg()
129                .and_then(|distance| usize::try_from(distance).ok())
130                .and_then(|distance| self.cursor.checked_sub(distance))
131        };
132
133        absolute
134            .and_then(|index| self.data.get(&self.source, index))
135            .map_or(EOF, |ch| ch as i32)
136    }
137
138    fn index(&self) -> usize {
139        self.cursor
140    }
141
142    fn seek(&mut self, index: usize) {
143        self.cursor = index.min(self.data.len(&self.source));
144    }
145
146    fn size(&self) -> usize {
147        self.data.len(&self.source)
148    }
149
150    fn source_name(&self) -> &str {
151        &self.source_name
152    }
153}
154
155impl CharStream for InputStream {
156    /// Returns text for an inclusive interval of Unicode scalar indices.
157    fn text(&self, interval: TextInterval) -> String {
158        if let Some((source, start, stop)) = self.text_source_interval(interval) {
159            return source[start..stop].to_owned();
160        }
161        String::new()
162    }
163
164    fn text_source_interval(&self, interval: TextInterval) -> Option<(Rc<str>, usize, usize)> {
165        let len = self.data.len(&self.source);
166        if interval.is_empty() || len == 0 {
167            return None;
168        }
169
170        let start = interval.start.min(len);
171        let stop = interval.stop.min(len.saturating_sub(1));
172        if start > stop {
173            return None;
174        }
175
176        let (start_byte, stop_byte) = self.data.byte_bounds(&self.source, start, stop)?;
177        Some((Rc::clone(&self.source), start_byte, stop_byte))
178    }
179}
180
181#[cfg(test)]
182mod tests {
183    use super::*;
184
185    #[test]
186    fn lookahead_and_text_are_codepoint_indexed() {
187        let mut input = InputStream::with_source_name("aβ\n", "sample");
188        assert_eq!(input.source_name(), "sample");
189        assert_eq!(input.size(), 3);
190        assert_eq!(input.la(1), 'a' as i32);
191        assert_eq!(input.la(2), 'β' as i32);
192        assert_eq!(input.text(TextInterval::new(0, 1)), "aβ");
193        input.consume();
194        assert_eq!(input.index(), 1);
195        assert_eq!(input.la(-1), 'a' as i32);
196        assert_eq!(input.la(isize::MIN), EOF);
197        input.seek(99);
198        assert_eq!(input.la(1), EOF);
199    }
200}
antlr4_runtime/char_stream.rs

antlr4_runtime/
char_stream.rs
