antlr_rust_sleagon/
token_stream.rs

1//! `IntStream` that produces tokens for Parser
2use std::borrow::Borrow;
3use std::cmp::min;
4use std::marker::PhantomData;
5
6use crate::char_stream::InputData;
7use crate::int_stream::{IntStream, IterWrapper};
8use crate::token::{OwningToken, Token, TOKEN_EOF, TOKEN_INVALID_TYPE};
9use crate::token_factory::TokenFactory;
10use crate::token_source::TokenSource;
11use better_any::{Tid, TidAble};
12use std::fmt::{Debug, Formatter};
13
14/// An `IntSteam` of `Token`s
15///
16/// Used as an input for `Parser`s
17/// If there is an existing source of tokens, you should implement
18/// `TokenSource`, not `TokenStream`
19pub trait TokenStream<'input>: IntStream {
20    /// Token factory that created tokens in this stream
21    type TF: TokenFactory<'input> + 'input;
22
23    /// Lookahead for tokens, same as `IntSteam::la` but return reference to full token
24    fn lt(&mut self, k: isize) -> Option<&<Self::TF as TokenFactory<'input>>::Tok>;
25    /// Returns reference to token at `index`
26    fn get(&self, index: isize) -> &<Self::TF as TokenFactory<'input>>::Tok;
27
28    /// Token source that produced data for tokens for this stream
29    fn get_token_source(&self) -> &dyn TokenSource<'input, TF = Self::TF>;
30    //    fn set_token_source(&self,source: Box<TokenSource>);
31    /// Get combined text of all tokens in this stream
32    fn get_all_text(&self) -> String {
33        self.get_text_from_interval(0, self.size() - 1)
34    }
35    /// Get combined text of tokens in start..=stop interval
36    fn get_text_from_interval(&self, start: isize, stop: isize) -> String;
37    //    fn get_text_from_rule_context(&self,context: RuleContext) -> String;
38    /// Get combined text of tokens in between `a` and `b`
39    fn get_text_from_tokens<T: Token + ?Sized>(&self, a: &T, b: &T) -> String
40    where
41        Self: Sized,
42    {
43        self.get_text_from_interval(a.get_token_index(), b.get_token_index())
44    }
45}
46
47/// Iterator over tokens in `T`
48#[derive(Debug)]
49pub struct TokenIter<'a, 'input: 'a, T: TokenStream<'input>>(
50    &'a mut T,
51    bool,
52    PhantomData<fn() -> &'input str>,
53);
54
55impl<'a, 'input: 'a, T: TokenStream<'input>> Iterator for TokenIter<'a, 'input, T> {
56    type Item = OwningToken;
57
58    fn next(&mut self) -> Option<Self::Item> {
59        if self.1 {
60            return None;
61        }
62        let result = self.0.lt(1).unwrap().borrow().to_owned();
63        if result.get_token_type() == TOKEN_EOF {
64            self.1 = true;
65        } else {
66            self.0.consume();
67        }
68        Some(result)
69    }
70}
71
72/// Token stream that keeps all data in internal Vec
73pub struct UnbufferedTokenStream<'input, T: TokenSource<'input>> {
74    token_source: T,
75    pub(crate) tokens: Vec<<T::TF as TokenFactory<'input>>::Tok>,
76    //todo prev token for lt(-1)
77    pub(crate) current_token_index: isize,
78    markers_count: isize,
79    pub(crate) p: isize,
80}
81better_any::tid! { impl<'input,T> TidAble<'input> for UnbufferedTokenStream<'input, T> where T: TokenSource<'input>}
82
83impl<'input, T: TokenSource<'input>> Debug for UnbufferedTokenStream<'input, T> {
84    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
85        f.debug_struct("UnbufferedTokenStream")
86            .field("tokens", &self.tokens)
87            .field("current_token_index", &self.current_token_index)
88            .field("markers_count", &self.markers_count)
89            .field("p(buffer index)", &self.p)
90            .finish()
91    }
92}
93
94impl<'input, T: TokenSource<'input>> UnbufferedTokenStream<'input, T> {
95    /// Creates iterator over this token stream
96    pub fn iter(&mut self) -> IterWrapper<'_, Self> {
97        IterWrapper(self)
98    }
99
100    /// Creates iterator over tokens in this token stream
101    pub fn token_iter(&mut self) -> TokenIter<'_, 'input, Self> {
102        TokenIter(self, false, PhantomData)
103    }
104
105    /// Creates token stream that keeps all tokens inside
106    pub fn new_buffered(source: T) -> Self {
107        let mut a = UnbufferedTokenStream::new_unbuffered(source);
108        a.mark();
109        a
110    }
111
112    /// Creates token stream that keeps only tokens required by `mark`
113    pub fn new_unbuffered(source: T) -> Self {
114        UnbufferedTokenStream {
115            token_source: source,
116            tokens: vec![],
117            current_token_index: 0,
118            markers_count: 0,
119            p: 0,
120        }
121    }
122
123    fn sync(&mut self, want: isize) {
124        let need = (self.p + want - 1) - self.tokens.len() as isize + 1;
125        if need > 0 {
126            self.fill(need);
127        }
128    }
129
130    fn get_buffer_start_index(&self) -> isize {
131        self.current_token_index - self.p
132    }
133
134    pub(crate) fn fill(&mut self, need: isize) -> isize {
135        for i in 0..need {
136            if self.tokens.len() > 0
137                && self.tokens.last().unwrap().borrow().get_token_type() == TOKEN_EOF
138            {
139                return i;
140            }
141            let token = self.token_source.next_token();
142            token
143                .borrow()
144                .set_token_index(self.get_buffer_start_index() + self.tokens.len() as isize);
145            self.tokens.push(token);
146        }
147
148        need
149    }
150}
151
152impl<'input, T: TokenSource<'input>> TokenStream<'input> for UnbufferedTokenStream<'input, T> {
153    type TF = T::TF;
154
155    #[inline]
156    fn lt(&mut self, i: isize) -> Option<&<Self::TF as TokenFactory<'input>>::Tok> {
157        if i == -1 {
158            return self.tokens.get(self.p as usize - 1);
159        }
160
161        self.sync(i);
162
163        self.tokens.get((self.p + i - 1) as usize)
164    }
165
166    #[inline]
167    fn get(&self, index: isize) -> &<Self::TF as TokenFactory<'input>>::Tok {
168        &self.tokens[(index - self.get_buffer_start_index()) as usize]
169    }
170
171    fn get_token_source(&self) -> &dyn TokenSource<'input, TF = Self::TF> {
172        &self.token_source
173    }
174
175    fn get_text_from_interval(&self, start: isize, stop: isize) -> String {
176        //        println!("get_text_from_interval {}..{}",start,stop);
177        //        println!("all tokens {:?}",self.tokens.iter().map(|x|x.as_ref().to_owned()).collect::<Vec<OwningToken>>());
178
179        let buffer_start_index = self.get_buffer_start_index();
180        let buffer_stop_index = buffer_start_index + self.tokens.len() as isize - 1;
181        if start < buffer_start_index || stop > buffer_stop_index {
182            panic!(
183                "interval {}..={} not in token buffer window: {}..{}",
184                start, stop, buffer_start_index, buffer_stop_index
185            );
186        }
187
188        let a = start - buffer_start_index;
189        let b = stop - buffer_start_index;
190
191        let mut buf = String::new();
192        for i in a..(b + 1) {
193            let t = self.tokens[i as usize].borrow();
194            if t.get_token_type() == TOKEN_EOF {
195                break;
196            }
197            buf.extend(t.get_text().to_display().chars());
198        }
199
200        return buf;
201    }
202}
203
204impl<'input, T: TokenSource<'input>> IntStream for UnbufferedTokenStream<'input, T> {
205    #[inline]
206    fn consume(&mut self) {
207        if self.la(1) == TOKEN_EOF {
208            panic!("cannot consume EOF");
209        }
210
211        if self.p == self.tokens.len() as isize && self.markers_count == 0 {
212            self.tokens.clear();
213            self.p = -1;
214        }
215
216        self.p += 1;
217        self.current_token_index += 1;
218
219        self.sync(1);
220        // Ok(())
221    }
222
223    #[inline]
224    fn la(&mut self, i: isize) -> isize {
225        self.lt(i)
226            .map(|t| t.borrow().get_token_type())
227            .unwrap_or(TOKEN_INVALID_TYPE)
228    }
229
230    #[inline]
231    fn mark(&mut self) -> isize {
232        self.markers_count += 1;
233        -self.markers_count
234    }
235
236    #[inline]
237    fn release(&mut self, marker: isize) {
238        assert_eq!(marker, -self.markers_count);
239
240        self.markers_count -= 1;
241        if self.markers_count == 0 {
242            if self.p > 0 {
243                self.tokens.drain(0..self.p as usize);
244                //todo drain assembly is almost 2x longer than
245                // unsafe manual copy but need to bench before using unsafe
246                //let new_len = self.tokens.len() - self.p as usize;
247                // unsafe {
248                //     // drop first p elements
249                //     for i in 0..(self.p as usize) {
250                //         drop_in_place(&mut self.tokens[i]);
251                //     }
252                //     // move len-p elements to beginning
253                //     std::intrinsics::copy(
254                //         &self.tokens[self.p as usize],
255                //         &mut self.tokens[0],
256                //         new_len,
257                //     );
258                //     self.tokens.set_len(new_len);
259                // }
260
261                self.p = 0;
262            }
263        }
264    }
265
266    #[inline(always)]
267    fn index(&self) -> isize {
268        self.current_token_index
269    }
270
271    #[inline]
272    fn seek(&mut self, mut index: isize) {
273        if self.current_token_index == index {
274            return;
275        }
276        if index > self.current_token_index {
277            self.sync(index - self.current_token_index);
278            index = min(index, self.get_buffer_start_index() + self.size() + 1);
279        }
280        let i = index - self.get_buffer_start_index();
281        if i < 0 || i >= self.tokens.len() as isize {
282            panic!()
283        }
284
285        self.p = i;
286        self.current_token_index = index;
287    }
288
289    #[inline(always)]
290    fn size(&self) -> isize {
291        self.tokens.len() as isize
292    }
293
294    fn get_source_name(&self) -> String {
295        self.token_source.get_source_name()
296    }
297}
antlr_rust_sleagon/token_stream.rs

antlr_rust_sleagon/
token_stream.rs
