antlr_rust_sleagon/
common_token_stream.rs

1//! Channel based `TokenStream`
2use std::borrow::Borrow;
3
4use crate::int_stream::{IntStream, IterWrapper, EOF};
5use crate::token::{Token, TOKEN_DEFAULT_CHANNEL, TOKEN_INVALID_TYPE};
6use crate::token_factory::TokenFactory;
7use crate::token_source::TokenSource;
8use crate::token_stream::{TokenStream, UnbufferedTokenStream};
9use better_any::{Tid, TidAble};
10
11/// Default token stream that skips token that not correspond to current channel.
12#[derive(Debug)]
13pub struct CommonTokenStream<'input, T: TokenSource<'input>> {
14    base: UnbufferedTokenStream<'input, T>,
15    channel: isize,
16}
17
18better_any::tid! { impl<'input,T> TidAble<'input> for CommonTokenStream<'input, T> where T: TokenSource<'input>}
19
20impl<'input, T: TokenSource<'input>> IntStream for CommonTokenStream<'input, T> {
21    #[inline]
22    fn consume(&mut self) {
23        self.base.consume();
24        //        self.base.p = self.next_token_on_channel(self.base.p,self.channel);
25        //        self.base.current_token_index = self.base.p;
26        let next = self.next_token_on_channel(self.base.p, self.channel, 1);
27        self.base.seek(next);
28        // Ok(())
29    }
30
31    #[inline]
32    fn la(&mut self, i: isize) -> isize {
33        self.lt(i)
34            .map(|t| t.borrow().get_token_type())
35            .unwrap_or(TOKEN_INVALID_TYPE)
36    }
37
38    #[inline(always)]
39    fn mark(&mut self) -> isize {
40        0
41    }
42
43    #[inline(always)]
44    fn release(&mut self, _marker: isize) {}
45
46    #[inline(always)]
47    fn index(&self) -> isize {
48        self.base.index()
49    }
50
51    #[inline(always)]
52    fn seek(&mut self, index: isize) {
53        self.base.seek(index);
54    }
55
56    #[inline(always)]
57    fn size(&self) -> isize {
58        self.base.size()
59    }
60
61    fn get_source_name(&self) -> String {
62        self.base.get_source_name()
63    }
64}
65
66impl<'input, T: TokenSource<'input>> TokenStream<'input> for CommonTokenStream<'input, T> {
67    type TF = T::TF;
68
69    #[inline(always)]
70    fn lt(&mut self, k: isize) -> Option<&<Self::TF as TokenFactory<'input>>::Tok> {
71        if k == 1 {
72            return self.base.tokens.get(self.base.p as usize);
73        }
74        if k == 0 {
75            panic!();
76        }
77        if k < 0 {
78            return self.lb(-k);
79        }
80        self.lt_inner(k)
81    }
82
83    #[inline]
84    fn get(&self, index: isize) -> &<Self::TF as TokenFactory<'input>>::Tok {
85        self.base.get(index)
86    }
87
88    fn get_token_source(&self) -> &dyn TokenSource<'input, TF = Self::TF> {
89        self.base.get_token_source()
90    }
91
92    fn get_text_from_interval(&self, start: isize, stop: isize) -> String {
93        self.base.get_text_from_interval(start, stop)
94    }
95}
96
97impl<'input, T: TokenSource<'input>> CommonTokenStream<'input, T> {
98    /// Creates CommonTokenStream that produces tokens from `TOKEN_DEFAULT_CHANNEL`
99    pub fn new(lexer: T) -> CommonTokenStream<'input, T> {
100        Self::with_channel(lexer, TOKEN_DEFAULT_CHANNEL)
101    }
102
103    /// Creates CommonTokenStream that produces tokens from `channel`
104    pub fn with_channel(lexer: T, channel: isize) -> CommonTokenStream<'input, T> {
105        let mut r = CommonTokenStream {
106            base: UnbufferedTokenStream::new_buffered(lexer),
107            channel,
108        };
109        r.sync(0);
110        r
111    }
112
113    fn lt_inner(&mut self, k: isize) -> Option<&<T::TF as TokenFactory<'input>>::Tok> {
114        let mut i = self.base.p;
115        let mut n = 1; // we know tokens[p] is a good one
116                       // find k good tokens
117        while n < k {
118            // skip off-channel tokens, but make sure to not look past EOF
119            if self.sync(i + 1) {
120                i = self.next_token_on_channel(i + 1, self.channel, 1);
121            }
122            n += 1;
123        }
124        //		if ( i>range ) range = i;
125        return self.base.tokens.get(i as usize);
126    }
127
128    /// Restarts this token stream
129    pub fn reset(&mut self) {
130        self.base.p = 0;
131        self.base.current_token_index = 0;
132    }
133
134    /// Creates iterator over this token stream
135    pub fn iter(&mut self) -> IterWrapper<'_, Self> {
136        IterWrapper(self)
137    }
138
139    fn sync(&mut self, i: isize) -> bool {
140        let need = i - self.size() + 1;
141        if need > 0 {
142            let fetched = self.base.fill(need);
143            return fetched >= need;
144        }
145
146        true
147    }
148    //
149    //    fn fetch(&self, n: isize) -> int { unimplemented!() }
150    //
151    //    fn get_tokens(&self, start: isize, stop: isize, types: &IntervalSet) -> Vec<Token> { unimplemented!() }
152    //
153    //    fn lazy_init(&self) { unimplemented!() }
154    //
155    //    fn setup(&self) { unimplemented!() }
156    //
157    //    fn get_token_source(&self) -> TokenSource { unimplemented!() }
158    //
159    //    fn set_token_source(&self, tokenSource: TokenSource) { unimplemented!() }
160
161    //todo make this const generic over direction
162    fn next_token_on_channel(&mut self, mut i: isize, channel: isize, direction: isize) -> isize {
163        self.sync(i);
164        if i >= self.size() {
165            return self.size() - 1;
166        }
167
168        let mut token = self.base.tokens[i as usize].borrow();
169        while token.get_channel() != channel {
170            if token.get_token_type() == EOF || i < 0 {
171                return i;
172            }
173
174            i += direction;
175            self.sync(i);
176            token = self.base.tokens[i as usize].borrow();
177        }
178
179        return i;
180    }
181    //
182    //    fn previous_token_on_channel(&self, i: isize, channel: isize) -> int { unimplemented!() }
183    //
184    //    fn get_hidden_tokens_to_right(&self, tokenIndex: isize, channel: isize) -> Vec<Token> { unimplemented!() }
185    //
186    //    fn get_hidden_tokens_to_left(&self, tokenIndex: isize, channel: isize) -> Vec<Token> { unimplemented!() }
187    //
188    //    fn filter_for_channel(&self, left: isize, right: isize, channel: isize) -> Vec<Token> { unimplemented!() }
189    //
190    //    fn get_source_name(&self) -> String { unimplemented!() }
191    //
192    //    fn get_all_text(&self) -> String { unimplemented!() }
193    //
194    //    fn get_text_from_tokens(&self, start: Token, end: Token) -> String { unimplemented!() }
195    //
196    //    fn get_text_from_rule_context(&self, interval: RuleContext) -> String { unimplemented!() }
197    //
198    //    fn get_text_from_interval(&self, interval: &Interval) -> String { unimplemented!() }
199    //
200    //    fn fill(&self) { unimplemented!() }
201    //
202    //    fn adjust_seek_index(&self, i: isize) -> int { unimplemented!() }
203
204    fn lb(
205        &mut self,
206        k: isize,
207    ) -> Option<&<<Self as TokenStream<'input>>::TF as TokenFactory<'input>>::Tok> {
208        if k == 0 || (self.base.p - k) < 0 {
209            return None;
210        }
211
212        let mut i = self.base.p;
213        let mut n = 1;
214        // find k good tokens looking backwards
215        while n <= k && i > 0 {
216            // skip off-channel tokens
217            i = self.next_token_on_channel(i - 1, self.channel, -1);
218            n += 1;
219        }
220        if i < 0 {
221            return None;
222        }
223
224        return self.base.tokens.get(i as usize);
225    }
226
227    //    fn get_number_of_on_channel_tokens(&self) -> int { unimplemented!() }
228}
antlr_rust_sleagon/common_token_stream.rs

antlr_rust_sleagon/
common_token_stream.rs
