slk_tokenstream/tokenstream.rs
1use crate::bookmark::Mark;
2
3/// A generic TokenStream struct that manages a stream of tokens with cursor and bookmark functionality.
4///
5/// # Examples
6///
7/// ``` rust
8/// use slk_tokenstream::TokenStream;
9/// use slk_tokenstream::Mark;
10///
11/// let tokens = &[1, 2, 3];
12/// let mut token_stream = TokenStream::new(tokens);
13///
14/// assert_eq!(token_stream.consume(), Some(&1));
15/// assert_eq!(token_stream.peek(), Some(&2));
16/// assert_eq!(token_stream.tokens_remaining(), 2);
17/// ```
18#[derive(Debug)]
19pub struct TokenStream<'a, T> {
20 data: &'a [T],
21 cursor: usize,
22}
23
24impl<'a, T> TokenStream<'a, T> {
25 /// Creates a new TokenStream from a vector of tokens. Sets cursor to 0.
26 ///
27 /// # Examples
28 ///
29 /// ``` rust
30 /// use slk_tokenstream::TokenStream;
31 ///
32 /// let tokens = &[1, 2, 3];
33 /// let token_stream = TokenStream::new(tokens);
34 ///
35 /// assert_eq!(token_stream.peek(), Some(&1));
36 /// assert_eq!(token_stream.peek_offset(1), Some(&2));
37 /// assert_eq!(token_stream.peek_offset(2), Some(&3));
38 /// ```
39 pub fn new(data: &'a [T]) -> Self {
40 TokenStream { data, cursor: 0 }
41 }
42 /// Advances the cursor and returns the next token if available, otherwise returns None.
43 ///
44 /// # Examples
45 ///
46 /// ``` rust
47 /// use slk_tokenstream::TokenStream;
48 ///
49 /// let tokens = &[1, 2, 3];
50 /// let mut token_stream = TokenStream::new(tokens);
51 ///
52 /// assert_eq!(token_stream.consume(), Some(&1));
53 /// assert_eq!(token_stream.consume(), Some(&2));
54 /// assert_eq!(token_stream.consume(), Some(&3));
55 /// assert_eq!(token_stream.consume(), None);
56 /// ```
57 pub fn consume(&mut self) -> Option<&T> {
58 self.data.get(self.cursor).inspect(|_| self.cursor += 1)
59 }
60 /// Peeks at the token at the current cursor position without advancing the cursor.
61 ///
62 /// # Examples
63 /// ``` rust
64 /// use slk_tokenstream::TokenStream;
65 ///
66 /// let tokens = &[1, 2, 3];
67 /// let token_stream = TokenStream::new(tokens);
68 ///
69 /// assert_eq!(token_stream.peek(), Some(&1));
70 /// ```
71 pub fn peek(&self) -> Option<&T> {
72 self.peek_offset(0)
73 }
74 /// Peeks at the current cursor position plus an offset without advancing the cursor.
75 ///
76 /// # Examples
77 ///
78 /// ``` rust
79 /// use slk_tokenstream::TokenStream;
80 ///
81 /// let tokens = &[1, 2, 3];
82 /// let token_stream = TokenStream::new(tokens);
83 ///
84 /// assert_eq!(token_stream.peek(), Some(&1));
85 /// assert_eq!(token_stream.peek_offset(1), Some(&2));
86 /// assert_eq!(token_stream.peek_offset(2), Some(&3));
87 /// ```
88 pub fn peek_offset(&self, offset: usize) -> Option<&T> {
89 self.data.get(self.cursor.saturating_add(offset))
90 }
91 /// Moves the cursor back by one position, saturating at zero.
92 ///
93 /// # Examples
94 /// ``` rust
95 /// use slk_tokenstream::TokenStream;
96 ///
97 /// let tokens = &[1, 2, 3];
98 /// let mut token_stream = TokenStream::new(tokens);
99 ///
100 /// assert_eq!(token_stream.consume(), Some(&1));
101 /// token_stream.rewind();
102 /// assert_eq!(token_stream.consume(), Some(&1));
103 /// ```
104 pub fn rewind(&mut self) {
105 self.rewind_offset(1);
106 }
107 /// Rewinds the cursor a specified amount of times, saturating at 0.
108 ///
109 /// # Examples
110 /// ``` rust
111 /// use slk_tokenstream::TokenStream;
112 ///
113 /// let tokens = &[1, 2, 3];
114 /// let mut token_stream = TokenStream::new(tokens);
115 ///
116 /// assert_eq!(token_stream.consume(), Some(&1));
117 /// assert_eq!(token_stream.consume(), Some(&2));
118 /// assert_eq!(token_stream.consume(), Some(&3));
119 /// assert_eq!(token_stream.consume(), None);
120 /// token_stream.rewind_offset(2);
121 /// assert_eq!(token_stream.consume(), Some(&2));
122 /// ```
123 pub fn rewind_offset(&mut self, offset: usize) {
124 self.cursor = self.cursor.saturating_sub(offset);
125 }
126 /// Returns a mark to the current cursor position.
127 ///
128 /// # Examples
129 /// ``` rust
130 /// use slk_tokenstream::TokenStream;
131 ///
132 /// let tokens = &[1, 2, 3];
133 /// let mut token_stream = TokenStream::new(tokens);
134 /// let mark = token_stream.mark();
135 ///
136 /// token_stream.advance(5);
137 /// assert_eq!(token_stream.peek(), None);
138 /// token_stream.reset(&mark);
139 /// assert_eq!(token_stream.peek(), Some(&1));
140 /// ```
141 pub fn mark(&self) -> Mark {
142 Mark::new(self.cursor)
143 }
144 /// Moves the cursor to the position of a previously registered bookmark by handle and returns the previous position
145 ///
146 /// # Examples
147 /// ``` rust
148 /// use slk_tokenstream::TokenStream;
149 ///
150 /// let tokens = &[1, 2, 3];
151 /// let mut token_stream = TokenStream::new(tokens);
152 /// let mark = token_stream.mark();
153 ///
154 /// token_stream.advance(3);
155 /// assert_eq!(token_stream.peek(), None);
156 /// assert_eq!(token_stream.reset(&mark), 3);
157 /// assert_eq!(token_stream.peek(), Some(&1));
158 /// ```
159 pub fn reset(&mut self, bookmark: &Mark) -> usize {
160 let old = self.cursor;
161 self.cursor = bookmark.position();
162 old
163 }
164 /// Returns the amount of tokens remaining, including the current token
165 ///
166 /// # Examples
167 ///
168 /// ``` rust
169 /// use slk_tokenstream::TokenStream;
170 ///
171 /// let tokens = &[1, 2, 3];
172 /// let mut token_stream = TokenStream::new(tokens);
173 ///
174 /// assert_eq!(token_stream.tokens_remaining(), 3);
175 /// assert_eq!(token_stream.consume(), Some(&1));
176 /// assert_eq!(token_stream.tokens_remaining(), 2);
177 /// assert_eq!(token_stream.consume(), Some(&2));
178 /// assert_eq!(token_stream.tokens_remaining(), 1);
179 /// ```
180 pub fn tokens_remaining(&self) -> usize {
181 self.data.len().saturating_sub(self.cursor)
182 }
183 /// Returns if the current token is the end of file
184 ///
185 /// # Examples
186 ///
187 /// ``` rust
188 /// use slk_tokenstream::TokenStream;
189 ///
190 /// let tokens = &[1, 2, 3];
191 /// let mut token_stream = TokenStream::new(tokens);
192 ///
193 /// assert!(!token_stream.is_eof());
194 /// assert_eq!(token_stream.consume(), Some(&1));
195 /// assert_eq!(token_stream.consume(), Some(&2));
196 /// assert_eq!(token_stream.consume(), Some(&3));
197 /// assert!(token_stream.is_eof());
198 /// ```
199 pub fn is_eof(&self) -> bool {
200 self.peek().is_none()
201 }
202 /// Returns a slice which starts on the earliest mark and ends on the latest.
203 ///
204 /// # Examples
205 ///
206 /// ``` rust
207 /// use slk_tokenstream::TokenStream;
208 ///
209 /// let tokens = &[1, 2, 3];
210 /// let mut token_stream = TokenStream::new(tokens);
211 /// let mark_1 = token_stream.mark();
212 /// token_stream.advance(3);
213 /// let mark_2 = token_stream.mark();
214 ///
215 /// assert_eq!(token_stream.slice_from_marks(&mark_1, &mark_2), &[1, 2, 3]);
216 /// assert_eq!(token_stream.slice_from_marks(&mark_2, &mark_1), &[1, 2, 3]);
217 /// ```
218 pub fn slice_from_marks(&self, mark_1: &Mark, mark_2: &Mark) -> &[T] {
219 let mut idx_1 = mark_1.position();
220 let mut idx_2 = mark_2.position();
221 if idx_1 >= idx_2 {
222 core::mem::swap(&mut idx_1, &mut idx_2);
223 }
224 &self.data[idx_1..idx_2]
225 }
226 /// Advances the cursor by specified amount
227 ///
228 /// Cursor is clamped to the length of the data
229 ///
230 /// # Examples
231 ///
232 /// ``` rust
233 /// use slk_tokenstream::TokenStream;
234 ///
235 /// let tokens = &[1, 2, 3];
236 /// let mut token_stream = TokenStream::new(tokens);
237 ///
238 /// token_stream.advance(2);
239 /// assert_eq!(token_stream.peek(), Some(&3));
240 /// ```
241 pub fn advance(&mut self, offset: usize) {
242 self.cursor = self.data.len().min(self.cursor.saturating_add(offset));
243 }
244 /// Returns the next item if it exists and the closure returns true
245 ///
246 /// # Examples
247 ///
248 /// ``` rust
249 /// use slk_tokenstream::TokenStream;
250 ///
251 /// let tokens = &[1, 2, 3];
252 /// let mut token_stream = TokenStream::new(tokens);
253 ///
254 /// assert_eq!(token_stream.peek_if(|token| *token == 1), Some(&1));
255 /// assert_eq!(token_stream.peek_if(|token| *token == 2), None);
256 /// ```
257 pub fn peek_if<F: Fn(&T) -> bool>(&self, f: F) -> Option<&T> {
258 match self.peek() {
259 Some(v) if f(v) => Some(v),
260 _ => None,
261 }
262 }
263 /// Returns the next item and advances the cursor if the item exists and the closure returns true
264 ///
265 /// # Examples
266 ///
267 /// ``` rust
268 /// use slk_tokenstream::TokenStream;
269 ///
270 /// let tokens = &[1, 2, 3];
271 /// let mut token_stream = TokenStream::new(tokens);
272 ///
273 /// assert_eq!(token_stream.expect(|token| *token == 1), Some(&1));
274 /// assert_eq!(token_stream.expect(|token| *token == 2), Some(&2));
275 /// ```
276 pub fn expect<F: Fn(&T) -> bool>(&mut self, f: F) -> Option<&T> {
277 let ok = match self.peek() {
278 Some(v) if f(v) => true,
279 _ => false,
280 };
281 if ok { self.consume() } else { None }
282 }
283 /// Returns a slice of items starting from the cursor and ending when the closure returns false. The cursor remains on the first item failing the test
284 ///
285 /// # Examples
286 ///
287 /// ``` rust
288 /// use slk_tokenstream::TokenStream;
289 ///
290 /// let tokens = &[1, 2, 3];
291 /// let mut token_stream = TokenStream::new(tokens);
292 ///
293 /// assert_eq!(token_stream.consume_while(|token| *token < 3), &[1, 2]);
294 /// assert_eq!(token_stream.peek(), Some(&3));
295 /// ```
296 pub fn consume_while<F: Fn(&T) -> bool>(&mut self, f: F) -> &[T] {
297 let m1 = self.mark();
298 while self.expect(&f).is_some() {}
299 let m2 = self.mark();
300 let slice = self.slice_from_marks(&m1, &m2);
301 slice
302 }
303 /// Returns a slice of items starting from the cursor and ending when the closure returns false. The cursor remains in the original position
304 ///
305 /// # Examples
306 ///
307 /// ``` rust
308 /// use slk_tokenstream::TokenStream;
309 ///
310 /// let tokens = &[1, 2, 3];
311 /// let mut token_stream = TokenStream::new(tokens);
312 ///
313 /// assert_eq!(token_stream.peek_while(|token| *token < 3), &[1, 2]);
314 /// assert_eq!(token_stream.peek(), Some(&1));
315 /// ```
316 pub fn peek_while<F: Fn(&T) -> bool>(&self, f: F) -> &[T] {
317 let len = self.data[self.cursor..]
318 .iter().take_while(|item| f(item))
319 .count();
320 &self.data[self.cursor..self.cursor + len]
321 }
322 /// Advances the cursor 1 step
323 ///
324 /// # Examples
325 ///
326 /// ``` rust
327 /// use slk_tokenstream::TokenStream;
328 ///
329 /// let tokens = &[1, 2, 3];
330 /// let mut token_stream = TokenStream::new(tokens);
331 ///
332 /// token_stream.skip();
333 /// assert_eq!(token_stream.peek(), Some(&2));
334 /// token_stream.skip();
335 /// assert_eq!(token_stream.peek(), Some(&3));
336 /// ```
337 pub fn skip(&mut self) {
338 self.advance(1);
339 }
340 /// Advances the cursor one step if the closure returns true
341 ///
342 /// # Examples
343 ///
344 /// ``` rust
345 /// use slk_tokenstream::TokenStream;
346 ///
347 /// let tokens = &[1, 2, 3];
348 /// let mut token_stream = TokenStream::new(tokens);
349 ///
350 /// token_stream.skip_if(|token| *token == 1);
351 /// assert_eq!(token_stream.peek(), Some(&2));
352 /// token_stream.skip_if(|token| *token == 1);
353 /// assert_eq!(token_stream.peek(), Some(&2));
354 /// ```
355 pub fn skip_if<F: Fn(&T) -> bool>(&mut self, f: F) {
356 match self.peek_if(f) {
357 Some(_) => self.skip(),
358 None => {}
359 }
360 }
361 /// Advances the cursor until the closure returns false
362 ///
363 /// # Examples
364 ///
365 /// ``` rust
366 /// use slk_tokenstream::TokenStream;
367 ///
368 /// let tokens = &[1, 2, 3];
369 /// let mut token_stream = TokenStream::new(tokens);
370 ///
371 /// token_stream.skip_while(|token| *token < 3);
372 /// assert_eq!(token_stream.peek(), Some(&3));
373 /// ```
374 pub fn skip_while<F: Fn(&T) -> bool>(&mut self, f: F) {
375 while self.peek_if(&f).is_some() {
376 self.advance(1);
377 }
378 }
379
380 /// Returns the current position of the cursor
381 ///
382 /// # Examples
383 ///
384 /// ``` rust
385 /// use slk_tokenstream::TokenStream;
386 /// let tokens = &[1, 2, 3];
387 /// let mut token_stream = TokenStream::new(tokens);
388 ///
389 /// assert_eq!(token_stream.position(), 0);
390 /// assert_eq!(token_stream.consume(), Some(&1));
391 /// assert_eq!(token_stream.position(), 1);
392 /// assert_eq!(token_stream.consume(), Some(&2));
393 /// assert_eq!(token_stream.position(), 2);
394 /// assert_eq!(token_stream.consume(), Some(&3));
395 /// assert_eq!(token_stream.position(), 3);
396 /// ```
397 pub fn position(&self) -> usize {
398 self.cursor
399 }
400}