slk_tokenstream/tokenstream.rs
1use crate::{bookmark::Mark, span::TokenstreamSpan};
2
3/// A generic TokenStream struct that manages a stream of tokens with cursor and bookmark functionality.
4///
5/// # Examples
6///
7/// ``` rust
8/// use slk_tokenstream::TokenStream;
9/// use slk_tokenstream::Mark;
10///
11/// let tokens = &[1, 2, 3];
12/// let mut token_stream = TokenStream::new(tokens);
13///
14/// assert_eq!(token_stream.consume(), Some(&1));
15/// assert_eq!(token_stream.peek(), Some(&2));
16/// assert_eq!(token_stream.tokens_remaining(), 2);
17/// ```
18#[derive(Debug)]
19pub struct TokenStream<'a, T> {
20 data: &'a [T],
21 cursor: usize,
22}
23
24impl<'a, T> TokenStream<'a, T> {
25 /// Creates a new TokenStream from a vector of tokens. Sets cursor to 0.
26 ///
27 /// # Examples
28 ///
29 /// ``` rust
30 /// use slk_tokenstream::TokenStream;
31 ///
32 /// let tokens = &[1, 2, 3];
33 /// let token_stream = TokenStream::new(tokens);
34 ///
35 /// assert_eq!(token_stream.peek(), Some(&1));
36 /// assert_eq!(token_stream.peek_offset(1), Some(&2));
37 /// assert_eq!(token_stream.peek_offset(2), Some(&3));
38 /// ```
39 pub fn new(data: &'a [T]) -> Self {
40 TokenStream { data, cursor: 0 }
41 }
42 /// Advances the cursor and returns the next token if available, otherwise returns None.
43 ///
44 /// # Examples
45 ///
46 /// ``` rust
47 /// use slk_tokenstream::TokenStream;
48 ///
49 /// let tokens = &[1, 2, 3];
50 /// let mut token_stream = TokenStream::new(tokens);
51 ///
52 /// assert_eq!(token_stream.consume(), Some(&1));
53 /// assert_eq!(token_stream.consume(), Some(&2));
54 /// assert_eq!(token_stream.consume(), Some(&3));
55 /// assert_eq!(token_stream.consume(), None);
56 /// ```
57 pub fn consume(&mut self) -> Option<&T> {
58 self.data.get(self.cursor).inspect(|_| self.cursor += 1)
59 }
60 /// Peeks at the token at the current cursor position without advancing the cursor.
61 ///
62 /// # Examples
63 /// ``` rust
64 /// use slk_tokenstream::TokenStream;
65 ///
66 /// let tokens = &[1, 2, 3];
67 /// let token_stream = TokenStream::new(tokens);
68 ///
69 /// assert_eq!(token_stream.peek(), Some(&1));
70 /// ```
71 pub fn peek(&self) -> Option<&T> {
72 self.peek_offset(0)
73 }
74 /// Peeks at the current cursor position plus an offset without advancing the cursor.
75 ///
76 /// # Examples
77 ///
78 /// ``` rust
79 /// use slk_tokenstream::TokenStream;
80 ///
81 /// let tokens = &[1, 2, 3];
82 /// let token_stream = TokenStream::new(tokens);
83 ///
84 /// assert_eq!(token_stream.peek(), Some(&1));
85 /// assert_eq!(token_stream.peek_offset(1), Some(&2));
86 /// assert_eq!(token_stream.peek_offset(2), Some(&3));
87 /// ```
88 pub fn peek_offset(&self, offset: usize) -> Option<&T> {
89 self.data.get(self.cursor.saturating_add(offset))
90 }
91 /// Moves the cursor back by one position, saturating at zero.
92 ///
93 /// # Examples
94 /// ``` rust
95 /// use slk_tokenstream::TokenStream;
96 ///
97 /// let tokens = &[1, 2, 3];
98 /// let mut token_stream = TokenStream::new(tokens);
99 ///
100 /// assert_eq!(token_stream.consume(), Some(&1));
101 /// token_stream.rewind();
102 /// assert_eq!(token_stream.consume(), Some(&1));
103 /// ```
104 pub fn rewind(&mut self) {
105 self.rewind_offset(1);
106 }
107 /// Rewinds the cursor a specified amount of times, saturating at 0.
108 ///
109 /// # Examples
110 /// ``` rust
111 /// use slk_tokenstream::TokenStream;
112 ///
113 /// let tokens = &[1, 2, 3];
114 /// let mut token_stream = TokenStream::new(tokens);
115 ///
116 /// assert_eq!(token_stream.consume(), Some(&1));
117 /// assert_eq!(token_stream.consume(), Some(&2));
118 /// assert_eq!(token_stream.consume(), Some(&3));
119 /// assert_eq!(token_stream.consume(), None);
120 /// token_stream.rewind_offset(2);
121 /// assert_eq!(token_stream.consume(), Some(&2));
122 /// ```
123 pub fn rewind_offset(&mut self, offset: usize) {
124 self.cursor = self.cursor.saturating_sub(offset);
125 }
126 /// Returns a mark to the current cursor position.
127 ///
128 /// # Examples
129 /// ``` rust
130 /// use slk_tokenstream::TokenStream;
131 ///
132 /// let tokens = &[1, 2, 3];
133 /// let mut token_stream = TokenStream::new(tokens);
134 /// let mark = token_stream.mark();
135 ///
136 /// token_stream.advance(5);
137 /// assert_eq!(token_stream.peek(), None);
138 /// token_stream.reset(&mark);
139 /// assert_eq!(token_stream.peek(), Some(&1));
140 /// ```
141 pub fn mark(&self) -> Mark {
142 Mark::new(self.cursor)
143 }
144 /// Moves the cursor to the position of a previously registered bookmark by handle and returns the previous position
145 ///
146 /// # Examples
147 /// ``` rust
148 /// use slk_tokenstream::TokenStream;
149 ///
150 /// let tokens = &[1, 2, 3];
151 /// let mut token_stream = TokenStream::new(tokens);
152 /// let mark = token_stream.mark();
153 ///
154 /// token_stream.advance(3);
155 /// assert_eq!(token_stream.peek(), None);
156 /// assert_eq!(token_stream.reset(&mark), 3);
157 /// assert_eq!(token_stream.peek(), Some(&1));
158 /// ```
159 pub fn reset(&mut self, bookmark: &Mark) -> usize {
160 let old = self.cursor;
161 self.cursor = bookmark.position();
162 old
163 }
164 /// Returns the amount of tokens remaining, including the current token
165 ///
166 /// # Examples
167 ///
168 /// ``` rust
169 /// use slk_tokenstream::TokenStream;
170 ///
171 /// let tokens = &[1, 2, 3];
172 /// let mut token_stream = TokenStream::new(tokens);
173 ///
174 /// assert_eq!(token_stream.tokens_remaining(), 3);
175 /// assert_eq!(token_stream.consume(), Some(&1));
176 /// assert_eq!(token_stream.tokens_remaining(), 2);
177 /// assert_eq!(token_stream.consume(), Some(&2));
178 /// assert_eq!(token_stream.tokens_remaining(), 1);
179 /// ```
180 pub fn tokens_remaining(&self) -> usize {
181 self.data.len().saturating_sub(self.cursor)
182 }
183 /// Returns if the current token is the end of file
184 ///
185 /// # Examples
186 ///
187 /// ``` rust
188 /// use slk_tokenstream::TokenStream;
189 ///
190 /// let tokens = &[1, 2, 3];
191 /// let mut token_stream = TokenStream::new(tokens);
192 ///
193 /// assert!(!token_stream.is_eof());
194 /// assert_eq!(token_stream.consume(), Some(&1));
195 /// assert_eq!(token_stream.consume(), Some(&2));
196 /// assert_eq!(token_stream.consume(), Some(&3));
197 /// assert!(token_stream.is_eof());
198 /// ```
199 pub fn is_eof(&self) -> bool {
200 self.peek().is_none()
201 }
202 /// Returns a slice from a span
203 ///
204 /// # Examples
205 ///
206 /// ``` rust
207 /// use slk_tokenstream::TokenStream;
208 ///
209 /// let tokens = &[1, 2, 3];
210 /// let mut token_stream = TokenStream::new(tokens);
211 /// let mark_1 = token_stream.mark();
212 /// token_stream.advance(3);
213 /// let mark_2 = token_stream.mark();
214 ///
215 /// let span = token_stream.span_from_marks(mark_1, mark_2);
216 ///
217 /// assert_eq!(token_stream.slice_from_span(&span), &[1, 2, 3]);
218 /// ```
219 pub fn slice_from_span(&self, span: &TokenstreamSpan) -> &[T] {
220 let idx_1 = span.start().position();
221 let idx_2 = span.end().position();
222 &self.data[idx_1..idx_2]
223 }
224 /// Advances the cursor by specified amount
225 ///
226 /// Cursor is clamped to the length of the data
227 ///
228 /// # Examples
229 ///
230 /// ``` rust
231 /// use slk_tokenstream::TokenStream;
232 ///
233 /// let tokens = &[1, 2, 3];
234 /// let mut token_stream = TokenStream::new(tokens);
235 ///
236 /// token_stream.advance(2);
237 /// assert_eq!(token_stream.peek(), Some(&3));
238 /// ```
239 pub fn advance(&mut self, offset: usize) {
240 self.cursor = self.data.len().min(self.cursor.saturating_add(offset));
241 }
242 /// Returns the next item if it exists and the closure returns true
243 ///
244 /// # Examples
245 ///
246 /// ``` rust
247 /// use slk_tokenstream::TokenStream;
248 ///
249 /// let tokens = &[1, 2, 3];
250 /// let mut token_stream = TokenStream::new(tokens);
251 ///
252 /// assert_eq!(token_stream.peek_if(|token| *token == 1), Some(&1));
253 /// assert_eq!(token_stream.peek_if(|token| *token == 2), None);
254 /// ```
255 pub fn peek_if<F: Fn(&T) -> bool>(&self, f: F) -> Option<&T> {
256 match self.peek() {
257 Some(v) if f(v) => Some(v),
258 _ => None,
259 }
260 }
261 /// Returns the next item and advances the cursor if the item exists and the closure returns true
262 ///
263 /// # Examples
264 ///
265 /// ``` rust
266 /// use slk_tokenstream::TokenStream;
267 ///
268 /// let tokens = &[1, 2, 3];
269 /// let mut token_stream = TokenStream::new(tokens);
270 ///
271 /// assert_eq!(token_stream.consume_if(|token| *token == 1), Some(&1));
272 /// assert_eq!(token_stream.consume_if(|token| *token == 2), Some(&2));
273 /// ```
274 pub fn consume_if<F: Fn(&T) -> bool>(&mut self, f: F) -> Option<&T> {
275 let ok = match self.peek() {
276 Some(v) if f(v) => true,
277 _ => false,
278 };
279 if ok { self.consume() } else { None }
280 }
281 /// Returns a slice of items starting from the cursor and ending when the closure returns false. The cursor remains on the first item failing the test
282 ///
283 /// # Examples
284 ///
285 /// ``` rust
286 /// use slk_tokenstream::TokenStream;
287 ///
288 /// let tokens = &[1, 2, 3];
289 /// let mut token_stream = TokenStream::new(tokens);
290 ///
291 /// assert_eq!(token_stream.consume_while(|token| *token < 3), &[1, 2]);
292 /// assert_eq!(token_stream.peek(), Some(&3));
293 /// ```
294 pub fn consume_while<F: Fn(&T) -> bool>(&mut self, f: F) -> &[T] {
295 let m1 = self.mark();
296 while self.consume_if(&f).is_some() {}
297 let m2 = self.mark();
298 let slice = self.slice_from_span(&self.span_from_marks(m1, m2));
299 slice
300 }
301 /// Returns a slice of items starting from the cursor and ending when the closure returns false. The cursor remains in the original position
302 ///
303 /// # Examples
304 ///
305 /// ``` rust
306 /// use slk_tokenstream::TokenStream;
307 ///
308 /// let tokens = &[1, 2, 3];
309 /// let mut token_stream = TokenStream::new(tokens);
310 ///
311 /// assert_eq!(token_stream.peek_while(|token| *token < 3), &[1, 2]);
312 /// assert_eq!(token_stream.peek(), Some(&1));
313 /// ```
314 pub fn peek_while<F: Fn(&T) -> bool>(&self, f: F) -> &[T] {
315 let len = self.data[self.cursor..]
316 .iter().take_while(|item| f(item))
317 .count();
318 &self.data[self.cursor..self.cursor + len]
319 }
320 /// Advances the cursor 1 step
321 ///
322 /// # Examples
323 ///
324 /// ``` rust
325 /// use slk_tokenstream::TokenStream;
326 ///
327 /// let tokens = &[1, 2, 3];
328 /// let mut token_stream = TokenStream::new(tokens);
329 ///
330 /// token_stream.skip();
331 /// assert_eq!(token_stream.peek(), Some(&2));
332 /// token_stream.skip();
333 /// assert_eq!(token_stream.peek(), Some(&3));
334 /// ```
335 pub fn skip(&mut self) {
336 self.advance(1);
337 }
338 /// Advances the cursor one step if the closure returns true
339 ///
340 /// # Examples
341 ///
342 /// ``` rust
343 /// use slk_tokenstream::TokenStream;
344 ///
345 /// let tokens = &[1, 2, 3];
346 /// let mut token_stream = TokenStream::new(tokens);
347 ///
348 /// token_stream.skip_if(|token| *token == 1);
349 /// assert_eq!(token_stream.peek(), Some(&2));
350 /// token_stream.skip_if(|token| *token == 1);
351 /// assert_eq!(token_stream.peek(), Some(&2));
352 /// ```
353 pub fn skip_if<F: Fn(&T) -> bool>(&mut self, f: F) {
354 match self.peek_if(f) {
355 Some(_) => self.skip(),
356 None => {}
357 }
358 }
359 /// Advances the cursor until the closure returns false
360 ///
361 /// # Examples
362 ///
363 /// ``` rust
364 /// use slk_tokenstream::TokenStream;
365 ///
366 /// let tokens = &[1, 2, 3];
367 /// let mut token_stream = TokenStream::new(tokens);
368 ///
369 /// token_stream.skip_while(|token| *token < 3);
370 /// assert_eq!(token_stream.peek(), Some(&3));
371 /// ```
372 pub fn skip_while<F: Fn(&T) -> bool>(&mut self, f: F) {
373 while self.peek_if(&f).is_some() {
374 self.advance(1);
375 }
376 }
377
378 /// Returns the current position of the cursor
379 ///
380 /// # Examples
381 ///
382 /// ``` rust
383 /// use slk_tokenstream::TokenStream;
384 /// let tokens = &[1, 2, 3];
385 /// let mut token_stream = TokenStream::new(tokens);
386 ///
387 /// assert_eq!(token_stream.position(), 0);
388 /// assert_eq!(token_stream.consume(), Some(&1));
389 /// assert_eq!(token_stream.position(), 1);
390 /// assert_eq!(token_stream.consume(), Some(&2));
391 /// assert_eq!(token_stream.position(), 2);
392 /// assert_eq!(token_stream.consume(), Some(&3));
393 /// assert_eq!(token_stream.position(), 3);
394 /// ```
395 pub fn position(&self) -> usize {
396 self.cursor
397 }
398
399 /// Creates a span from two marks
400 ///
401 /// # Examples
402 ///
403 /// ```rust
404 /// use slk_tokenstream::TokenStream;
405 /// let tokens = &[1, 2, 3];
406 /// let mut token_stream = TokenStream::new(tokens);
407 ///
408 /// let m1 = token_stream.mark();
409 /// token_stream.consume();
410 /// let m2 = token_stream.mark();
411 ///
412 /// let span = token_stream.span_from_marks(m1, m2);
413 ///
414 /// assert_eq!(token_stream.slice_from_span(&span), &[1]);
415 /// ```
416 pub fn span_from_marks(&self, start: Mark, end: Mark) -> TokenstreamSpan {
417 TokenstreamSpan::new(start, end)
418 }
419}