Struct TokenStream 

pub struct TokenStream<'a, T> { /* private fields */ }

A generic TokenStream struct that manages a stream of tokens with cursor and bookmark functionality.

Examples

use slk_tokenstream::TokenStream;
use slk_tokenstream::Mark;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);

assert_eq!(token_stream.consume(), Some(&1));
assert_eq!(token_stream.peek(), Some(&2));
assert_eq!(token_stream.tokens_remaining(), 2);

Implementations

impl<'a, T> TokenStream<'a, T>

pub fn new(data: &'a [T]) -> Self

Creates a new TokenStream from a vector of tokens. Sets cursor to 0.

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let token_stream = TokenStream::new(tokens);
 
assert_eq!(token_stream.peek(), Some(&1));
assert_eq!(token_stream.peek_offset(1), Some(&2));
assert_eq!(token_stream.peek_offset(2), Some(&3));

pub fn consume(&mut self) -> Option<&T>

Advances the cursor and returns the next token if available, otherwise returns None.

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
assert_eq!(token_stream.consume(), Some(&1));
assert_eq!(token_stream.consume(), Some(&2));
assert_eq!(token_stream.consume(), Some(&3));
assert_eq!(token_stream.consume(), None);

pub fn peek(&self) -> Option<&T>

Peeks at the token at the current cursor position without advancing the cursor.

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let token_stream = TokenStream::new(tokens);
 
assert_eq!(token_stream.peek(), Some(&1));

pub fn peek_offset(&self, offset: usize) -> Option<&T>

Peeks at the current cursor position plus an offset without advancing the cursor.

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let token_stream = TokenStream::new(tokens);
 
assert_eq!(token_stream.peek(), Some(&1));
assert_eq!(token_stream.peek_offset(1), Some(&2));
assert_eq!(token_stream.peek_offset(2), Some(&3));

pub fn rewind(&mut self)

Moves the cursor back by one position, saturating at zero.

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
assert_eq!(token_stream.consume(), Some(&1));
token_stream.rewind();
assert_eq!(token_stream.consume(), Some(&1));

pub fn rewind_offset(&mut self, offset: usize)

Rewinds the cursor a specified amount of times, saturating at 0.

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
assert_eq!(token_stream.consume(), Some(&1));
assert_eq!(token_stream.consume(), Some(&2));
assert_eq!(token_stream.consume(), Some(&3));
assert_eq!(token_stream.consume(), None);
token_stream.rewind_offset(2);
assert_eq!(token_stream.consume(), Some(&2));

pub fn mark(&self) -> Mark

Returns a mark to the current cursor position.

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
let mark = token_stream.mark();
 
token_stream.advance(5);
assert_eq!(token_stream.peek(), None);
token_stream.reset(&mark);
assert_eq!(token_stream.peek(), Some(&1));

pub fn reset(&mut self, bookmark: &Mark) -> usize

Moves the cursor to the position of a previously registered bookmark by handle and returns the previous position

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
let mark = token_stream.mark();
 
token_stream.advance(3);
assert_eq!(token_stream.peek(), None);
assert_eq!(token_stream.reset(&mark), 3);
assert_eq!(token_stream.peek(), Some(&1));

pub fn tokens_remaining(&self) -> usize

Returns the amount of tokens remaining, including the current token

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
assert_eq!(token_stream.tokens_remaining(), 3);
assert_eq!(token_stream.consume(), Some(&1));
assert_eq!(token_stream.tokens_remaining(), 2);
assert_eq!(token_stream.consume(), Some(&2));
assert_eq!(token_stream.tokens_remaining(), 1);

pub fn is_eof(&self) -> bool

Returns if the current token is the end of file

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
assert!(!token_stream.is_eof());
assert_eq!(token_stream.consume(), Some(&1));
assert_eq!(token_stream.consume(), Some(&2));
assert_eq!(token_stream.consume(), Some(&3));
assert!(token_stream.is_eof());

pub fn slice_from_marks(&self, mark_1: &Mark, mark_2: &Mark) -> &[T]

Returns a slice which starts on the earliest mark and ends on the latest.

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
let mark_1 = token_stream.mark();
token_stream.advance(3);
let mark_2 = token_stream.mark();
 
assert_eq!(token_stream.slice_from_marks(&mark_1, &mark_2), &[1, 2, 3]);
assert_eq!(token_stream.slice_from_marks(&mark_2, &mark_1), &[1, 2, 3]);

pub fn advance(&mut self, offset: usize)

Advances the cursor by specified amount

Cursor is clamped to the length of the data

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
token_stream.advance(2);
assert_eq!(token_stream.peek(), Some(&3));

pub fn peek_if<F: Fn(&T) -> bool>(&self, f: F) -> Option<&T>

Returns the next item if it exists and the closure returns true

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
assert_eq!(token_stream.peek_if(|token| *token == 1), Some(&1));
assert_eq!(token_stream.peek_if(|token| *token == 2), None);

pub fn expect<F: Fn(&T) -> bool>(&mut self, f: F) -> Option<&T>

Returns the next item and advances the cursor if the item exists and the closure returns true

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
assert_eq!(token_stream.expect(|token| *token == 1), Some(&1));
assert_eq!(token_stream.expect(|token| *token == 2), Some(&2));

pub fn consume_while<F: Fn(&T) -> bool>(&mut self, f: F) -> &[T]

Returns a slice of items starting from the cursor and ending when the closure returns false. The cursor remains on the first item failing the test

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
assert_eq!(token_stream.consume_while(|token| *token < 3), &[1, 2]);
assert_eq!(token_stream.peek(), Some(&3));

pub fn peek_while<F: Fn(&T) -> bool>(&self, f: F) -> &[T]

Returns a slice of items starting from the cursor and ending when the closure returns false. The cursor remains in the original position

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
assert_eq!(token_stream.peek_while(|token| *token < 3), &[1, 2]);
assert_eq!(token_stream.peek(), Some(&1));

pub fn skip(&mut self)

Advances the cursor 1 step

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
token_stream.skip();
assert_eq!(token_stream.peek(), Some(&2));
token_stream.skip();
assert_eq!(token_stream.peek(), Some(&3));

pub fn skip_if<F: Fn(&T) -> bool>(&mut self, f: F)

Advances the cursor one step if the closure returns true

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
token_stream.skip_if(|token| *token == 1);
assert_eq!(token_stream.peek(), Some(&2));
token_stream.skip_if(|token| *token == 1);
assert_eq!(token_stream.peek(), Some(&2));

pub fn skip_while<F: Fn(&T) -> bool>(&mut self, f: F)

Advances the cursor until the closure returns false

Examples

use slk_tokenstream::TokenStream;
 
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
token_stream.skip_while(|token| *token < 3);
assert_eq!(token_stream.peek(), Some(&3));

pub fn position(&self) -> usize

Returns the current position of the cursor

Examples

use slk_tokenstream::TokenStream;
let tokens = &[1, 2, 3];
let mut token_stream = TokenStream::new(tokens);
 
assert_eq!(token_stream.position(), 0);
assert_eq!(token_stream.consume(), Some(&1));
assert_eq!(token_stream.position(), 1);
assert_eq!(token_stream.consume(), Some(&2));
assert_eq!(token_stream.position(), 2);
assert_eq!(token_stream.consume(), Some(&3));
assert_eq!(token_stream.position(), 3);

Trait Implementations

impl<'a, T: Debug> Debug for TokenStream<'a, T>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.