Struct simple_tokenizer::Tokens

source ·

pub struct Tokens<'s> { /* private fields */ }

Expand description

Tokens instance.

Implementations§

source §

impl<'s> Tokens<'s>

source

pub fn new(input: &'s str) -> Self

Construct a new instance from a string slice.

source

pub fn input(&self) -> &str

Returns the original full input.

source

pub fn remainder(&self) -> &str

Part of the input string that hasn’t been consumed yet.

source

pub fn span(&self) -> Span

Byte span of the last token.

source

pub fn position(&self) -> Position

Current position (just after the last token).

source

pub fn offset(&self) -> Offset

Current byte offset in the source.

source

pub fn set_offset(&mut self, offset: Offset) -> bool

Sets the offset if it is valid, updating position and span. Returns true if the offset is valid, false otherwise.

source

pub fn is_at_start(&self) -> bool

Returns true if the current position is the start of input.

source

pub fn is_at_end(&self) -> bool

Returns true if the input has been exhausted.

source

pub fn peek(&self) -> Option<char>

Peeks at the next character of the input.

source

pub fn consume_all(&mut self) -> &str

Consumes the rest of input.

Example

use simple_tokenizer::*;

let mut tokens = "tokens".as_tokens();

assert_eq!(tokens.consume_all(), "tokens");
assert!(tokens.remainder().is_empty());

source

pub fn token(&mut self, token: impl AsRef<str>) -> bool

Consume the next substring equal to token or nothing. Basically a shortcut for bytes_if(token.len(), |s| s == token).is_some().

Example

use simple_tokenizer::*;

let mut tokens = "tok1 tok2".as_tokens();

assert!(tokens.token("tok1"));
assert_eq!(tokens.remainder(), " tok2");

assert!(!tokens.token(" tok3"));
assert_eq!(tokens.remainder(), " tok2");

source

pub fn tokens( &mut self, tokens: impl IntoIterator<Item = impl AsRef<str>> ) -> Option<&str>

Try to consume a substring equal to one of the provided tokens. Returns the first successful substring.

Example

use simple_tokenizer::*;

let mut tokens = "tok1 tok2".as_tokens();

assert_eq!(tokens.tokens(&["tok", "tok1"]), Some("tok"));
assert_eq!(tokens.remainder(), "1 tok2");

assert_eq!(tokens.tokens(&["1 tok3", "2 tok2"]), None);
assert_eq!(tokens.remainder(), "1 tok2");

source

pub fn char(&mut self) -> Option<char>

Consume the next character.

Example

use simple_tokenizer::*;

let mut tokens = "tokens".as_tokens();

assert_eq!(tokens.char(), Some('t'));
assert_eq!(tokens.remainder(), "okens");

source

pub fn char_if(&mut self, f: impl FnOnce(char) -> bool) -> Option<char>

Consume the next character if it matches a predicate.

Example

use simple_tokenizer::*;

let mut tokens = "tokens".as_tokens();

assert_eq!(tokens.char_if(char::is_alphabetic), Some('t'));
assert_eq!(tokens.remainder(), "okens");

assert_eq!(tokens.char_if(char::is_numeric), None);
assert_eq!(tokens.remainder(), "okens");

source

pub fn bytes(&mut self, n: usize) -> Option<&str>

Consume the next n bytes.

Example

use simple_tokenizer::*;

let mut tokens = "tokens123".as_tokens();

assert_eq!(tokens.bytes(6), Some("tokens"));
assert_eq!(tokens.remainder(), "123");

assert_eq!(tokens.bytes(5), None);
assert_eq!(tokens.remainder(), "123");

source

pub fn bytes_if( &mut self, n: usize, f: impl FnOnce(&str) -> bool ) -> Option<&str>

Consume the next n bytes if they match a predicate.

Example

use simple_tokenizer::*;

let mut tokens = "1231234".as_tokens();

assert_eq!(tokens.bytes_if(3, |s| s.chars().all(char::is_numeric)), Some("123"));
assert_eq!(tokens.remainder(), "1234");

assert_eq!(tokens.bytes_if(5, |s| s.chars().all(char::is_numeric)), None);
assert_eq!(tokens.remainder(), "1234");

source

pub fn chars(&mut self, n: usize) -> Option<&str>

Consume the next n characters. Doesn’t advance if there aren’t enough characters left.

Example

use simple_tokenizer::*;

let mut tokens = "tokens123".as_tokens();

assert_eq!(tokens.chars(6), Some("tokens"));
assert_eq!(tokens.remainder(), "123");

assert_eq!(tokens.chars(5), None);
assert_eq!(tokens.remainder(), "123");

source

pub fn chars_if( &mut self, n: usize, f: impl FnOnce(&str) -> bool ) -> Option<&str>

Consume the next n characters if they match a predicate. Doesn’t advance if there aren’t enough characters left.

Example

use simple_tokenizer::*;

let mut tokens = "1231234".as_tokens();

assert_eq!(tokens.chars_if(3, |s| s.chars().all(char::is_numeric)), Some("123"));
assert_eq!(tokens.remainder(), "1234");

assert_eq!(tokens.chars_if(5, |s| s.chars().all(char::is_numeric)), None);
assert_eq!(tokens.remainder(), "1234");

source

pub fn take_while(&mut self, f: impl FnMut(char) -> bool) -> &str

Consume characters while f returns true. Returns the consumed substring.

Example

use simple_tokenizer::*;

let mut tokens = "12345word".as_tokens();

assert_eq!(tokens.take_while(char::is_numeric), "12345");
assert_eq!(tokens.remainder(), "word");