Struct pattern_lexer::token::Tokenizer

pub struct Tokenizer<'a> { /* private fields */ }

Produce Token that match a Pattern

Implementations

impl<'a> Tokenizer<'a>

pub fn new<P: Pattern<'a> + 'static>(kind: TokenKind<'a>, pat: P) -> Self

Create a Tokenizer

Example

use regex::Regex;
use lexer::token::{TokenKind, Tokenizer};
// Create a token that matches variable names
let id_regex = Regex::new(r"[a-zA-Z_$][a-zA-Z_$0-9]*").unwrap();
let id = Tokenizer::new(TokenKind::IDENTIFIER, id_regex);

pub fn tokenize(&self, value: &'a str) -> Option<Token<'a>>

Return a Token from the given string if it find a match

Example

use lexer::token::{TokenKind, Token, Tokenizer};

let kind = TokenKind::KEYWORD("FUNC");
let function = Tokenizer::new(kind.clone(), "fn");
assert!(function.tokenize("test").is_none());
assert_eq!(function.tokenize("fn"), Some(Token::new(kind, "fn")));

Trait Implementations

impl<'a> Debug for Tokenizer<'a>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'a> PartialEq for Tokenizer<'a>

fn eq(&self, other: &Self) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl<'a> PartialOrd for Tokenizer<'a>

fn partial_cmp(&self, other: &Self) -> Option<Ordering>

This method returns an ordering between self and other values if one exists.

fn lt(&self, other: &Rhs) -> bool

This method tests less than (for self and other) and is used by the < operator.

fn le(&self, other: &Rhs) -> bool

This method tests less than or equal to (for self and other) and is used by the <= operator.

fn gt(&self, other: &Rhs) -> bool

This method tests greater than (for self and other) and is used by the > operator.

fn ge(&self, other: &Rhs) -> bool

This method tests greater than or equal to (for self and other) and is used by the >= operator.