Struct Lexer 

pub struct Lexer<'a> { /* private fields */ }

A plugin-based lexer. It perform lexical analysis on strings

Implementations

impl<'a> Lexer<'a>

pub fn builder() -> LexerBuilder<'a>

Create a builder

Example

let int = Tokenizer::new(TokenKind::LITERAL("INT"), |s: &str| { s.chars().all(|c| c.is_digit(10)) });
let lex = Lexer::builder().add(int).build();

pub fn tokenize(&self, value: &'a str) -> Result<Vec<Token<'a>>>

Return a vector of Token from the given &str

Errors

Return an Error if a part of the string doesn’t match any token

Example

// Simple math lexer example
let plus = Tokenizer::new(TokenKind::OPERATOR("PLUS"), '+');
let minus = Tokenizer::new(TokenKind::OPERATOR("MINUS"), '-');
let star = Tokenizer::new(TokenKind::OPERATOR("STAR"), '*');
let slash = Tokenizer::new(TokenKind::OPERATOR("SLASH"), '/');
let equal = Tokenizer::new(TokenKind::OPERATOR("EQUAL"), '=');
let number = Tokenizer::new(TokenKind::LITERAL("NUMBER"), |s: &str| {
  let mut dot_seen = false;

  for ch in s.chars() {
    if !ch.is_digit(10) && (ch != '.' || dot_seen) {
      return false;
    } else if ch == '.' {
      dot_seen = true;
    }
  }
   
  true
});
let id_regex = Regex::new(r"[a-zA-Z_$][a-zA-Z_$0-9]*").unwrap();
let id = Tokenizer::new(TokenKind::IDENTIFIER, id_regex);
let whitespace = Tokenizer::new(TokenKind::WHITESPACE("SPACE"), ' ');
let lexer = Lexer::builder()
  .extend(vec![plus, minus, star, slash, equal, number, id, whitespace])
  .build();

assert!(lexer.tokenize("x_4 = 1 + 3 = 8 * 0.25").is_ok());
// Our lexer doesn't handle parenthesis...
assert!(lexer.tokenize("x_4 = (1 + 3)").is_err());