Crate regex_tokenizer

source ·
Expand description

A regex-based tokenizer.

use regex_tokenizer::tokenizer;

tokenizer! {
    Test

    r"[a-zA-Z]\w*" => Identifier
    r"\d+" => Number
    r"\s+" => _
}

let tokenizer = Test::new();
let query = "Identifier  11";

let mut tokens = tokenizer.tokenize(query);
let token = tokens.next().unwrap();

assert_eq!(token.position, 0);
assert_eq!(token.value, "Identifier");
assert_eq!(token.type_, Test_types::Identifier);

let token = tokens.next().unwrap();

assert_eq!(token.position, 12);
assert_eq!(token.value, "11");
assert_eq!(token.type_, Test_types::Number);

Macros

  • tokenizer
    Parse a tokenizer DSL, where three kinds of statements are admitted:

Structs

  • Matcher
    A matcher capable of extracting tokens and associating them to a class of tokens

Traits