regex-tokenizer 0.1.1

A regex tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

use regex_tokenizer::tokenizer;

tokenizer! {
    Test

    r"[a-zA-Z]\w*" => Identifier
    r"\d+" => Number
    r"\s+" => _
}

fn main() {
    let query = "Identifier  11";
    let tokenizer = Test::new();

    let mut tokens = tokenizer.tokenize(query);

    let token = tokens.next().unwrap();

    assert_eq!(token.position, 0);
    assert_eq!(token.value, "Identifier");
    assert_eq!(token.type_, Test_types::Identifier);

    let token = tokens.next().unwrap();

    assert_eq!(token.position, 12);
    assert_eq!(token.value, "11");
    assert_eq!(token.type_, Test_types::Number);
}