Trait Tokenizer

pub trait Tokenizer<'a, 'b, T: ValidMatcher> {
    // Required method
    fn tokenize(
        &'a self,
        data: &'b str,
    ) -> TokenGenerator<'a, 'b, T, T::TokenType>;
}

Required Methods

fn tokenize(&'a self, data: &'b str) -> TokenGenerator<'a, 'b, T, T::TokenType>

Prepare an iterator to extract tokens from a string

tokenizer! {
    Test

    r"[a-zA-Z]\w*" => Identifier
    r"\d+" => Number
    r"\s+" => _
}

let tokenizer = Test::new();
let query = "Identifier  11";

let mut tokens = tokenizer.tokenize(query);
let token = tokens.next().unwrap();

Implementors

impl<'a, 'b, T> Tokenizer<'a, 'b, T> for T

where T: ValidMatcher, 'a: 'b,