Macro plexer::lexer

macro_rules! lexer {
    ($($token:tt $(($($field: ty),+))? {$( $pattern:expr => $build:expr,)+}),* $(,)*) => { ... };
}

Macro to build your own plugin-based lexer.

Usage

lexer!(
    // Ordered by priority
    NAME(optional types, ...) {
        impl Pattern => |value: String| -> Token,
        ...,
    },
    ...,
);

Example

Here is an example for a simple condition statement lexer.

lexer!(
    DELIMITER(char) {
        '{' => |_| Token::DELIMITER('{'),
        '}' => |_| Token::DELIMITER('}'),
    },
    KEYWORD(String) {
        "if" => |v: String| Token::KEYWORD(v),
        "else" => |v: String| Token::KEYWORD(v),
    },
    IDENTIFIER(String) {
        regex!(r"[a-zA-Z_$][a-zA-Z_$0-9]*")
            => |v: String| Token::IDENTIFIER(v),
    },
    WHITESPACE {
        [' ', '\n', '\t'] => |_| Token::WHITESPACE,
    },
);

let mut lex = gen::Lexer::tokenize("if test { one } else { two }");
assert_eq!(lex.next(), Some(Ok(gen::Token::KEYWORD(String::from("if")))));