Crate regex_lexer

A regex-based lexer (tokenizer).

use regex_lexer::{LexerBuilder, Token};

#[derive(Debug, PartialEq, Eq, Clone, Copy)]
enum Tok {
    Num,
    Add,
    Sub,
    Mul,
    Div,
    Open,
    Close,
}

let lexer = LexerBuilder::new()
    .token(r"[0-9]+", Tok::Num)
    .token(r"\+", Tok::Add)
    .token(r"-", Tok::Sub)
    .token(r"\*", Tok::Mul)
    .token(r"/", Tok::Div)
    .token(r"\(", Tok::Open)
    .token(r"\)", Tok::Close)
    .ignore(r"\s+")
    .build()?;

let source = "(1 + 2) * 3";
assert_eq!(
    lexer.tokens(source).collect::<Vec<_>>(),
    vec![
        Token { kind: Tok::Open, span: 0..1, text: "(" }, 
        Token { kind: Tok::Num, span: 1..2, text: "1" }, 
        Token { kind: Tok::Add, span: 3..4, text: "+" }, 
        Token { kind: Tok::Num, span: 5..6, text: "2" },
        Token { kind: Tok::Close, span: 6..7, text: ")" },
        Token { kind: Tok::Mul, span: 8..9, text: "*" },
        Token { kind: Tok::Num, span: 10..11, text: "3" },
    ],
);

Structs

Lexer

A regex-based lexer.

LexerBuilder

Builder struct for Lexer.

Token

A token returned by the lexer.

Tokens

The type returned by Lexer::tokens.

Enums

Error

An error that occurred during parsing or compiling a regular expression.