jayce

jayce is a tokenizer 🌌

Example

use jayce::{regexify, Tokenizer};
use regex::Regex;

const SOURCE: &str = "Excalibur = 5000$; // Your custom lang";

lazy_static::lazy_static! (
    static ref DUOS: Vec<(&'static str, Regex)> = vec![
        ("price", regexify!(r"^[0-9]+\$")),
        ("semicolon", regexify!(r"^;")),
        ("operator", regexify!(r"^=")),
        ("name", regexify!(r"^[a-zA-Z_]+")),
    ];
);

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut tokenizer = Tokenizer::new(SOURCE, &DUOS);

    while let Some(token) = tokenizer.next()? {
        println!("{:?}", token);
    }

    Ok(())
}

Result

Token { kind: "name", value: "Excalibur", pos: (1, 1) }
Token { kind: "operator", value: "=", pos: (1, 11) }
Token { kind: "price", value: "5000$", pos: (1, 13) }
Token { kind: "semicolon", value: ";", pos: (1, 18) }

Info

functions available are next and tokenize_all

the Result has 3 variants

Ok(Some(token)) If a match is found
Ok(None) Reaching the source ends
Err(error) When nothing matches

Note

whitespaces, comments and block comments are skipped for performance reasons

Performances

initialization in 1.83 nanoseconds

tokenization of 19 979 tokens in 3.69 milliseconds