jayce

jayce is a tokenizer 🌌

Example

use jayce::{regexify, Tokenizer};
use regex::Regex;

const SOURCE: &str = "Excalibur = 5000$; // Your custom lang";

lazy_static::lazy_static! (
    static ref DUOS: Vec<(&'static str, Regex)> = vec![
        ("price", regexify!(r"^[0-9]+\$")),
        ("semicolon", regexify!(r"^;")),
        ("operator", regexify!(r"^=")),
        ("name", regexify!(r"^[a-zA-Z_]+")),
    ];
);

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut tokenizer = Tokenizer::new(SOURCE, &DUOS);

    while let Some(token) = tokenizer.next()? {
        println!("{:?}", token);
    }

    Ok(())
}

Result

Token { kind: "name", value: "Excalibur", pos: (1, 1) }
Token { kind: "operator", value: "=", pos: (1, 11) }
Token { kind: "price", value: "5000$", pos: (1, 13) }
Token { kind: "semicolon", value: ";", pos: (1, 18) }

Info

next returns a Result with 3 possible values

Ok(Some(token)) If a match is found
Ok(None) Reaching the source ends
Err(error) When an error occurs

tokenize_all returns a Result with 2 possible values

Ok(tokens) If all tokens are found
Err(error) When an error occurs

Note

whitespaces, comments and block comments are skipped for performance reasons

Performances

initialization in 1.83 nanoseconds

tokenization of 19 979 tokens in 3.69 milliseconds