sentience-tokenize 0.2.3

Tiny Rust zero-dep tokenizer (ident, number, string, parens, operators, keywords).
Documentation 
1
2
3
4
5
6
7
8
9

use sentience_tokenize::tokenize;

fn main() {
    let code = r#"let rule greet(n) = "hi, " + n"#;
    let toks = tokenize(code).unwrap();
    for t in toks {
        println!("{:?} @{}..{}", t.kind, t.span.start, t.span.end);
    }
}