tinytoken 0.1.4

Library for tokenizing text into words, numbers, symbols, and more, with customizable parsing options.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

use std::time::SystemTime;

use tinytoken::Tokenizer;

const TO_PARSE: &str = include_str!("./test.rs");

fn main() {
    let tokenizer = Tokenizer::builder()
        .parse_char_as_string(true)
        .allow_digit_separator(tinytoken::Choice::Yes('_'))
        .add_symbols(&['{', '}', '(', ')', ';', '#', ',', '[', ']'])
        .add_operators(&['+', '-', '*', '%', '/', '&'])
        .ignore_numbers(true)
        .build(TO_PARSE);
    // A little comment 77777.
    let start_time = SystemTime::now();
    match tokenizer.tokenize() {
        Ok(tokens) => {
            eprintln!(
                "-> elapsed: {}µs",
                start_time.elapsed().unwrap().as_micros()
            );
            println!("---------\nparsed {} token(s)\n---------", tokens.len());
            println!("{tokens:?}");
        }
        Err(e) => {
            println!("{e}")
        }
    }
}