test/
test.rs

1use std::time::SystemTime;
2
3use tinytoken::Tokenizer;
4
5const TO_PARSE: &str = include_str!("./test.rs");
6
7fn main() {
8    let tokenizer = Tokenizer::builder()
9        .parse_char_as_string(true)
10        .allow_digit_separator(tinytoken::Choice::Yes('_'))
11        .add_symbols(&['{', '}', '(', ')', ';', '#', ',', '[', ']'])
12        .add_operators(&['+', '-', '*', '%', '/', '&'])
13        .ignore_numbers(true)
14        .build(TO_PARSE);
15    // A little comment 77777.
16    let start_time = SystemTime::now();
17    match tokenizer.tokenize() {
18        Ok(tokens) => {
19            eprintln!(
20                "-> elapsed: {}µs",
21                start_time.elapsed().unwrap().as_micros()
22            );
23            println!("---------\nparsed {} token(s)\n---------", tokens.len());
24            println!("{tokens:?}");
25        }
26        Err(e) => {
27            println!("{e}")
28        }
29    }
30}