Struct Tokenizer

pub struct Tokenizer {
    pub lines: Vec<Vec<char>>,
    /* private fields */
}

Primary struct for tokenizing an input string, with methods for parsing and generating tokens

Fields

lines: Vec<Vec<char>>

Implementations

impl Tokenizer

pub fn builder() -> TokenizerBuilder

Creates a TokenizerBuilder instance for configuring and initializing the tokenizer

Examples found in repository

examples/test.rs (line 8)

fn main() {
    let tokenizer = Tokenizer::builder()
        .parse_char_as_string(true)
        .allow_digit_separator(tinytoken::Choice::Yes('_'))
        .add_symbols(&['{', '}', '(', ')', ';', '#', ',', '[', ']'])
        .add_operators(&['+', '-', '*', '%', '/', '&'])
        .build(TO_PARSE);
    let start_time = SystemTime::now();
    match tokenizer.tokenize() {
        Ok(tokens) => {
            eprintln!(
                "-> elapsed: {}µs",
                start_time.elapsed().unwrap().as_micros()
            );
            println!("---------\nparsed {} token(s)\n---------", tokens.len());
            println!("{tokens:?}");
        }
        Err(e) => {
            println!("{e}")
        }
    }
}

pub fn new<T>(input: T, config: TokenizerConfig) -> Self

where T: ToString,

Initializes the tokenizer with input text and a configuration

pub fn tokenize(self) -> Result<Vec<Token>, TokenizationError>

Tokenizes the input and returns a list of Tokens or a TokenizationError if parsing fails

Examples found in repository

examples/test.rs (line 15)

fn main() {
    let tokenizer = Tokenizer::builder()
        .parse_char_as_string(true)
        .allow_digit_separator(tinytoken::Choice::Yes('_'))
        .add_symbols(&['{', '}', '(', ')', ';', '#', ',', '[', ']'])
        .add_operators(&['+', '-', '*', '%', '/', '&'])
        .build(TO_PARSE);
    let start_time = SystemTime::now();
    match tokenizer.tokenize() {
        Ok(tokens) => {
            eprintln!(
                "-> elapsed: {}µs",
                start_time.elapsed().unwrap().as_micros()
            );
            println!("---------\nparsed {} token(s)\n---------", tokens.len());
            println!("{tokens:?}");
        }
        Err(e) => {
            println!("{e}")
        }
    }
}