Struct TokenizerBuilder

pub struct TokenizerBuilder { /* private fields */ }

A builder struct for creating a TokenizerConfig instance with customized options

Implementations

impl TokenizerBuilder

pub fn new() -> TokenizerBuilder

Creates a default TokenizerBuilder

pub fn parse_char_as_string(self, set_to: bool) -> Self

Configures character parsing behavior

Examples found in repository

examples/test.rs (line 9)

fn main() {
    let tokenizer = Tokenizer::builder()
        .parse_char_as_string(true)
        .allow_digit_separator(tinytoken::Choice::Yes('_'))
        .add_symbols(&['{', '}', '(', ')', ';', '#', ',', '[', ']'])
        .add_operators(&['+', '-', '*', '%', '/', '&'])
        .build(TO_PARSE);
    let start_time = SystemTime::now();
    match tokenizer.tokenize() {
        Ok(tokens) => {
            eprintln!(
                "-> elapsed: {}µs",
                start_time.elapsed().unwrap().as_micros()
            );
            println!("---------\nparsed {} token(s)\n---------", tokens.len());
            println!("{tokens:?}");
        }
        Err(e) => {
            println!("{e}")
        }
    }
}

pub fn allow_digit_separator(self, choice: Choice<char>) -> Self

Sets the digit separator

Examples found in repository

examples/test.rs (line 10)

fn main() {
    let tokenizer = Tokenizer::builder()
        .parse_char_as_string(true)
        .allow_digit_separator(tinytoken::Choice::Yes('_'))
        .add_symbols(&['{', '}', '(', ')', ';', '#', ',', '[', ']'])
        .add_operators(&['+', '-', '*', '%', '/', '&'])
        .build(TO_PARSE);
    let start_time = SystemTime::now();
    match tokenizer.tokenize() {
        Ok(tokens) => {
            eprintln!(
                "-> elapsed: {}µs",
                start_time.elapsed().unwrap().as_micros()
            );
            println!("---------\nparsed {} token(s)\n---------", tokens.len());
            println!("{tokens:?}");
        }
        Err(e) => {
            println!("{e}")
        }
    }
}

pub fn add_symbol(self, sym: char) -> Self

Adds a symbol character

pub fn add_symbols(self, syms: &[char]) -> Self

Adds multiple symbol characters

Examples found in repository

examples/test.rs (line 11)

fn main() {
    let tokenizer = Tokenizer::builder()
        .parse_char_as_string(true)
        .allow_digit_separator(tinytoken::Choice::Yes('_'))
        .add_symbols(&['{', '}', '(', ')', ';', '#', ',', '[', ']'])
        .add_operators(&['+', '-', '*', '%', '/', '&'])
        .build(TO_PARSE);
    let start_time = SystemTime::now();
    match tokenizer.tokenize() {
        Ok(tokens) => {
            eprintln!(
                "-> elapsed: {}µs",
                start_time.elapsed().unwrap().as_micros()
            );
            println!("---------\nparsed {} token(s)\n---------", tokens.len());
            println!("{tokens:?}");
        }
        Err(e) => {
            println!("{e}")
        }
    }
}

pub fn add_operator(self, op: char) -> Self

Adds an operator character

pub fn add_operators(self, ops: &[char]) -> Self

Adds multiple operator characters

Examples found in repository

examples/test.rs (line 12)

fn main() {
    let tokenizer = Tokenizer::builder()
        .parse_char_as_string(true)
        .allow_digit_separator(tinytoken::Choice::Yes('_'))
        .add_symbols(&['{', '}', '(', ')', ';', '#', ',', '[', ']'])
        .add_operators(&['+', '-', '*', '%', '/', '&'])
        .build(TO_PARSE);
    let start_time = SystemTime::now();
    match tokenizer.tokenize() {
        Ok(tokens) => {
            eprintln!(
                "-> elapsed: {}µs",
                start_time.elapsed().unwrap().as_micros()
            );
            println!("---------\nparsed {} token(s)\n---------", tokens.len());
            println!("{tokens:?}");
        }
        Err(e) => {
            println!("{e}")
        }
    }
}

pub fn build<T>(self, with_input: T) -> Tokenizer

where T: ToString,

Constructs a Tokenizer with the specified input and configuration.

Examples found in repository

examples/test.rs (line 13)

fn main() {
    let tokenizer = Tokenizer::builder()
        .parse_char_as_string(true)
        .allow_digit_separator(tinytoken::Choice::Yes('_'))
        .add_symbols(&['{', '}', '(', ')', ';', '#', ',', '[', ']'])
        .add_operators(&['+', '-', '*', '%', '/', '&'])
        .build(TO_PARSE);
    let start_time = SystemTime::now();
    match tokenizer.tokenize() {
        Ok(tokens) => {
            eprintln!(
                "-> elapsed: {}µs",
                start_time.elapsed().unwrap().as_micros()
            );
            println!("---------\nparsed {} token(s)\n---------", tokens.len());
            println!("{tokens:?}");
        }
        Err(e) => {
            println!("{e}")
        }
    }
}

Trait Implementations

impl Clone for TokenizerBuilder

fn clone(&self) -> TokenizerBuilder

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenizerBuilder

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.