Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

Implementations

impl Tokenizer

pub fn new(s: impl ToString) -> Self

pub fn add_str_pat<T: ToString>(&mut self, src: T)

pub fn add_str_pattern_array<T: ToString, const N: usize>( &mut self, src: [T; N], )

pub fn add_str_pattern_vec<T: ToString>(&mut self, src: Vec<T>)

pub fn add_pattern_array<T: MatcherTrait + Any, const N: usize>( &mut self, src: [T; N], )

pub fn add_pattern_vec<T: MatcherTrait + Any>(&mut self, src: Vec<T>)

pub fn add_pat<T: MatcherTrait + Any>(&mut self, src: T)

pub fn add_regex_pat(&mut self, src: impl ToString) -> Result<()>

pub fn add_regex_pattern_array<T: ToString, const N: usize>( &mut self, src: [T; N], ) -> Result<()>

pub fn add_regex_pattern_vec<T: ToString>(&mut self, src: Vec<T>) -> Result<()>

pub fn add_ws_pat(&mut self)

pub fn start(&mut self) -> Result<Vec<Token>>