pub struct Tokenizer<'a> { /* private fields */ }Expand description
Tokenizer state on the data
Implementations
sourceimpl<'a> Tokenizer<'a>
impl<'a> Tokenizer<'a>
sourcepub fn new_with_config(data: &'a str, cfg: TokenizerConfig) -> Self
pub fn new_with_config(data: &'a str, cfg: TokenizerConfig) -> Self
Create a new tokenizer from the data stream with an associated config
sourcepub fn next(&mut self) -> Result<Option<SpannedToken<'a>>, TokenError>
pub fn next(&mut self) -> Result<Option<SpannedToken<'a>>, TokenError>
Return the next token, or none if reach the end of stream
Auto Trait Implementations
impl<'a> RefUnwindSafe for Tokenizer<'a>
impl<'a> Send for Tokenizer<'a>
impl<'a> Sync for Tokenizer<'a>
impl<'a> Unpin for Tokenizer<'a>
impl<'a> UnwindSafe for Tokenizer<'a>
Blanket Implementations
sourceimpl<T> BorrowMut<T> for T where
T: ?Sized,
impl<T> BorrowMut<T> for T where
T: ?Sized,
const: unstable · sourcefn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more