pub struct PlainTokenizer {}
Implementations§
Source§impl PlainTokenizer
impl PlainTokenizer
pub fn new() -> PlainTokenizer
pub fn next_token<'a>( &mut self, state: &mut TokenizerState<'a>, chunk: &mut Vec<char>, _token_chunks: &Vec<TokenChunk>, ) -> TokenType
Auto Trait Implementations§
impl Freeze for PlainTokenizer
impl RefUnwindSafe for PlainTokenizer
impl Send for PlainTokenizer
impl Sync for PlainTokenizer
impl Unpin for PlainTokenizer
impl UnwindSafe for PlainTokenizer
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more