pub struct Tokenizer { /* private fields */ }Implementations§
Source§impl Tokenizer
impl Tokenizer
pub fn new(s: &str) -> Self
pub fn get_bytes(&self, range: ByteRange) -> Bytes
pub fn get_bytes_trimmed(&self, range: ByteRange) -> Bytes
pub fn write_range( &self, range: ByteRange, writer: &mut dyn Write, ) -> Result<usize>
pub fn next( &mut self, lexer_result: Option<LexerToken>, bs: &mut ByteStorage, ) -> TokenizerResult<Option<Token>>
Auto Trait Implementations§
impl !Freeze for Tokenizer
impl RefUnwindSafe for Tokenizer
impl Send for Tokenizer
impl Sync for Tokenizer
impl Unpin for Tokenizer
impl UnwindSafe for Tokenizer
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more