pub struct Tokenizer<T: Grinder> { /* private fields */ }Expand description
A grinder that combines character bundles into lexical tokens. This is the last stage of lexical analysis.
Implementations
sourceimpl<T: Grinder> Tokenizer<T> where
T: Grinder<Item = Option<Spanned<Bundle>>, Error = DiagBuilder2>,
impl<T: Grinder> Tokenizer<T> where
T: Grinder<Item = Option<Spanned<Bundle>>, Error = DiagBuilder2>,
Trait Implementations
sourceimpl<T> Grinder for Tokenizer<T> where
T: Grinder<Item = Option<Spanned<Bundle>>, Error = DiagBuilder2>,
impl<T> Grinder for Tokenizer<T> where
T: Grinder<Item = Option<Spanned<Bundle>>, Error = DiagBuilder2>,
type Item = Option<Spanned<Token>>
type Error = DiagBuilder2
fn emit(&mut self, err: Self::Error)
fn next(&mut self) -> Self::Item
fn vent<F, E>(self, f: F) -> Vent<Self, F, E> where
F: Fn(E),
fn unwrap(self) -> Unwrap<Self>
fn map_err<F, E>(self, f: F) -> MapErrGrinder<Self, F, E> where
F: Fn(E) -> Self::Error,
fn peekable(self) -> Peekable<Self>
fn lookaheadable(self) -> Lookahead<Self>
fn filter<F>(self, f: F) -> Filter<Self, F> where
F: Fn(&<Self::Item as Chisel>::Value) -> bool,
Auto Trait Implementations
impl<T> RefUnwindSafe for Tokenizer<T> where
T: RefUnwindSafe,
<T as Grinder>::Item: RefUnwindSafe,
impl<T> Send for Tokenizer<T> where
T: Send,
<T as Grinder>::Item: Send,
impl<T> Sync for Tokenizer<T> where
T: Sync,
<T as Grinder>::Item: Sync,
impl<T> Unpin for Tokenizer<T> where
T: Unpin,
<T as Grinder>::Item: Unpin,
impl<T> UnwindSafe for Tokenizer<T> where
T: UnwindSafe,
<T as Grinder>::Item: UnwindSafe,
Blanket Implementations
sourceimpl<T> BorrowMut<T> for T where
T: ?Sized,
impl<T> BorrowMut<T> for T where
T: ?Sized,
const: unstable · sourcepub fn borrow_mut(&mut self) -> &mut T
pub fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more