Struct Tokenizer 

pub struct Tokenizer<T: Grinder> { /* private fields */ }

A grinder that combines character bundles into lexical tokens. This is the last stage of lexical analysis.

Implementations

impl<T> Tokenizer<T>

where T: Grinder<Item = Option<Spanned<Bundle>>, Error = DiagBuilder2> + Grinder,

pub fn new<I>(inner: I) -> Tokenizer<T>

where I: Into<Lookahead<T>>,

Create a new bundler.

Trait Implementations

impl<T> Grinder for Tokenizer<T>

where T: Grinder<Item = Option<Spanned<Bundle>>, Error = DiagBuilder2>,

type Item = Option<Spanned<Token>>

type Error = DiagBuilder2

fn emit(&mut self, err: Self::Error)

fn next(&mut self) -> Self::Item

fn vent<F, E>(self, f: F) -> Vent<Self, F, E>

where Self: Sized, F: Fn(E),

fn unwrap(self) -> Unwrap<Self>

where Self: Sized,

fn map_err<F, E>(self, f: F) -> MapErrGrinder<Self, F, E>

where Self: Sized, F: Fn(E) -> Self::Error,

fn peekable(self) -> Peekable<Self>

where Self: Sized,

fn lookaheadable(self) -> Lookahead<Self>

where Self: Sized,

fn filter<F>(self, f: F) -> Filter<Self, F>

where Self: Sized, F: Fn(&<Self::Item as Chisel>::Value) -> bool,