Struct moore_vhdl_syntax::lexer::tokenizer::Tokenizer

pub struct Tokenizer<T: Grinder> { /* fields omitted */ }

A grinder that combines character bundles into lexical tokens. This is the last stage of lexical analysis.

Methods

impl<T: Grinder> Tokenizer<T> where
    T: Grinder<Item = Option<Spanned<Bundle>>, Error = DiagBuilder2>, 

pub fn new<I>(inner: I) -> Tokenizer<T> where
    I: Into<Lookahead<T>>, 

Create a new bundler.

Trait Implementations

impl<T> Grinder for Tokenizer<T> where
    T: Grinder<Item = Option<Spanned<Bundle>>, Error = DiagBuilder2>, 

type Item = Option<Spanned<Token>>

type Error = DiagBuilder2

fn emit(&mut self, err: Self::Error)

fn next(&mut self) -> Self::Item

fn vent<F, E>(self, f: F) -> Vent<Self, F, E> where
    F: Fn(E), 

fn unwrap(self) -> Unwrap<Self>

fn map_err<F, E>(self, f: F) -> MapErrGrinder<Self, F, E> where
    F: Fn(E) -> Self::Error, 

fn peekable(self) -> Peekable<Self>

fn lookaheadable(self) -> Lookahead<Self>

fn filter<F>(self, f: F) -> Filter<Self, F> where
    F: Fn(&<Self::Item as Chisel>::Value) -> bool,