Trait Tokenizer

pub trait Tokenizer<'t, F> {
    type Error;

    // Required methods
    fn new(input: &'t str, lexer: F) -> Self;
    fn try_into_vec(self) -> Result<Vec<TokenMatch<'t>>, Self::Error>;
    fn try_collect_adapted<A>(
        self,
    ) -> Result<Vec<TokenMatch<'t>>, ParseErrors<A>>
       where &'t str: Original<A>,
             Self::Error: ErrorAdapter<A, &'t str>;

    // Provided method
    fn into_vec(self) -> Vec<TokenMatch<'t>>
       where Self::Error: Debug,
             Self: Sized { ... }
}

Required Associated Types

type Error

Required Methods

fn new(input: &'t str, lexer: F) -> Self

fn try_into_vec(self) -> Result<Vec<TokenMatch<'t>>, Self::Error>

fn try_collect_adapted<A>(self) -> Result<Vec<TokenMatch<'t>>, ParseErrors<A>>

where &'t str: Original<A>, Self::Error: ErrorAdapter<A, &'t str>,

Provided Methods

fn into_vec(self) -> Vec<TokenMatch<'t>>

where Self::Error: Debug, Self: Sized,

Dyn Compatibility

This trait is not dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety", so this trait is not object safe.

Implementors

impl<'t, F> Tokenizer<'t, F> for kodept_parse::tokenizer::EagerTokenizer<'t, F::Error<'t>, F>

where F: EagerTokensProducer,

type Error = <F as EagerTokensProducer>::Error<'t>

impl<'t, F> Tokenizer<'t, F> for kodept_parse::tokenizer::LazyTokenizer<'t, F>

where F: TokenProducer,

type Error = <F as TokenProducer>::Error<'t>