Trait text_tokenizer::IntoTokenizer

pub trait IntoTokenizer {
    type IntoTokens: Tokenizer;

    // Required methods
    fn into_tokens(self) -> Self::IntoTokens;
    fn into_tokens_with_options(
        self,
        options: BTreeSet<TokenizerOptions>
    ) -> Self::IntoTokens;
    fn basic_tokens(self) -> Self::IntoTokens;
    fn complex_tokens(self) -> Self::IntoTokens;
}

Required Associated Types

type IntoTokens: Tokenizer

Required Methods

fn into_tokens(self) -> Self::IntoTokens

fn into_tokens_with_options( self, options: BTreeSet<TokenizerOptions> ) -> Self::IntoTokens

fn basic_tokens(self) -> Self::IntoTokens

fn complex_tokens(self) -> Self::IntoTokens

Implementations on Foreign Types

impl<'t> IntoTokenizer for &'t str

type IntoTokens = Tokens<'t>

fn into_tokens(self) -> Self::IntoTokens

fn into_tokens_with_options( self, options: BTreeSet<TokenizerOptions> ) -> Self::IntoTokens

fn complex_tokens(self) -> Self::IntoTokens

fn basic_tokens(self) -> Self::IntoTokens

Implementors