Struct NgramTokenizer 

pub struct NgramTokenizer { /* private fields */ }

Implementations

impl NgramTokenizer

pub fn new( min_gram: usize, max_gram: usize, prefix_only: bool, ) -> Result<Self, NgramError>

pub fn all_ngrams(min_gram: usize, max_gram: usize) -> Result<Self, NgramError>

pub fn prefix_only(min_gram: usize, max_gram: usize) -> Result<Self, NgramError>

Trait Implementations

impl Clone for NgramTokenizer

fn clone(&self) -> NgramTokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for NgramTokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Tokenizer for NgramTokenizer

type TokenStream<'a> = NgramTokenStream<'a>

Stream type emitted by the tokenizer.

fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a>

Create a token stream for the provided text.