Struct cang_jie::tokenizer::CangJieTokenizer

pub struct CangJieTokenizer {
    pub worker: Arc<Jieba>,
    pub option: TokenizerOption,
}

Fields

worker: Arc<Jieba>

Separation algorithm provider

option: TokenizerOption

Separation config

Trait Implementations

impl Default for CangJieTokenizer

fn default() -> Self

Returns the "default value" for a type.

impl Clone for CangJieTokenizer

fn clone(&self) -> CangJieTokenizer

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for CangJieTokenizer

fn fmt(&self, f: &mut Formatter) -> Result

Formats the value using the given formatter.

impl<'a> Tokenizer<'a> for CangJieTokenizer

type TokenStreamImpl = CangjieTokenStream<'a>

Type associated to the resulting tokenstream tokenstream.

fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl

Cut text into tokens

fn filter<NewFilter>(
    self,
    new_filter: NewFilter
) -> ChainTokenizer<NewFilter, Self> where
    NewFilter: TokenFilter<Self::TokenStreamImpl>, 

Appends a token filter to the current tokenizer.