Struct TokenizerBuilder 

pub struct TokenizerBuilder { /* private fields */ }

Implementations

impl TokenizerBuilder

pub fn new() -> LinderaResult<Self>

pub fn from_file(file_path: &Path) -> LinderaResult<Self>

pub fn from_config(config: TokenizerConfig) -> LinderaResult<Self>

pub fn set_segmenter_mode(&mut self, mode: &Mode) -> &mut Self

pub fn set_segmenter_dictionary(&mut self, uri: &str) -> &mut Self

pub fn set_segmenter_user_dictionary(&mut self, uri: &str) -> &mut Self

pub fn set_segmenter_keep_whitespace( &mut self, keep_whitespace: bool, ) -> &mut Self

pub fn set_segmenter_max_grouping_len( &mut self, max_grouping_len: usize, ) -> &mut Self

Caps unknown-word grouping (MeCab’s max-grouping-size semantics); 0 or an absent key means unbounded grouping, the default.

Arguments

• max_grouping_len - Maximum grouped characters beyond the first.

Returns

A mutable reference to self, for chaining.

pub fn set_segmenter_unknown_word_ladder( &mut self, unknown_word_ladder: bool, ) -> &mut Self

Enables/disables the unknown-word length ladder (see Segmenter::unknown_word_ladder; defaults to true).

Arguments

• unknown_word_ladder - Whether to emit the length ladder.

Returns

A mutable reference to self, for chaining.

pub fn set_segmenter_space_penalty( &mut self, space_penalty: Option<&SpacePenaltyConfig>, ) -> &mut Self

Sets the left-space penalty rules (see Segmenter::space_penalty). Pass None to turn the penalty off (space_penalty: false), which also overrides the rules the dictionary ships and applies by default; leave the key unset to keep those.

Arguments

• space_penalty - The rules, or None to disable the penalty.

Returns

A mutable reference to self, for chaining.

pub fn set_segmenter_space_penalty_from_dictionary( &mut self, enabled: bool, ) -> &mut Self

Enables the left-space penalty with the rules the dictionary ships in its metadata.json (see Segmenter::space_penalty_from_dictionary), i.e. space_penalty: true in the config; false turns it off. The shipped rules are already the default when the key is absent, so true mainly makes a dictionary that ships none fail loudly.

Arguments

• enabled - Whether to use the dictionary’s rules.

Returns

A mutable reference to self, for chaining.

pub fn reset_segmenter_space_penalty(&mut self) -> &mut Self

Removes any left-space penalty setting, so the Segmenter falls back to its default: the rules the dictionary ships, if any. This undoes Self::set_segmenter_space_penalty and Self::set_segmenter_space_penalty_from_dictionary.

Returns

A mutable reference to self, for chaining.

pub fn set_segmenter_use_mmap(&mut self, use_mmap: bool) -> &mut Self

Set whether to route filesystem-loaded dictionaries through memory-mapped reads. Ignored for embedded:// dictionaries.

Arguments

• use_mmap - Whether to request memory-mapped dictionary loading.

Returns

A mutable reference to self, for chaining.

pub fn append_character_filter(&mut self, kind: &str, args: &Value) -> &mut Self

pub fn append_token_filter(&mut self, kind: &str, args: &Value) -> &mut Self

pub fn build(&self) -> LinderaResult<Tokenizer>

Trait Implementations

impl Debug for TokenizerBuilder

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.