Struct JsTokenizerBuilder 

pub struct JsTokenizerBuilder { /* private fields */ }

Builder for creating a Tokenizer with custom configuration.

The builder pattern allows for fluent configuration of tokenizer parameters including dictionaries, modes, and filter pipelines.

Implementations

impl JsTokenizerBuilder

pub fn new() -> Result<Self>

Creates a new TokenizerBuilder with default configuration.

pub fn from_file(&self, file_path: String) -> Result<JsTokenizerBuilder>

Loads configuration from a JSON file.

Arguments

• file_path - Path to the configuration file.

Returns

A new TokenizerBuilder with the loaded configuration.

pub fn set_mode(&mut self, mode: String) -> Result<()>

Sets the tokenization mode.

Arguments

• mode - Mode string (“normal” or “decompose”).

pub fn set_dictionary(&mut self, path: String)

Sets the dictionary path or URI.

Arguments

• path - Path to the dictionary directory or embedded URI (e.g. “embedded://ipadic”).

pub fn set_user_dictionary(&mut self, uri: String)

Sets the user dictionary URI.

Arguments

• uri - URI to the user dictionary.

pub fn set_keep_whitespace(&mut self, keep_whitespace: bool)

Sets whether to keep whitespace in tokenization results.

Arguments

• keep_whitespace - If true, whitespace tokens will be included in results.

pub fn append_character_filter( &mut self, kind: String, args: Option<Value>, ) -> Result<()>

Appends a character filter to the preprocessing pipeline.

Arguments

• kind - Type of character filter to add (e.g. “unicode_normalize”, “mapping”).
• args - Optional filter arguments as a JSON-compatible object.

pub fn append_token_filter( &mut self, kind: String, args: Option<Value>, ) -> Result<()>

Appends a token filter to the postprocessing pipeline.

Arguments

• kind - Type of token filter to add (e.g. “lowercase”, “japanese_stop_tags”).
• args - Optional filter arguments as a JSON-compatible object.

pub fn build(&self) -> Result<JsTokenizer>

Builds the tokenizer with the configured settings.

Returns

A configured Tokenizer instance ready for use.

Trait Implementations

impl FromNapiMutRef for JsTokenizerBuilder

unsafe fn from_napi_mut_ref( env: napi_env, napi_val: napi_value, ) -> Result<&'static mut Self>

Safety

impl FromNapiRef for JsTokenizerBuilder

unsafe fn from_napi_ref( env: napi_env, napi_val: napi_value, ) -> Result<&'static Self>

Safety

impl JavaScriptClassExt for JsTokenizerBuilder

fn into_instance<'scope>( self, env: &'scope Env, ) -> Result<ClassInstance<'scope, Self>>

fn into_reference(self, env: Env) -> Result<Reference<Self>>

fn instance_of<'env, V: JsValue<'env>>(env: &Env, value: &V) -> Result<bool>

impl ObjectFinalize for JsTokenizerBuilder

fn finalize(self, env: Env) -> Result<(), Error>

impl ToNapiValue for JsTokenizerBuilder

unsafe fn to_napi_value( env: napi_env, val: JsTokenizerBuilder, ) -> Result<napi_value>

Safety

fn into_unknown(self, env: &Env) -> Result<Unknown<'_>, Error>

impl TypeName for &JsTokenizerBuilder

fn type_name() -> &'static str

fn value_type() -> ValueType

impl TypeName for &mut JsTokenizerBuilder

fn type_name() -> &'static str

fn value_type() -> ValueType

impl TypeName for JsTokenizerBuilder

fn type_name() -> &'static str

fn value_type() -> ValueType

impl ValidateNapiValue for &JsTokenizerBuilder

unsafe fn validate(env: napi_env, napi_val: napi_value) -> Result<napi_value>

Safety

impl ValidateNapiValue for &mut JsTokenizerBuilder

unsafe fn validate(env: napi_env, napi_val: napi_value) -> Result<napi_value>

Safety