Struct JsTokenizer 

pub struct JsTokenizer { /* private fields */ }

Tokenizer for performing morphological analysis.

The tokenizer processes text and returns tokens with their morphological features.

Implementations

impl JsTokenizer

pub fn new( dictionary: &JsDictionary, mode: Option<String>, user_dictionary: Option<&JsUserDictionary>, ) -> Result<Self>

Creates a new tokenizer with the given dictionary and mode.

Arguments

• dictionary - Dictionary to use for tokenization.
• mode - Tokenization mode (“normal” or “decompose”). Default: “normal”.
• user_dictionary - Optional user dictionary for custom words.

pub fn tokenize(&self, text: String) -> Result<Vec<JsToken>>

Tokenizes the given text.

Arguments

• text - Text to tokenize.

Returns

An array of Token objects containing morphological features.

pub fn tokenize_nbest( &self, text: String, n: u32, unique: Option<bool>, cost_threshold: Option<i64>, ) -> Result<Vec<JsNbestResult>>

Tokenizes the given text and returns N-best results.

Arguments

• text - Text to tokenize.
• n - Number of N-best results to return.
• unique - If true, deduplicate results (default: false).
• cost_threshold - Maximum cost difference from the best path (default: undefined).

Returns

An array of NbestResult objects, each containing tokens and their cost.

Trait Implementations

impl FromNapiMutRef for JsTokenizer

unsafe fn from_napi_mut_ref( env: napi_env, napi_val: napi_value, ) -> Result<&'static mut Self>

Safety

impl FromNapiRef for JsTokenizer

unsafe fn from_napi_ref( env: napi_env, napi_val: napi_value, ) -> Result<&'static Self>

Safety

impl JavaScriptClassExt for JsTokenizer

fn into_instance<'scope>( self, env: &'scope Env, ) -> Result<ClassInstance<'scope, Self>>

fn into_reference(self, env: Env) -> Result<Reference<Self>>

fn instance_of<'env, V: JsValue<'env>>(env: &Env, value: &V) -> Result<bool>

impl ObjectFinalize for JsTokenizer

fn finalize(self, env: Env) -> Result<(), Error>

impl ToNapiValue for JsTokenizer

unsafe fn to_napi_value(env: napi_env, val: JsTokenizer) -> Result<napi_value>

Safety

fn into_unknown(self, env: &Env) -> Result<Unknown<'_>, Error>

impl TypeName for &JsTokenizer

fn type_name() -> &'static str

fn value_type() -> ValueType

impl TypeName for &mut JsTokenizer

fn type_name() -> &'static str

fn value_type() -> ValueType

impl TypeName for JsTokenizer

fn type_name() -> &'static str

fn value_type() -> ValueType

impl ValidateNapiValue for &JsTokenizer

unsafe fn validate(env: napi_env, napi_val: napi_value) -> Result<napi_value>

Safety

impl ValidateNapiValue for &mut JsTokenizer

unsafe fn validate(env: napi_env, napi_val: napi_value) -> Result<napi_value>

Safety