Struct NativeTokenizer 

pub struct NativeTokenizer { /* private fields */ }

Implementations

impl NativeTokenizer

pub fn new( dict_path: String, options: Option<NativeTokenizerOptions>, ) -> Result<Self>

Load a system dictionary and create the native tokenizer.

pub fn from_shared( shared: &NativeSharedDictionary, options: Option<NativeTokenizerOptions>, ) -> Result<Self>

Create a tokenizer from a shared dictionary handle. The ~150 MB trie is shared via Arc (16-byte refcount bump, not a copy).

pub fn add_user_dictionary(&mut self, dict_path: String) -> Result<()>

Add a user dictionary.

pub fn tokenize( &self, bytes: Uint8Array, can_bow: Uint8Array, char_categories: Uint32Array, word_candidate_lengths: Uint32Array, continuous_lengths: Uint32Array, code_point_byte_lengths_flat: Uint8Array, code_point_offsets: Uint32Array, ) -> Result<Vec<NativePathNode>>

The hot path — builds lattice and returns best path nodes.

pub fn get_word_info(&self, word_id: i32) -> Result<NativeWordInfo>

Get word info by packed word ID. Called during TS post-processing.

pub fn get_pos(&self, pos_id: u32) -> Result<Vec<String>>

Get POS tags by ID.

pub fn pos_count(&self) -> u32

Total number of POS entries.

Trait Implementations

impl FromNapiMutRef for NativeTokenizer

unsafe fn from_napi_mut_ref( env: napi_env, napi_val: napi_value, ) -> Result<&'static mut Self>

Safety

impl FromNapiRef for NativeTokenizer

unsafe fn from_napi_ref( env: napi_env, napi_val: napi_value, ) -> Result<&'static Self>

Safety

impl JavaScriptClassExt for NativeTokenizer

fn into_instance<'scope>( self, env: &'scope Env, ) -> Result<ClassInstance<'scope, Self>>

fn into_reference(self, env: Env) -> Result<Reference<Self>>

fn instance_of<'env, V: JsValue<'env>>(env: &Env, value: &V) -> Result<bool>

impl ObjectFinalize for NativeTokenizer

fn finalize(self, env: Env) -> Result<(), Error>

impl ToNapiValue for NativeTokenizer

unsafe fn to_napi_value( env: napi_env, val: NativeTokenizer, ) -> Result<napi_value>

Safety

fn into_unknown(self, env: &Env) -> Result<Unknown<'_>, Error>

impl TypeName for &NativeTokenizer

fn type_name() -> &'static str

fn value_type() -> ValueType

impl TypeName for &mut NativeTokenizer

fn type_name() -> &'static str

fn value_type() -> ValueType

impl TypeName for NativeTokenizer

fn type_name() -> &'static str

fn value_type() -> ValueType

impl ValidateNapiValue for &NativeTokenizer

unsafe fn validate(env: napi_env, napi_val: napi_value) -> Result<napi_value>

Safety

impl ValidateNapiValue for &mut NativeTokenizer

unsafe fn validate(env: napi_env, napi_val: napi_value) -> Result<napi_value>

Safety