lindera_nodejs/
token.rs

1//! Token representation for morphological analysis results.
2//!
3//! This module provides the Token class that wraps morphological analysis results
4//! and exposes token properties to JavaScript.
5
6use lindera::token::Token;
7
8/// A morphological token.
9///
10/// Represents a single token from morphological analysis with its surface form,
11/// position information, and morphological details.
12#[napi(js_name = "Token")]
13#[derive(Clone)]
14pub struct JsToken {
15    /// Surface form of the token.
16    surface: String,
17    /// Start byte position in the original text.
18    byte_start: u32,
19    /// End byte position in the original text.
20    byte_end: u32,
21    /// Position index of the token.
22    position: u32,
23    /// Word ID in the dictionary.
24    word_id: u32,
25    /// Whether this token is an unknown word.
26    is_unknown: bool,
27    /// Morphological details of the token.
28    details: Vec<String>,
29}
30
31#[napi]
32impl JsToken {
33    /// Surface form of the token.
34    #[napi(getter)]
35    pub fn surface(&self) -> String {
36        self.surface.clone()
37    }
38
39    /// Start byte position in the original text.
40    #[napi(getter)]
41    pub fn byte_start(&self) -> u32 {
42        self.byte_start
43    }
44
45    /// End byte position in the original text.
46    #[napi(getter)]
47    pub fn byte_end(&self) -> u32 {
48        self.byte_end
49    }
50
51    /// Position index of the token.
52    #[napi(getter)]
53    pub fn position(&self) -> u32 {
54        self.position
55    }
56
57    /// Word ID in the dictionary.
58    #[napi(getter)]
59    pub fn word_id(&self) -> u32 {
60        self.word_id
61    }
62
63    /// Whether this token is an unknown word (not found in the dictionary).
64    #[napi(getter)]
65    pub fn is_unknown(&self) -> bool {
66        self.is_unknown
67    }
68
69    /// Morphological details of the token (part of speech, reading, etc.).
70    #[napi(getter)]
71    pub fn details(&self) -> Vec<String> {
72        self.details.clone()
73    }
74
75    /// Returns the detail string at the specified index.
76    ///
77    /// # Arguments
78    ///
79    /// * `index` - Zero-based index into the details array.
80    ///
81    /// # Returns
82    ///
83    /// The detail string if found, or `null` if the index is out of range.
84    #[napi]
85    pub fn get_detail(&self, index: u32) -> Option<String> {
86        self.details.get(index as usize).cloned()
87    }
88}
89
90impl JsToken {
91    /// Creates a JsToken from a lindera Token.
92    ///
93    /// # Arguments
94    ///
95    /// * `token` - The lindera Token to convert.
96    ///
97    /// # Returns
98    ///
99    /// A new JsToken instance.
100    pub fn from_token(token: Token) -> Self {
101        Self::from_view(lindera_binding_core::TokenView::from_token(token))
102    }
103
104    /// Creates a JsToken from a binding-core `TokenView`.
105    ///
106    /// # Arguments
107    ///
108    /// * `view` - The token view produced by the binding-core tokenizer.
109    ///
110    /// # Returns
111    ///
112    /// A new JsToken instance.
113    pub fn from_view(view: lindera_binding_core::TokenView) -> Self {
114        Self {
115            surface: view.surface,
116            byte_start: view.byte_start as u32,
117            byte_end: view.byte_end as u32,
118            position: view.position as u32,
119            word_id: view.word_id,
120            is_unknown: view.is_unknown,
121            details: view.details,
122        }
123    }
124}
125
126/// N-best tokenization result.
127///
128/// Contains a list of tokens and their total path cost.
129#[napi(js_name = "NbestResult")]
130pub struct JsNbestResult {
131    /// Tokens in this result.
132    tokens: Vec<JsToken>,
133    /// Total path cost of this tokenization.
134    cost: i64,
135}
136
137#[napi]
138impl JsNbestResult {
139    /// Tokens in this result.
140    #[napi(getter)]
141    pub fn tokens(&self) -> Vec<JsToken> {
142        self.tokens.clone()
143    }
144
145    /// Total path cost of this tokenization.
146    #[napi(getter)]
147    pub fn cost(&self) -> i64 {
148        self.cost
149    }
150}
151
152impl JsNbestResult {
153    /// Creates a new JsNbestResult.
154    ///
155    /// # Arguments
156    ///
157    /// * `tokens` - The tokens in this result.
158    /// * `cost` - The total path cost.
159    ///
160    /// # Returns
161    ///
162    /// A new JsNbestResult instance.
163    pub fn new(tokens: Vec<JsToken>, cost: i64) -> Self {
164        Self { tokens, cost }
165    }
166}
lindera_nodejs/token.rs

lindera_nodejs/
token.rs
