lindera_ruby/
token.rs

1//! Token representation for morphological analysis results.
2//!
3//! This module wraps Lindera tokens for use in Ruby.
4
5use magnus::prelude::*;
6use magnus::{Error, Ruby, method};
7
8use lindera::token::Token;
9
10/// Token object wrapping the Rust Token data.
11///
12/// This class provides access to token fields and details.
13#[magnus::wrap(class = "Lindera::Token", free_immediately, size)]
14pub struct RbToken {
15    /// Surface form of the token.
16    surface: String,
17    /// Start byte position in the original text.
18    byte_start: usize,
19    /// End byte position in the original text.
20    byte_end: usize,
21    /// Position index of the token.
22    position: usize,
23    /// Word ID in the dictionary.
24    word_id: u32,
25    /// Whether this token is an unknown word.
26    is_unknown: bool,
27    /// Morphological details of the token.
28    details: Vec<String>,
29}
30
31impl RbToken {
32    /// Creates a new `RbToken` from a Lindera `Token`.
33    ///
34    /// # Arguments
35    ///
36    /// * `token` - Lindera token to convert.
37    ///
38    /// # Returns
39    ///
40    /// A new `RbToken` instance.
41    pub fn from_token(token: Token) -> Self {
42        Self::from_view(lindera_binding_core::TokenView::from_token(token))
43    }
44
45    /// Creates a new `RbToken` from a binding-core `TokenView`.
46    ///
47    /// # Arguments
48    ///
49    /// * `view` - Token view produced by the binding-core tokenizer.
50    ///
51    /// # Returns
52    ///
53    /// A new `RbToken` instance.
54    pub fn from_view(view: lindera_binding_core::TokenView) -> Self {
55        Self {
56            surface: view.surface,
57            byte_start: view.byte_start,
58            byte_end: view.byte_end,
59            position: view.position,
60            word_id: view.word_id,
61            is_unknown: view.is_unknown,
62            details: view.details,
63        }
64    }
65
66    /// Returns the surface form of the token.
67    fn surface(&self) -> String {
68        self.surface.clone()
69    }
70
71    /// Returns the start byte position.
72    fn byte_start(&self) -> usize {
73        self.byte_start
74    }
75
76    /// Returns the end byte position.
77    fn byte_end(&self) -> usize {
78        self.byte_end
79    }
80
81    /// Returns the position index.
82    fn position(&self) -> usize {
83        self.position
84    }
85
86    /// Returns the word ID.
87    fn word_id(&self) -> u32 {
88        self.word_id
89    }
90
91    /// Returns whether this token is an unknown word.
92    fn is_unknown(&self) -> bool {
93        self.is_unknown
94    }
95
96    /// Returns the morphological details of the token.
97    fn details(&self) -> Vec<String> {
98        self.details.clone()
99    }
100
101    /// Returns the detail at the specified index.
102    ///
103    /// # Arguments
104    ///
105    /// * `index` - Index of the detail to retrieve.
106    ///
107    /// # Returns
108    ///
109    /// The detail string if found, otherwise nil.
110    fn get_detail(&self, index: usize) -> Option<String> {
111        self.details.get(index).cloned()
112    }
113
114    /// Returns the string representation of the token.
115    fn to_s(&self) -> String {
116        self.surface.clone()
117    }
118
119    /// Returns the inspect representation of the token.
120    fn inspect(&self) -> String {
121        format!(
122            "#<Lindera::Token surface='{}', start={}, end={}, position={}, word_id={}, unknown={}>",
123            self.surface,
124            self.byte_start,
125            self.byte_end,
126            self.position,
127            self.word_id,
128            self.is_unknown
129        )
130    }
131}
132
133/// Defines the Token class in the given Ruby module.
134///
135/// # Arguments
136///
137/// * `ruby` - Ruby runtime handle.
138/// * `module` - Parent Ruby module.
139///
140/// # Returns
141///
142/// `Ok(())` on success, or a Magnus `Error` on failure.
143pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
144    let token_class = module.define_class("Token", ruby.class_object())?;
145    token_class.define_method("surface", method!(RbToken::surface, 0))?;
146    token_class.define_method("byte_start", method!(RbToken::byte_start, 0))?;
147    token_class.define_method("byte_end", method!(RbToken::byte_end, 0))?;
148    token_class.define_method("position", method!(RbToken::position, 0))?;
149    token_class.define_method("word_id", method!(RbToken::word_id, 0))?;
150    token_class.define_method("unknown?", method!(RbToken::is_unknown, 0))?;
151    token_class.define_method("details", method!(RbToken::details, 0))?;
152    token_class.define_method("get_detail", method!(RbToken::get_detail, 1))?;
153    token_class.define_method("to_s", method!(RbToken::to_s, 0))?;
154    token_class.define_method("inspect", method!(RbToken::inspect, 0))?;
155
156    Ok(())
157}
lindera_ruby/token.rs

lindera_ruby/
token.rs
