lindera/
token.rs

1use pyo3::prelude::*;
2
3use lindera::token::Token;
4
5/// Token object wrapping the Rust Token data.
6///
7/// This class provides robust access to token field and details.
8#[pyclass(name = "Token")]
9pub struct PyToken {
10    /// Surface form of the token.
11    #[pyo3(get)]
12    pub surface: String,
13
14    /// Start byte position in the original text.
15    #[pyo3(get)]
16    pub byte_start: usize,
17
18    /// End byte position in the original text.
19    #[pyo3(get)]
20    pub byte_end: usize,
21
22    /// Position index of the token.
23    #[pyo3(get)]
24    pub position: usize,
25
26    /// Word ID in the dictionary.
27    #[pyo3(get)]
28    pub word_id: u32,
29
30    /// Whether this token is an unknown word (not found in the dictionary).
31    #[pyo3(get)]
32    pub is_unknown: bool,
33
34    /// Morphological details of the token.
35    #[pyo3(get)]
36    pub details: Vec<String>,
37}
38
39#[pymethods]
40impl PyToken {
41    /// Returns the detail at the specified index.
42    ///
43    /// # Arguments
44    ///
45    /// * `index` - Index of the detail to retrieve.
46    ///
47    /// # Returns
48    ///
49    /// The detail string if found, otherwise None.
50    #[pyo3(signature = (index))]
51    fn get_detail(&self, index: usize) -> Option<String> {
52        self.details.get(index).cloned()
53    }
54
55    /// Returns a string representation of the token.
56    fn __repr__(&self) -> String {
57        format!(
58            "<Token surface='{}', start={}, end={}, position={}, word_id={}, is_unknown={}>",
59            self.surface,
60            self.byte_start,
61            self.byte_end,
62            self.position,
63            self.word_id,
64            self.is_unknown
65        )
66    }
67}
68
69impl PyToken {
70    /// Builds a `PyToken` from a binding-core [`TokenView`].
71    pub fn from_view(view: lindera_binding_core::TokenView) -> Self {
72        Self {
73            surface: view.surface,
74            byte_start: view.byte_start,
75            byte_end: view.byte_end,
76            position: view.position,
77            word_id: view.word_id,
78            is_unknown: view.is_unknown,
79            details: view.details,
80        }
81    }
82
83    /// Builds a `PyToken` from a `lindera` token via [`TokenView`].
84    pub fn from_token(token: Token) -> Self {
85        Self::from_view(lindera_binding_core::TokenView::from_token(token))
86    }
87}
88
89pub fn register(parent_module: &Bound<'_, PyModule>) -> PyResult<()> {
90    let py = parent_module.py();
91    let m = PyModule::new(py, "token")?;
92    m.add_class::<PyToken>()?;
93    parent_module.add_submodule(&m)?;
94    Ok(())
95}
lindera/token.rs

lindera/
token.rs
