lindera/
token.rs

1use pyo3::prelude::*;
2
3// Simplified Token representation for Python
4#[pyclass(name = "Token")]
5#[derive(Clone)]
6pub struct PyToken {
7    #[pyo3(get)]
8    pub text: String,
9    #[pyo3(get)]
10    pub byte_start: usize,
11    #[pyo3(get)]
12    pub byte_end: usize,
13    #[pyo3(get)]
14    pub position: usize,
15    #[pyo3(get)]
16    pub position_length: usize,
17    #[pyo3(get)]
18    pub details: Vec<String>,
19}
20
21#[pymethods]
22impl PyToken {
23    #[new]
24    pub fn new(
25        text: String,
26        byte_start: usize,
27        byte_end: usize,
28        position: usize,
29        position_length: usize,
30        details: Vec<String>,
31    ) -> Self {
32        PyToken {
33            text,
34            byte_start,
35            byte_end,
36            position,
37            position_length,
38            details,
39        }
40    }
41
42    fn __str__(&self) -> String {
43        format!("Token(text='{}', pos={})", self.text, self.position)
44    }
45
46    fn __repr__(&self) -> String {
47        format!(
48            "Token(text='{}', byte_start={}, byte_end={}, position={}, details={:?})",
49            self.text, self.byte_start, self.byte_end, self.position, self.details
50        )
51    }
52}
lindera/token.rs

lindera/
token.rs
