lindera_binding_core/
token.rs

1use lindera::token::Token;
2
3/// FFI-independent view of a [`lindera::token::Token`].
4///
5/// Holds the pure data each binding needs to expose, so the extraction logic
6/// (loading details, reading the word id) lives in one place. Each binding
7/// then maps these fields onto its own FFI token type.
8#[derive(Debug, Clone)]
9pub struct TokenView {
10    pub surface: String,
11    pub byte_start: usize,
12    pub byte_end: usize,
13    pub position: usize,
14    pub word_id: u32,
15    pub is_unknown: bool,
16    pub details: Vec<String>,
17}
18
19impl TokenView {
20    /// Extracts the binding-facing data from a `lindera` token.
21    pub fn from_token(mut token: Token) -> Self {
22        let details = token.details().iter().map(|s| s.to_string()).collect();
23
24        Self {
25            surface: token.surface.to_string(),
26            byte_start: token.byte_start,
27            byte_end: token.byte_end,
28            position: token.position,
29            word_id: token.word_id.id(),
30            is_unknown: token.word_id.is_unknown(),
31            details,
32        }
33    }
34}
lindera_binding_core/token.rs

lindera_binding_core/
token.rs
