lindera_binding_core/
token.rs

1use lindera::token::Token;
2
3/// FFI-independent view of a [`lindera::token::Token`].
4///
5/// Holds the pure data each binding needs to expose, so the extraction logic
6/// (loading details, reading the word id) lives in one place. Each binding
7/// then maps these fields onto its own FFI token type.
8#[derive(Debug, Clone)]
9pub struct TokenView {
10    pub surface: String,
11    pub byte_start: usize,
12    pub byte_end: usize,
13    pub position: usize,
14    pub word_id: u32,
15    pub is_unknown: bool,
16    pub details: Vec<String>,
17}
18
19impl TokenView {
20    /// Extracts the binding-facing data from a `lindera` token.
21    pub fn from_token(mut token: Token) -> Self {
22        // details_iter avoids the intermediate Vec<&str> that details()
23        // collects on every call.
24        let details = token.details_iter().map(|s| s.to_string()).collect();
25
26        Self {
27            surface: token.surface.to_string(),
28            byte_start: token.byte_start,
29            byte_end: token.byte_end,
30            position: token.position,
31            word_id: token.word_id.id(),
32            is_unknown: token.word_id.is_unknown(),
33            details,
34        }
35    }
36}
37
38/// Surface-only view of a [`lindera::token::Token`], for callers that do
39/// not need morphological details (wakati-style use).
40///
41/// Unlike [`TokenView::from_token`], constructing this never touches the
42/// dictionary's word details, skipping their materialization entirely
43/// (roughly one allocation per token instead of ~14 for IPADIC). Byte
44/// offsets are carried here so bindings can expose them later without a
45/// breaking change; the current language APIs return only the surfaces.
46#[derive(Debug, Clone)]
47pub struct SurfaceView {
48    /// The token's surface form.
49    pub surface: String,
50    /// Starting byte position in the original (pre-filter) text.
51    pub byte_start: usize,
52    /// Ending byte position (exclusive) in the original (pre-filter) text.
53    pub byte_end: usize,
54}
55
56impl SurfaceView {
57    /// Extracts the surface data from a `lindera` token without loading
58    /// details.
59    ///
60    /// # 引数
61    ///
62    /// * `token` - The token to consume; a borrowed surface is copied
63    ///   once, an owned surface is moved.
64    ///
65    /// # 戻り値
66    ///
67    /// The surface-only view.
68    pub fn from_token(token: Token) -> Self {
69        Self {
70            surface: token.surface.into_owned(),
71            byte_start: token.byte_start,
72            byte_end: token.byte_end,
73        }
74    }
75}
lindera_binding_core/token.rs

lindera_binding_core/
token.rs
