lindera_filter/
token.rs

1use serde::Serialize;
2
3use lindera_core::word_entry::WordId;
4
5#[derive(Serialize, Clone)]
6pub struct Token {
7    /// Text content of the token.
8    pub text: String,
9
10    /// Starting position of the token in bytes.
11    pub byte_start: usize,
12
13    /// Ending position of the token in bytes.
14    pub byte_end: usize,
15
16    /// Position, expressed in number of tokens.
17    pub position: usize,
18
19    /// The length expressed in terms of number of original tokens.
20    pub position_length: usize,
21
22    /// The ID of the word and a flag to indicate whether the word is registered in the dictionary.
23    pub word_id: WordId,
24
25    /// Detailes about the token.
26    /// It contains metadata for tokens, such as part-of-speech information.
27    pub details: Vec<String>,
28}