lindera-filter 0.32.3

Character and token filters for Lindera.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

use serde::Serialize;

use lindera_core::word_entry::WordId;

#[derive(Serialize, Clone)]
pub struct Token {
    /// Text content of the token.
    pub text: String,

    /// Starting position of the token in bytes.
    pub byte_start: usize,

    /// Ending position of the token in bytes.
    pub byte_end: usize,

    /// Position, expressed in number of tokens.
    pub position: usize,

    /// The length expressed in terms of number of original tokens.
    pub position_length: usize,

    /// The ID of the word and a flag to indicate whether the word is registered in the dictionary.
    pub word_id: WordId,

    /// Detailes about the token.
    /// It contains metadata for tokens, such as part-of-speech information.
    pub details: Vec<String>,
}