Type Alias lindera::Token

pub type Token<'a> = Token<'a>;

Aliased Type

struct Token<'a> {
    pub text: &'a str,
    pub byte_start: usize,
    pub byte_end: usize,
    pub position: usize,
    pub position_length: usize,
    pub word_id: WordId,
    pub dictionary: &'a Dictionary,
    pub user_dictionary: Option<&'a UserDictionary>,
    /* private fields */
}

Fields

text: &'a str

Text content of the token.

byte_start: usize

Starting position of the token in bytes.

byte_end: usize

Ending position of the token in bytes.

position: usize

Position, expressed in number of tokens.

position_length: usize

The length expressed in terms of number of original tokens.

word_id: WordId

The ID of the word and a flag to indicate whether the word is registered in the dictionary.

dictionary: &'a Dictionary

Reference of dictionary.

user_dictionary: Option<&'a UserDictionary>

Reference of user dictionary.