Struct lindera_tokenizer::token::Token

pub struct Token<'a> {
    pub text: &'a str,
    pub byte_start: usize,
    pub byte_end: usize,
    pub position: usize,
    pub position_length: usize,
    pub word_id: WordId,
    pub dictionary: &'a Dictionary,
    pub user_dictionary: Option<&'a UserDictionary>,
    /* private fields */
}

Fields

text: &'a str

Text content of the token.

byte_start: usize

Starting position of the token in bytes.

byte_end: usize

Ending position of the token in bytes.

position: usize

Position, expressed in number of tokens.

position_length: usize

The length expressed in terms of number of original tokens.

word_id: WordId

The ID of the word and a flag to indicate whether the word is registered in the dictionary.

dictionary: &'a Dictionary

Reference of dictionary.

user_dictionary: Option<&'a UserDictionary>

Reference of user dictionary.

Implementations

impl<'a> Token<'a>

pub fn new( text: &'a str, start: usize, end: usize, position: usize, word_id: WordId, dictionary: &'a Dictionary, user_dictionary: Option<&'a UserDictionary> ) -> Self

pub fn get_details(&mut self) -> Option<Vec<&str>>

pub fn set_details(&mut self, details: Option<Vec<String>>) -> &Token<'a>

Trait Implementations

impl<'a> Clone for Token<'a>

fn clone(&self) -> Token<'a>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<'a> Serialize for Token<'a>

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.