Struct lindera_core::token::Token

pub struct Token<'a> {
    pub byte_start: usize,
    pub byte_end: usize,
    pub position: usize,
    pub position_length: usize,
    pub word_id: WordId,
    pub dictionary: &'a Dictionary,
    pub user_dictionary: Option<&'a UserDictionary>,
    /* private fields */
}

Fields

byte_start: usize

Starting position of the token in bytes.

byte_end: usize

Ending position of the token in bytes.

position: usize

Position, expressed in number of tokens.

position_length: usize

The length expressed in term of number of original tokens.

word_id: WordId

The ID of the word and a flag to indicate whether the word is registered in the dictionary.

dictionary: &'a Dictionary

Reference of dictionary.

user_dictionary: Option<&'a UserDictionary>

Reference of user dictionary.

Implementations

impl<'a> Token<'a>

pub fn new(
    text: &str,
    start: usize,
    end: usize,
    position: usize,
    word_id: WordId,
    dictionary: &'a Dictionary,
    user_dictionary: Option<&'a UserDictionary>
) -> Self

pub fn get_text(&self) -> &str

pub fn set_text(&mut self, text: String) -> &Token<'a>

pub fn get_details(&mut self) -> Option<Vec<&str>>

pub fn set_details(&mut self, details: Option<Vec<String>>) -> &Token<'a>

Trait Implementations

impl<'a> Clone for Token<'a>

fn clone(&self) -> Token<'a>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<'a> Serialize for Token<'a>

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where
    __S: Serializer,

Serialize this value into the given Serde serializer.