Struct nlprule::types::Token

pub struct Token<'t> {
    pub word: Word<'t>,
    pub char_span: (usize, usize),
    pub byte_span: (usize, usize),
    pub has_space_before: bool,
    pub chunks: Vec<String>,
    pub sentence: &'t str,
    pub tagger: &'t Tagger,
}

A finished token with all information set.

Fields

word: Word<'t>

char_span: (usize, usize)

byte_span: (usize, usize)

has_space_before: bool

chunks: Vec<String>

sentence: &'t str

tagger: &'t Tagger

Implementations

impl<'t> Token<'t>

pub fn sent_start(sentence: &'t str, tagger: &'t Tagger) -> Self

Get the special sentence start token.

pub fn to_owned_token(&self) -> Token

Trait Implementations

impl<'t> Debug for Token<'t>

fn fmt(&self, __f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'t> From<IncompleteToken<'t>> for Token<'t>

fn from(data: IncompleteToken<'t>) -> Self

Performs the conversion.