Trait rust_tokenizers::TokenTrait

pub trait TokenTrait {
    fn offset(&self) -> Option<Offset>;
    fn mask(&self) -> Mask;
    fn as_str(&self) -> &str;
}

Token abstraction trait to access token fields, irrespective of their form (reference of owned)

Required methods

fn offset(&self) -> Option<Offset>

Returns the offset of the token with respect to the original string

fn mask(&self) -> Mask

Returns the token mask

fn as_str(&self) -> &str

Returns a string representation for the token

Loading content...

Implementors

impl TokenTrait for Token

fn offset(&self) -> Option<Offset>

fn mask(&self) -> Mask

fn as_str(&self) -> &str

impl<'a> TokenTrait for TokenRef<'a>

fn offset(&self) -> Option<Offset>

fn mask(&self) -> Mask

fn as_str(&self) -> &str

Loading content...