Trait rust_tokenizers::TokenTrait[−][src]

pub trait TokenTrait {
    fn offset(&self) -> Option<Offset>;
    fn mask(&self) -> Mask;
    fn as_str(&self) -> &str;
}

Expand description

Token abstraction trait to access token fields, irrespective of their form (reference of owned)

Required methods

fn offset(&self) -> Option<Offset>[src]

Returns the offset of the token with respect to the original string

fn mask(&self) -> Mask[src]

Returns the token mask

fn as_str(&self) -> &str[src]

Returns a string representation for the token

impl TokenTrait for Token[src]

fn offset(&self) -> Option<Offset>[src]

fn mask(&self) -> Mask[src]

fn as_str(&self) -> &str[src]

impl<'a> TokenTrait for TokenRef<'a>[src]

fn offset(&self) -> Option<Offset>[src]

fn mask(&self) -> Mask[src]

fn as_str(&self) -> &str[src]