Trait rust_tokenizers::preprocessing::tokenizer::base_tokenizer::TokenTrait

pub trait TokenTrait {
    fn offset(&self) -> Option<Offset>;
    fn mask(&self) -> Mask;
    fn as_str(&self) -> &str;
}

Required methods

fn offset(&self) -> Option<Offset>

fn mask(&self) -> Mask

fn as_str(&self) -> &str

Loading content...

Implementors

impl TokenTrait for Token

fn offset(&self) -> Option<Offset>

fn mask(&self) -> Mask

fn as_str(&self) -> &str

impl<'a> TokenTrait for TokenRef<'a>

fn offset(&self) -> Option<Offset>

fn mask(&self) -> Mask

fn as_str(&self) -> &str

Loading content...