Struct rust_tokenizers::preprocessing::tokenizer::base_tokenizer::TokenRef

pub struct TokenRef<'a> {
    pub text: &'a str,
    pub offset: Offset,
    pub mask: Mask,
}

A token that references the original text

Fields

text: &'a str

offset: Offset

mask: Mask

Implementations

impl<'a> TokenRef<'a>

pub fn new(text: &'a str) -> TokenRef<'a>

pub fn to_owned(self) -> Token

Trait Implementations

impl<'a> ConsolidatableTokens<TokenRef<'a>> for Vec<TokenRef<'a>>

fn iter_consolidate_tokens(&self) -> ConsolidatedTokenIterator<TokenRef<'a>>

impl<'a> Debug for TokenRef<'a>

fn fmt(&self, f: &mut Formatter) -> Result

Formats the value using the given formatter.

impl<'a> From<&'a Token> for TokenRef<'a>

fn from(other: &'a Token) -> Self

Performs the conversion.

impl<'a> From<&'a str> for TokenRef<'a>

fn from(text: &'a str) -> Self

Performs the conversion.

impl<'a> From<TokenRef<'a>> for Token

fn from(other: TokenRef<'a>) -> Self

Performs the conversion.

impl<'a> PartialEq<TokenRef<'a>> for TokenRef<'a>

fn eq(&self, other: &TokenRef<'a>) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &TokenRef<'a>) -> bool

This method tests for !=.

impl<'a> StructuralPartialEq for TokenRef<'a>

impl<'a> TokenTrait for TokenRef<'a>

fn offset(&self) -> Option<Offset>

fn mask(&self) -> Mask

fn as_str(&self) -> &str