Struct rust_tokenizers::preprocessing::tokenizer::base_tokenizer::Token

pub struct Token {
    pub text: String,
    pub offset: Offset,
    pub reference_offsets: Vec<OffsetSize>,
    pub mask: Mask,
}

A token that references the original text An owned token

Fields

text: String

offset: Offset

reference_offsets: Vec<OffsetSize>

mask: Mask

Implementations

impl Token

pub fn new(text: String) -> Token

pub fn as_ref(&self) -> TokenRef

Trait Implementations

impl Clone for Token

fn clone(&self) -> Token

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl ConsolidatableTokens<Token> for Vec<Token>

fn iter_consolidate_tokens(&self) -> ConsolidatedTokenIterator<Token>

impl Debug for Token

fn fmt(&self, f: &mut Formatter) -> Result

Formats the value using the given formatter.

impl<'_> From<&'_ str> for Token

fn from(text: &str) -> Self

Performs the conversion.

impl<'a> From<&'a Token> for TokenRef<'a>

fn from(other: &'a Token) -> Self

Performs the conversion.

impl<'a> From<TokenRef<'a>> for Token

fn from(other: TokenRef<'a>) -> Self

Performs the conversion.

impl PartialEq<Token> for Token

fn eq(&self, other: &Token) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Token) -> bool

This method tests for !=.

impl StructuralPartialEq for Token

impl TokenTrait for Token

fn offset(&self) -> Option<Offset>

fn mask(&self) -> Mask

fn as_str(&self) -> &str