Struct Encoding 

pub struct Encoding { /* private fields */ }

Represents the output of a Tokenizer.

Implementations

impl Encoding

pub fn new( ids: Vec<u32>, type_ids: Vec<u32>, tokens: Vec<String>, words: Vec<Option<u32>>, offsets: Vec<(usize, usize)>, special_tokens_mask: Vec<u32>, attention_mask: Vec<u32>, overflowing: Vec<Encoding>, sequence_ranges: AHashMap<usize, Range<usize>>, ) -> Encoding

pub fn with_capacity(len: usize) -> Encoding

pub fn from_tokens(tokens: Vec<Token>, type_id: u32) -> Encoding

pub fn is_empty(&self) -> bool

Whether this Encoding is empty

pub fn len(&self) -> usize

Return the total length of this Encoding

pub fn n_sequences(&self) -> usize

Return the number of sequences combined in this Encoding

pub fn set_sequence_id(&mut self, sequence_id: usize)

Set the given sequence id for the whole range of tokens contained in this Encoding

pub fn get_tokens(&self) -> &[String]

pub fn get_word_ids(&self) -> &[Option<u32>]

pub fn get_word_ids_mut(&mut self) -> &mut [Option<u32>]

pub fn get_sequence_ids(&self) -> Vec<Option<usize>>

pub fn get_ids(&self) -> &[u32]

pub fn get_type_ids(&self) -> &[u32]

pub fn set_type_ids(&mut self, type_ids: Vec<u32>)

pub fn get_offsets(&self) -> &[(usize, usize)]

pub fn get_offsets_mut(&mut self) -> &mut [(usize, usize)]

pub fn get_special_tokens_mask(&self) -> &[u32]

pub fn get_attention_mask(&self) -> &[u32]

pub fn get_overflowing(&self) -> &Vec<Encoding>

pub fn set_overflowing(&mut self, overflowing: Vec<Encoding>)

pub fn get_overflowing_mut(&mut self) -> &mut Vec<Encoding>

pub fn take_overflowing(&mut self) -> Vec<Encoding>

pub fn token_to_sequence(&self, token: usize) -> Option<usize>

Returns the index of the sequence containing the given token

pub fn word_to_tokens( &self, word: u32, sequence_id: usize, ) -> Option<(usize, usize)>

Get the encoded tokens corresponding to the word at the given index in the input sequence, with the form (start_token, end_token + 1)

pub fn word_to_chars( &self, word: u32, sequence_id: usize, ) -> Option<(usize, usize)>

Get the offsets of the word at the given index in the input sequence.

pub fn token_to_chars(&self, token: usize) -> Option<(usize, (usize, usize))>

Get the offsets of the token at the given index.

pub fn token_to_word(&self, token: usize) -> Option<(usize, u32)>

Get the word that contains the token at the given index.

pub fn char_to_token(&self, pos: usize, sequence_id: usize) -> Option<usize>

Get the token that contains the given char.

pub fn char_to_word(&self, pos: usize, sequence_id: usize) -> Option<u32>

Get the word that contains the given char.

pub fn truncate( &mut self, max_len: usize, stride: usize, direction: TruncationDirection, )

Truncate the current Encoding.

Panics if stride >= max_len

pub fn merge<I>(encodings: I, growing_offsets: bool) -> Encoding

where I: IntoIterator<Item = Encoding>,

Merge all Encodings together

pub fn merge_with(&mut self, pair: Encoding, growing_offsets: bool)

Merge ourself with the given Encoding. Happens in place.

pub fn pad( &mut self, target_length: usize, pad_id: u32, pad_type_id: u32, pad_token: &str, direction: PaddingDirection, )

Trait Implementations

impl Clone for Encoding

fn clone(&self) -> Encoding

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Encoding

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Default for Encoding

fn default() -> Encoding

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for Encoding

fn deserialize<__D>( __deserializer: __D, ) -> Result<Encoding, <__D as Deserializer<'de>>::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl FromIterator<(u32, String, (usize, usize), Option<u32>, u32)> for Encoding

fn from_iter<I>(iter: I) -> Encoding

where I: IntoIterator<Item = (u32, String, (usize, usize), Option<u32>, u32)>,

Creates a value from an iterator.

impl FromIterator<Encoding> for Encoding

fn from_iter<I>(iter: I) -> Encoding

where I: IntoIterator<Item = Encoding>,

Creates a value from an iterator.

impl PartialEq for Encoding

fn eq(&self, other: &Encoding) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Serialize for Encoding

fn serialize<__S>( &self, __serializer: __S, ) -> Result<<__S as Serializer>::Ok, <__S as Serializer>::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl StructuralPartialEq for Encoding