Struct TokensWithOffsets

Source

pub struct TokensWithOffsets {
    pub tokens: Vec<String>,
    pub offsets: Vec<Option<Offset>>,
    pub reference_offsets: Vec<Vec<OffsetSize>>,
    pub masks: Vec<Mask>,
}

Expand description

§Tokenized sequence

Intermediate tokenization steps before encoding, addition of special tokens and truncation

Fields§

§tokens: Vec<String>

Vector of token strings

§offsets: Vec<Option<Offset>>

Offset information (as start and end positions) in relation to the original text. Tokens that can not be related to the original source are registered as None.

§reference_offsets: Vec<Vec<OffsetSize>>

Offset information (as a sequence of positions) in relation to the original text. Tokens that can not be related to the original source are registered as None.

§masks: Vec<Mask>

Masks tokens providing information on the type of tokens. This vector has the same length as token_ids.

Struct TokensWithOffsets Copy item path

§Tokenized sequence

Fields§

Trait Implementations§

impl Clone for TokensWithOffsets

fn clone(&self) -> TokensWithOffsets

fn clone_from(&mut self, source: &Self)

impl Debug for TokensWithOffsets

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Auto Trait Implementations§

impl Freeze for TokensWithOffsets

impl RefUnwindSafe for TokensWithOffsets

impl Send for TokensWithOffsets

impl Sync for TokensWithOffsets

impl Unpin for TokensWithOffsets

impl UnwindSafe for TokensWithOffsets

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> CloneToUninit for Twhere T: Clone,

unsafe fn clone_to_uninit(&self, dest: *mut u8)

impl<T> From<T> for T

fn from(t: T) -> T

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> IntoEither for T

fn into_either(self, into_left: bool) -> Either<Self, Self>

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>where F: FnOnce(&Self) -> bool,

impl<T> Pointable for T

const ALIGN: usize

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T> ToOwned for Twhere T: Clone,

type Owned = T

fn to_owned(&self) -> T

fn clone_into(&self, target: &mut T)

impl<T, U> TryFrom<U> for Twhere U: Into<T>,

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for Twhere U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Struct TokensWithOffsets

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T> CloneToUninit for T
where T: Clone,

impl<T, U> Into<U> for T
where U: From<T>,

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

impl<T> ToOwned for T
where T: Clone,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,