Struct rust_tokenizers::TokensWithOffsets[−][src]

pub struct TokensWithOffsets {
    pub tokens: Vec<String>,
    pub offsets: Vec<Option<Offset>>,
    pub reference_offsets: Vec<Vec<OffsetSize>>,
    pub masks: Vec<Mask>,
}

Tokenized sequence

Intermediate tokenization steps before encoding, addition of special tokens and truncation

Fields

tokens: Vec<String>

Vector of token strings

offsets: Vec<Option<Offset>>

Offset information (as start and end positions) in relation to the original text. Tokens that can not be related to the original source are registered as None.

reference_offsets: Vec<Vec<OffsetSize>>

Offset information (as a sequence of positions) in relation to the original text. Tokens that can not be related to the original source are registered as None.

masks: Vec<Mask>

Masks tokens providing information on the type of tokens. This vector has the same length as token_ids.

Trait Implementations

`impl Clone for TokensWithOffsets`[src]

`fn clone(&self) -> TokensWithOffsets`[src]

`pub fn clone_from(&mut self, source: &Self)`1.0.0[src]

`impl Debug for TokensWithOffsets`[src]

`fn fmt(&self, f: &mut Formatter<'_>) -> Result`[src]

Auto Trait Implementations

`impl RefUnwindSafe for TokensWithOffsets`[src]

`impl Send for TokensWithOffsets`[src]

`impl Sync for TokensWithOffsets`[src]

`impl Unpin for TokensWithOffsets`[src]

`impl UnwindSafe for TokensWithOffsets`[src]

Blanket Implementations

`impl<T> Any for T where T: 'static + ?Sized,` [src]

`pub fn type_id(&self) -> TypeId`[src]

`impl<T> Borrow<T> for T where T: ?Sized,` [src]

`pub fn borrow(&self) -> &T`[src]

`impl<T> BorrowMut<T> for T where T: ?Sized,` [src]

`pub fn borrow_mut(&mut self) -> &mut T`[src]

`impl<T> From<T> for T`[src]

`pub fn from(t: T) -> T`[src]

`impl<T, U> Into for T where U: From<T>,` [src]

`pub fn into(self) -> U`[src]

`impl<T> Pointable for T`

`pub const ALIGN: usize`

`type Init = T`

The type for initializers.

`pub unsafe fn init(init: <T as Pointable>::Init) -> usize`

`pub unsafe fn deref<'a>(ptr: usize) -> &'a T`

`pub unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T`

`pub unsafe fn drop(ptr: usize)`

`impl<T> ToOwned for T where T: Clone,` [src]

`type Owned = T`

The resulting type after obtaining ownership.

`pub fn to_owned(&self) -> T`[src]

`pub fn clone_into(&self, target: &mut T)`[src]

`impl<T, U> TryFrom for T where U: Into<T>,` [src]

`type Error = Infallible`

The type returned in the event of a conversion error.

`pub fn try_from(value: U) -> Result<T, <T as TryFrom>::Error>`[src]

`impl<T, U> TryInto for T where U: TryFrom<T>,` [src]

`type Error = >::Error`

The type returned in the event of a conversion error.

Struct rust_tokenizers::TokensWithOffsets[−][src]

Tokenized sequence

Fields

Trait Implementations

impl Clone for TokensWithOffsets[src]

fn clone(&self) -> TokensWithOffsets[src]

pub fn clone_from(&mut self, source: &Self)1.0.0[src]

impl Debug for TokensWithOffsets[src]

fn fmt(&self, f: &mut Formatter<'_>) -> Result[src]

Auto Trait Implementations

impl RefUnwindSafe for TokensWithOffsets[src]

impl Send for TokensWithOffsets[src]

impl Sync for TokensWithOffsets[src]

impl Unpin for TokensWithOffsets[src]

impl UnwindSafe for TokensWithOffsets[src]

Blanket Implementations

impl<T> Any for T where T: 'static + ?Sized, [src]

pub fn type_id(&self) -> TypeId[src]

impl<T> Borrow<T> for T where T: ?Sized, [src]

pub fn borrow(&self) -> &T[src]

impl<T> BorrowMut<T> for T where T: ?Sized, [src]

pub fn borrow_mut(&mut self) -> &mut T[src]

impl<T> From<T> for T[src]

pub fn from(t: T) -> T[src]

impl<T, U> Into<U> for T where U: From<T>, [src]

pub fn into(self) -> U[src]

impl<T> Pointable for T

pub const ALIGN: usize

type Init = T

pub unsafe fn init(init: <T as Pointable>::Init) -> usize

pub unsafe fn deref<'a>(ptr: usize) -> &'a T

pub unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

pub unsafe fn drop(ptr: usize)

impl<T> ToOwned for T where T: Clone, [src]

type Owned = T

pub fn to_owned(&self) -> T[src]

pub fn clone_into(&self, target: &mut T)[src]

impl<T, U> TryFrom<U> for T where U: Into<T>, [src]

type Error = Infallible

pub fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>[src]

impl<T, U> TryInto<U> for T where U: TryFrom<T>, [src]

type Error = <U as TryFrom<T>>::Error

pub fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>[src]