Struct TokenizedInput

pub struct TokenizedInput {
    pub token_ids: Vec<i64>,
    pub segment_ids: Vec<i8>,
    pub special_tokens_mask: Vec<i8>,
    pub overflowing_tokens: Vec<i64>,
    pub num_truncated_tokens: usize,
}

Fields

token_ids: Vec<i64>

segment_ids: Vec<i8>

special_tokens_mask: Vec<i8>

overflowing_tokens: Vec<i64>

num_truncated_tokens: usize

Trait Implementations

impl Clone for TokenizedInput

fn clone(&self) -> TokenizedInput

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenizedInput

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl IntoPy<PyObject> for TokenizedInput

fn into_py(self, py: Python<'_>) -> PyObject

Performs the conversion.

impl PartialEq for TokenizedInput

fn eq(&self, other: &TokenizedInput) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl PartialOrd for TokenizedInput

fn partial_cmp(&self, other: &TokenizedInput) -> Option<Ordering>

This method returns an ordering between self and other values if one exists.

fn lt(&self, other: &Rhs) -> bool

Tests less than (for self and other) and is used by the < operator.

fn le(&self, other: &Rhs) -> bool

Tests less than or equal to (for self and other) and is used by the <= operator.

fn gt(&self, other: &Rhs) -> bool

Tests greater than (for self and other) and is used by the > operator.

fn ge(&self, other: &Rhs) -> bool

Tests greater than or equal to (for self and other) and is used by the >= operator.

impl PyObjectAlloc for TokenizedInput

unsafe fn alloc(_py: Python<'_>) -> *mut PyObject

unsafe fn dealloc(py: Python<'_>, obj: *mut PyObject)

Calls the rust destructor for the object and frees the memory (usually by calling ptr->ob_type->tp_free). This function is used as tp_dealloc implementation.

unsafe fn drop(py: Python<'_>, obj: *mut PyObject)

Calls the rust destructor for the object.

impl PyTypeInfo for TokenizedInput

const NAME: &'static str = "TokenizedInput"

Class name

const MODULE: Option<&'static str> = None

Module name, if any

const DESCRIPTION: &'static str = "\u{0}"

Class doc string

const FLAGS: usize = 0usize

Type flags (ie PY_TYPE_FLAG_GC, PY_TYPE_FLAG_WEAKREF)

const SIZE: usize = 120usize

Size of the rust PyObject structure (PyObject + rust structure)

const OFFSET: isize = 16isize

Type instance offset inside PyObject structure

type Type = TokenizedInput

Type of objects to store in PyObject struct

type BaseType = PyAny

Base class

unsafe fn type_object() -> &'static mut PyTypeObject

PyTypeObject instance for this type, which might still need to be initialized

fn is_instance(object: &PyAny) -> bool

Check if *mut ffi::PyObject is instance of this type

fn is_exact_instance(object: &PyAny) -> bool

Check if *mut ffi::PyObject is exact instance of this type

impl StructuralPartialEq for TokenizedInput