trustformers_tokenizers::tokenization_debugger

Struct TokenizationResult

pub struct TokenizationResult {
    pub tokenizer_name: String,
    pub tokens: Vec<String>,
    pub token_ids: Vec<u32>,
    pub token_count: usize,
    pub character_count: usize,
    pub compression_ratio: f64,
    pub processing_time_ms: f64,
    pub character_offsets: Option<Vec<(usize, usize)>>,
    pub oov_tokens: Vec<String>,
    pub special_tokens: Vec<String>,
}

Expand description

Results from tokenizing with a specific tokenizer

Fields§

§tokenizer_name: String§tokens: Vec<String>§token_ids: Vec<u32>§token_count: usize§character_count: usize§compression_ratio: f64§processing_time_ms: f64§character_offsets: Option<Vec<(usize, usize)>>§oov_tokens: Vec<String>§special_tokens: Vec<String>

TokenizationResult

Struct TokenizationResult Copy item path

Fields§

Trait Implementations§

impl Clone for TokenizationResult

fn clone(&self) -> TokenizationResult

fn clone_from(&mut self, source: &Self)

impl Debug for TokenizationResult

fn fmt(&self, f: &mut Formatter<'_>) -> Result

impl<'de> Deserialize<'de> for TokenizationResult

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where __D: Deserializer<'de>,

impl Serialize for TokenizationResult

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where __S: Serializer,

Auto Trait Implementations§

impl Freeze for TokenizationResult

impl RefUnwindSafe for TokenizationResult

impl Send for TokenizationResult

impl Sync for TokenizationResult

impl Unpin for TokenizationResult

impl UnsafeUnpin for TokenizationResult

impl UnwindSafe for TokenizationResult

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> CloneToUninit for Twhere T: Clone,

unsafe fn clone_to_uninit(&self, dest: *mut u8)

impl<T> ConfigSerializable for Twhere T: Serialize + for<'de> Deserialize<'de>,

fn to_json(&self) -> Result<String, TrustformersError>

fn from_json(json: &str) -> Result<T, TrustformersError>

fn save_to_file(&self, path: &Path) -> Result<(), TrustformersError>

fn load_from_file(path: &Path) -> Result<Self, TrustformersError>where Self: Sized,

impl<T> From<T> for T

fn from(t: T) -> T

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self>

fn in_current_span(self) -> Instrumented<Self>

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> IntoEither for T

fn into_either(self, into_left: bool) -> Either<Self, Self>

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>where F: FnOnce(&Self) -> bool,

impl<T> Pointable for T

const ALIGN: usize

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T> PolicyExt for Twhere T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>where T: Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>where T: Policy<B, E>, P: Policy<B, E>,

impl<T> Same for T

type Output = T

impl<T> ToOwned for Twhere T: Clone,

type Owned = T

fn to_owned(&self) -> T

fn clone_into(&self, target: &mut T)

impl<T, U> TryFrom<U> for Twhere U: Into<T>,

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for Twhere U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

impl<V, T> VZip<V> for Twhere V: MultiLane<T>,

fn vzip(self) -> V

impl<T> WithSubscriber for T

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>where S: Into<Dispatch>,

fn with_current_subscriber(self) -> WithDispatch<Self>

impl<T> DeserializeOwned for Twhere T: for<'de> Deserialize<'de>,

Struct TokenizationResult

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where __D: Deserializer<'de>,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where S: Serializer,

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T> CloneToUninit for T
where T: Clone,

impl<T> ConfigSerializable for T
where T: Serialize + for<'de> Deserialize<'de>,

fn load_from_file(path: &Path) -> Result<Self, TrustformersError>
where Self: Sized,

impl<T, U> Into<U> for T
where U: From<T>,

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

impl<T> PolicyExt for T
where T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

impl<T> ToOwned for T
where T: Clone,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

impl<T> DeserializeOwned for T
where T: for<'de> Deserialize<'de>,