trustformers_tokenizers::messagepack_serialization

Struct MessagePackTokenizerConfig

pub struct MessagePackTokenizerConfig {
    pub metadata: MessagePackTokenizerMetadata,
    pub vocabulary: Vec<MessagePackVocabEntry>,
    pub normalization_rules: Vec<MessagePackNormalizationRule>,
    pub merge_rules: Vec<MessagePackMergeRule>,
    pub preprocessing_config: HashMap<String, Vec<u8>>,
    pub postprocessing_config: HashMap<String, Vec<u8>>,
    pub training_config: Option<HashMap<String, Vec<u8>>>,
}

Expand description

MessagePack-compatible tokenizer configuration

Fields§

§metadata: MessagePackTokenizerMetadata§vocabulary: Vec<MessagePackVocabEntry>§normalization_rules: Vec<MessagePackNormalizationRule>§merge_rules: Vec<MessagePackMergeRule>§preprocessing_config: HashMap<String, Vec<u8>>§postprocessing_config: HashMap<String, Vec<u8>>§training_config: Option<HashMap<String, Vec<u8>>>

MessagePackTokenizerConfig

Struct MessagePackTokenizerConfig Copy item path

Fields§

Trait Implementations§

impl Clone for MessagePackTokenizerConfig

fn clone(&self) -> MessagePackTokenizerConfig

fn clone_from(&mut self, source: &Self)

impl Debug for MessagePackTokenizerConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

impl<'de> Deserialize<'de> for MessagePackTokenizerConfig

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where __D: Deserializer<'de>,

impl Serialize for MessagePackTokenizerConfig

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where __S: Serializer,

Auto Trait Implementations§

impl Freeze for MessagePackTokenizerConfig

impl RefUnwindSafe for MessagePackTokenizerConfig

impl Send for MessagePackTokenizerConfig

impl Sync for MessagePackTokenizerConfig

impl Unpin for MessagePackTokenizerConfig

impl UnsafeUnpin for MessagePackTokenizerConfig

impl UnwindSafe for MessagePackTokenizerConfig

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> CloneToUninit for Twhere T: Clone,

unsafe fn clone_to_uninit(&self, dest: *mut u8)

impl<T> ConfigSerializable for Twhere T: Serialize + for<'de> Deserialize<'de>,

fn to_json(&self) -> Result<String, TrustformersError>

fn from_json(json: &str) -> Result<T, TrustformersError>

fn save_to_file(&self, path: &Path) -> Result<(), TrustformersError>

fn load_from_file(path: &Path) -> Result<Self, TrustformersError>where Self: Sized,

impl<T> From<T> for T

fn from(t: T) -> T

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self>

fn in_current_span(self) -> Instrumented<Self>

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> IntoEither for T

fn into_either(self, into_left: bool) -> Either<Self, Self>

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>where F: FnOnce(&Self) -> bool,

impl<T> Pointable for T

const ALIGN: usize

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T> PolicyExt for Twhere T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>where T: Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>where T: Policy<B, E>, P: Policy<B, E>,

impl<T> Same for T

type Output = T

impl<T> ToOwned for Twhere T: Clone,

type Owned = T

fn to_owned(&self) -> T

fn clone_into(&self, target: &mut T)

impl<T, U> TryFrom<U> for Twhere U: Into<T>,

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for Twhere U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

impl<V, T> VZip<V> for Twhere V: MultiLane<T>,

fn vzip(self) -> V

impl<T> WithSubscriber for T

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>where S: Into<Dispatch>,

fn with_current_subscriber(self) -> WithDispatch<Self>

impl<T> DeserializeOwned for Twhere T: for<'de> Deserialize<'de>,

Struct MessagePackTokenizerConfig

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where __D: Deserializer<'de>,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where S: Serializer,

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T> CloneToUninit for T
where T: Clone,

impl<T> ConfigSerializable for T
where T: Serialize + for<'de> Deserialize<'de>,

fn load_from_file(path: &Path) -> Result<Self, TrustformersError>
where Self: Sized,

impl<T, U> Into<U> for T
where U: From<T>,

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

impl<T> PolicyExt for T
where T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

impl<T> ToOwned for T
where T: Clone,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

impl<T> DeserializeOwned for T
where T: for<'de> Deserialize<'de>,