Struct CustomTokenizerFormat

Source

pub struct CustomTokenizerFormat {
    pub format_name: String,
    pub format_version: String,
    pub vocabulary: CustomVocabulary,
    pub special_tokens: Vec<CustomSpecialToken>,
    pub normalization_rules: Vec<NormalizationRule>,
    pub pre_tokenization_rules: Vec<PreTokenizationRule>,
    pub post_processing_rules: Vec<PostProcessingRule>,
    pub metadata: HashMap<String, Value>,
}

Expand description

Represents a custom tokenizer format specification

Fields§

§format_name: String§format_version: String§vocabulary: CustomVocabulary§special_tokens: Vec<CustomSpecialToken>§normalization_rules: Vec<NormalizationRule>§pre_tokenization_rules: Vec<PreTokenizationRule>§post_processing_rules: Vec<PostProcessingRule>§metadata: HashMap<String, Value>

CustomTokenizerFormat

Struct CustomTokenizerFormat Copy item path

Fields§

Trait Implementations§

impl Clone for CustomTokenizerFormat

fn clone(&self) -> CustomTokenizerFormat

fn clone_from(&mut self, source: &Self)

impl Debug for CustomTokenizerFormat

fn fmt(&self, f: &mut Formatter<'_>) -> Result

impl<'de> Deserialize<'de> for CustomTokenizerFormat

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where __D: Deserializer<'de>,

impl Serialize for CustomTokenizerFormat

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where __S: Serializer,

Auto Trait Implementations§

impl Freeze for CustomTokenizerFormat

impl RefUnwindSafe for CustomTokenizerFormat

impl Send for CustomTokenizerFormat

impl Sync for CustomTokenizerFormat

impl Unpin for CustomTokenizerFormat

impl UnsafeUnpin for CustomTokenizerFormat

impl UnwindSafe for CustomTokenizerFormat

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> CloneToUninit for Twhere T: Clone,

unsafe fn clone_to_uninit(&self, dest: *mut u8)

impl<T> ConfigSerializable for Twhere T: Serialize + for<'de> Deserialize<'de>,

fn to_json(&self) -> Result<String, TrustformersError>

fn from_json(json: &str) -> Result<T, TrustformersError>

fn save_to_file(&self, path: &Path) -> Result<(), TrustformersError>

fn load_from_file(path: &Path) -> Result<Self, TrustformersError>where Self: Sized,

impl<T> From<T> for T

fn from(t: T) -> T

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self>

fn in_current_span(self) -> Instrumented<Self>

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> IntoEither for T

fn into_either(self, into_left: bool) -> Either<Self, Self>

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>where F: FnOnce(&Self) -> bool,

impl<T> Pointable for T

const ALIGN: usize

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T> PolicyExt for Twhere T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>where T: Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>where T: Policy<B, E>, P: Policy<B, E>,

impl<T> Same for T

type Output = T

impl<T> ToOwned for Twhere T: Clone,

type Owned = T

fn to_owned(&self) -> T

fn clone_into(&self, target: &mut T)

impl<T, U> TryFrom<U> for Twhere U: Into<T>,

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for Twhere U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

impl<V, T> VZip<V> for Twhere V: MultiLane<T>,

fn vzip(self) -> V

impl<T> WithSubscriber for T

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>where S: Into<Dispatch>,

fn with_current_subscriber(self) -> WithDispatch<Self>

impl<T> DeserializeOwned for Twhere T: for<'de> Deserialize<'de>,

Struct CustomTokenizerFormat

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where __D: Deserializer<'de>,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where S: Serializer,

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T> CloneToUninit for T
where T: Clone,

impl<T> ConfigSerializable for T
where T: Serialize + for<'de> Deserialize<'de>,

fn load_from_file(path: &Path) -> Result<Self, TrustformersError>
where Self: Sized,

impl<T, U> Into<U> for T
where U: From<T>,

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

impl<T> PolicyExt for T
where T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

impl<T> ToOwned for T
where T: Clone,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

impl<T> DeserializeOwned for T
where T: for<'de> Deserialize<'de>,