Struct SpecialTokens

Source

pub struct SpecialTokens {
    pub bos_token: Option<String>,
    pub eos_token: Option<String>,
    pub unk_token: Option<String>,
    pub pad_token: Option<String>,
    pub im_start_token: Option<String>,
    pub im_end_token: Option<String>,
    pub inst_start: Option<String>,
    pub inst_end: Option<String>,
    pub sys_start: Option<String>,
    pub sys_end: Option<String>,
}

Expand description

Special tokens used in chat templates

Fields§

§bos_token: Option<String>§eos_token: Option<String>§unk_token: Option<String>§pad_token: Option<String>§im_start_token: Option<String>§im_end_token: Option<String>§inst_start: Option<String>§inst_end: Option<String>§sys_start: Option<String>§sys_end: Option<String>

SpecialTokens

Struct SpecialTokens Copy item path

Fields§

Trait Implementations§

impl Clone for SpecialTokens

fn clone(&self) -> SpecialTokens

fn clone_from(&mut self, source: &Self)

impl Debug for SpecialTokens

fn fmt(&self, f: &mut Formatter<'_>) -> Result

impl Default for SpecialTokens

fn default() -> SpecialTokens

impl<'de> Deserialize<'de> for SpecialTokens

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where __D: Deserializer<'de>,

impl Serialize for SpecialTokens

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>where __S: Serializer,

Auto Trait Implementations§

impl Freeze for SpecialTokens

impl RefUnwindSafe for SpecialTokens

impl Send for SpecialTokens

impl Sync for SpecialTokens

impl Unpin for SpecialTokens

impl UnsafeUnpin for SpecialTokens

impl UnwindSafe for SpecialTokens

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> CloneToUninit for Twhere T: Clone,

unsafe fn clone_to_uninit(&self, dest: *mut u8)

impl<T> From<T> for T

fn from(t: T) -> T

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> IntoEither for T

fn into_either(self, into_left: bool) -> Either<Self, Self>

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>where F: FnOnce(&Self) -> bool,

impl<T> Pointable for T

const ALIGN: usize

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T> ToOwned for Twhere T: Clone,

type Owned = T

fn to_owned(&self) -> T

fn clone_into(&self, target: &mut T)

impl<T, U> TryFrom<U> for Twhere U: Into<T>,

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for Twhere U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

impl<V, T> VZip<V> for Twhere V: MultiLane<T>,

fn vzip(self) -> V

impl<T> DeserializeOwned for Twhere T: for<'de> Deserialize<'de>,

Struct SpecialTokens

fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where __D: Deserializer<'de>,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where S: Serializer,

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T> CloneToUninit for T
where T: Clone,

impl<T, U> Into<U> for T
where U: From<T>,

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

impl<T> ToOwned for T
where T: Clone,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

impl<T> DeserializeOwned for T
where T: for<'de> Deserialize<'de>,