use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SpecialTokens {
Unk,
Bos,
Eos,
BeginInst,
EndInst,
BeginTools,
EndTools,
BeginToolResults,
EndToolResults,
ToolCalls,
Img,
Pad,
ImgBreak,
ImgEnd,
Prefix,
Middle,
Suffix,
BeginSystem,
EndSystem,
BeginToolContent,
Audio,
BeginAudio,
Transcribe,
Args,
CallId,
BeginThink,
EndThink,
StreamingPad,
StreamingWord,
TextToAudio,
AudioToText,
BeginModelSettings,
EndModelSettings,
}
impl SpecialTokens {
#[must_use]
pub const fn as_str(&self) -> &'static str {
match self {
Self::Unk => "<unk>",
Self::Bos => "<s>",
Self::Eos => "</s>",
Self::BeginInst => "[INST]",
Self::EndInst => "[/INST]",
Self::BeginTools => "[AVAILABLE_TOOLS]",
Self::EndTools => "[/AVAILABLE_TOOLS]",
Self::BeginToolResults => "[TOOL_RESULTS]",
Self::EndToolResults => "[/TOOL_RESULTS]",
Self::ToolCalls => "[TOOL_CALLS]",
Self::Img => "[IMG]",
Self::Pad => "<pad>",
Self::ImgBreak => "[IMG_BREAK]",
Self::ImgEnd => "[IMG_END]",
Self::Prefix => "[PREFIX]",
Self::Middle => "[MIDDLE]",
Self::Suffix => "[SUFFIX]",
Self::BeginSystem => "[SYSTEM_PROMPT]",
Self::EndSystem => "[/SYSTEM_PROMPT]",
Self::BeginToolContent => "[TOOL_CONTENT]",
Self::Audio => "[AUDIO]",
Self::BeginAudio => "[BEGIN_AUDIO]",
Self::Transcribe => "[TRANSCRIBE]",
Self::Args => "[ARGS]",
Self::CallId => "[CALL_ID]",
Self::BeginThink => "[THINK]",
Self::EndThink => "[/THINK]",
Self::StreamingPad => "[STREAMING_PAD]",
Self::StreamingWord => "[STREAMING_WORD]",
Self::TextToAudio => "[NEXT_AUDIO_TEXT]",
Self::AudioToText => "[REPEAT_AUDIO_TEXT]",
Self::BeginModelSettings => "[MODEL_SETTINGS]",
Self::EndModelSettings => "[/MODEL_SETTINGS]",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SpecialTokenPolicy {
Ignore,
Keep,
Raise,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SpecialTokenInfo {
pub rank: usize,
pub token_str: String,
pub is_control: bool,
}