use std::path::{Path, PathBuf};
use thiserror::Error;
#[derive(Debug, Error)]
pub enum TokenizerError {
#[error("{}: {source}", path.display())]
Io {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("{}: larger than the {max}-byte cap for tokenizer files", path.display())]
FileTooLarge {
path: PathBuf,
max: u64,
},
#[error("{}: invalid JSON: {source}", path.display())]
Json {
path: PathBuf,
#[source]
source: serde_json::Error,
},
#[error("{}: tokenizer backend: {message}", path.display())]
Backend {
path: PathBuf,
message: String,
},
#[error("encode failed: {0}")]
Encode(String),
#[error("decode failed: {0}")]
Decode(String),
#[error("tokenizer_config.json has no chat_template string")]
MissingChatTemplate,
#[error("tokenizer_config.json sets add_bos_token=true, but this model never prepends BOS")]
BosNotAllowed,
#[error("generation_config.json eos_token_id {found:?} does not match expected {expected:?}")]
EosMismatch {
found: Vec<u32>,
expected: [u32; 2],
},
#[error("special token {token:?} maps to {found:?}, expected id {expected}")]
SpecialTokenMismatch {
token: &'static str,
found: Option<u32>,
expected: u32,
},
#[error("unsupported chat role {0:?} (supported: system, user, assistant, tool)")]
UnsupportedRole(String),
}
impl TokenizerError {
pub(crate) fn io(path: &Path, source: std::io::Error) -> Self {
Self::Io {
path: path.to_path_buf(),
source,
}
}
}