tpt_tokenizer_core/
error.rs

1//! Error type for tokenization and tokenizer loading.
2
3use alloc::string::String;
4use core::fmt;
5
6/// Errors that can occur while tokenizing text or loading a tokenizer.
7#[derive(Debug)]
8#[non_exhaustive]
9pub enum TokenizerError {
10    /// A sub-word produced during tokenization is not present in the
11    /// vocabulary, and no `<unk>` / `[UNK]` fallback id was configured.
12    UnknownToken(String),
13    /// An I/O error while loading a vocabulary or merge file (std builds only).
14    #[cfg(feature = "std")]
15    Io(std::io::Error),
16    /// A line in a vocabulary or merge file could not be parsed.
17    MalformedFile(String),
18}
19
20impl fmt::Display for TokenizerError {
21    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
22        match self {
23            TokenizerError::UnknownToken(t) => write!(f, "unknown token: {t}"),
24            #[cfg(feature = "std")]
25            TokenizerError::Io(e) => write!(f, "I/O error: {e}"),
26            TokenizerError::MalformedFile(m) => write!(f, "malformed tokenizer file: {m}"),
27        }
28    }
29}
30
31impl core::error::Error for TokenizerError {
32    fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
33        #[cfg(feature = "std")]
34        if let TokenizerError::Io(e) = self {
35            return Some(e);
36        }
37        None
38    }
39}
40
41#[cfg(feature = "std")]
42impl From<std::io::Error> for TokenizerError {
43    fn from(e: std::io::Error) -> Self {
44        TokenizerError::Io(e)
45    }
46}
tpt_tokenizer_core/error.rs

tpt_tokenizer_core/
error.rs
