Skip to main content

rover/tokenizer/
error.rs

1//! Tokenizer-module errors.
2
3use thiserror::Error;
4
5use crate::tokenizer::registry::Tokenizer;
6
7#[derive(Debug, Error)]
8pub enum TokenizerError {
9    #[error("unknown tokenizer family: {0}")]
10    UnknownFamily(String),
11
12    #[error("could not download tokenizer for {family}: {source}")]
13    Download {
14        family: Tokenizer,
15        #[source]
16        source: hf_hub::api::sync::ApiError,
17    },
18
19    #[error("tokenizer file for {family} is corrupt: {source}")]
20    Parse {
21        family: Tokenizer,
22        #[source]
23        source: Box<dyn std::error::Error + Send + Sync>,
24    },
25
26    #[error("io error reading tokenizer at {path}: {source}")]
27    Io {
28        path: String,
29        #[source]
30        source: std::io::Error,
31    },
32
33    #[error("tokenizer {0} is not loaded; call ensure_loaded() first")]
34    NotLoaded(Tokenizer),
35}