xgrammar/tokenizer/error.rs
1//! Errors from building [`TokenizerInfo`](super::TokenizerInfo).
2
3/// An error from constructing tokenizer info.
4#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
5pub enum TokenizerError {
6 /// The padded vocabulary size is smaller than the number of real tokens.
7 #[error("vocab_size ({vocab_size}) must be at least the number of tokens in the vocab ({num_tokens}).")]
8 VocabSizeTooSmall {
9 /// The requested vocabulary size.
10 vocab_size: i32,
11 /// The number of tokens in the encoded vocabulary.
12 num_tokens: usize,
13 },
14 /// The metadata JSON is malformed.
15 #[error("{0}")]
16 InvalidMetadata(String),
17}