entrenar/tokenizer/error.rs
1//! Tokenizer error types.
2
3use thiserror::Error;
4
5/// Tokenizer errors
6#[derive(Debug, Error)]
7pub enum TokenizerError {
8 #[error("Vocabulary not trained")]
9 NotTrained,
10
11 #[error("Unknown token: {0}")]
12 UnknownToken(String),
13
14 #[error("Invalid token ID: {0}")]
15 InvalidTokenId(u32),
16
17 #[error("Training error: {0}")]
18 Training(String),
19
20 #[error("IO error: {0}")]
21 Io(#[from] std::io::Error),
22
23 #[error("Serialization error: {0}")]
24 Serialization(String),
25}
26
27/// Result type for tokenizer operations
28pub type Result<T> = std::result::Result<T, TokenizerError>;