tekken/
errors.rs

1use thiserror::Error;
2
3/// Type alias for Results with `TokenizerError`.
4///
5/// This provides a convenient shorthand for Result types throughout the library.
6pub type Result<T> = std::result::Result<T, TokenizerError>;
7
8/// Comprehensive error type for tokenizer operations.
9///
10/// This enum covers all possible error conditions that can occur during
11/// tokenizer initialization, text/audio processing, and other operations.
12/// It uses the `thiserror` crate to provide detailed error messages and
13/// automatic conversion from underlying error types.
14///
15/// # Error Categories
16///
17/// * **I/O Errors**: File reading/writing operations
18/// * **Parsing Errors**: JSON deserialization and Base64 decoding
19/// * **Processing Errors**: Tokenization and audio processing failures
20/// * **Configuration Errors**: Invalid parameters or missing tokens
21/// * **Policy Errors**: Special token handling violations
22#[derive(Error, Debug)]
23pub enum TokenizerError {
24    /// I/O operation failed (file reading, writing, etc.).
25    #[error("IO error: {0}")]
26    Io(#[from] std::io::Error),
27
28    /// JSON parsing or serialization failed.
29    #[error("JSON error: {0}")]
30    Json(#[from] serde_json::Error),
31
32    /// Base64 decoding failed.
33    #[error("Base64 decode error: {0}")]
34    Base64(#[from] base64::DecodeError),
35
36    /// Error in the underlying tokenization engine.
37    #[error("Tokenizers error: {0}")]
38    Tokenizers(String),
39
40    /// Audio processing operation failed.
41    #[error("Audio error: {0}")]
42    Audio(String),
43
44    /// Configuration parameters are invalid or inconsistent.
45    #[error("Invalid configuration: {0}")]
46    InvalidConfig(String),
47
48    /// Required token (usually special token) was not found in vocabulary.
49    #[error("Token not found: {0}")]
50    TokenNotFound(String),
51
52    /// Operation violated the specified special token policy.
53    #[error("Special token policy violation: {0}")]
54    SpecialTokenPolicy(String),
55
56    /// File format or data format is not supported.
57    #[error("Unsupported format: {0}")]
58    UnsupportedFormat(String),
59}
tekken/errors.rs

tekken/
errors.rs
