entrenar 0.7.11

Training & Optimization library with autograd, LoRA, quantization, and model merging
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

//! Tokenizer error types.

use thiserror::Error;

/// Tokenizer errors
#[derive(Debug, Error)]
pub enum TokenizerError {
    #[error("Vocabulary not trained")]
    NotTrained,

    #[error("Unknown token: {0}")]
    UnknownToken(String),

    #[error("Invalid token ID: {0}")]
    InvalidTokenId(u32),

    #[error("Training error: {0}")]
    Training(String),

    #[error("IO error: {0}")]
    Io(#[from] std::io::Error),

    #[error("Serialization error: {0}")]
    Serialization(String),
}

/// Result type for tokenizer operations
pub type Result<T> = std::result::Result<T, TokenizerError>;