smoltok_core/regex/
errors.rs

1use crate::tokenizer::VocabSizeTooSmall;
2use std::fmt;
3
4/// Error type for regex compilation failures.
5#[derive(Debug)]
6pub struct RegexCompilationError(pub String);
7
8impl std::error::Error for RegexCompilationError {}
9
10impl fmt::Display for RegexCompilationError {
11    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
12        write!(f, "Failed to compile regex pattern: {}", self.0)
13    }
14}
15
16/// Error type for RegexBPETokenizer configuration.
17#[derive(Debug)]
18pub enum RegexBPETokenizerConfigError {
19    /// The vocabulary size is too small.
20    VocabSizeTooSmall(VocabSizeTooSmall),
21    /// The regex pattern failed to compile.
22    RegexCompilationError(RegexCompilationError),
23}
24
25impl std::error::Error for RegexBPETokenizerConfigError {}
26
27impl fmt::Display for RegexBPETokenizerConfigError {
28    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
29        match self {
30            Self::VocabSizeTooSmall(e) => write!(f, "{}", e),
31            Self::RegexCompilationError(e) => write!(f, "{}", e),
32        }
33    }
34}
35
36impl From<VocabSizeTooSmall> for RegexBPETokenizerConfigError {
37    fn from(e: VocabSizeTooSmall) -> Self {
38        Self::VocabSizeTooSmall(e)
39    }
40}
41
42impl From<RegexCompilationError> for RegexBPETokenizerConfigError {
43    fn from(e: RegexCompilationError) -> Self {
44        Self::RegexCompilationError(e)
45    }
46}
smoltok_core/regex/errors.rs

smoltok_core/regex/
errors.rs
