bpe_tokenizer/
errors.rs

1use thiserror::Error;
2
3/// Represents errors that can occur during BPE tokenization operations.
4#[derive(Error, Debug, Clone, PartialEq, Eq)]
5pub enum BytePairEncoderError {
6    /// Indicates an error occurred while reading the vocabulary file.
7    #[error("Error reading file: {0}")]
8    InvalidFile(String),
9
10    /// Indicates that the vocabulary input was invalid or could not be parsed correctly.
11    #[error("Invalid vocabulary input: Could not parse vocabulary file.")]
12    InvalidVocabularyInput,
13
14    /// Indicates an error occurred during decompression of the vocabulary data.
15    #[error("Error decompressing vocabulary data: {0}")]
16    DecompressionError(String),
17
18    /// Indicates an error occurred during deserialization of the vocabulary data.
19    #[error("Error deserializing vocabulary data: {0}")]
20    DeserializationError(String),
21
22    // Indicates attempt to use a default vocabulary without enabling its Cargo feature.
23    #[error("Error, must enable defualt-small, default-medium, and/or default-large feature(s) to use default vocabulary.")]
24    NoDefaultVocabFeature,
25}