kitoken 0.11.0

Fast tokenizer for language models, supporting BPE, Unigram and WordPiece tokenization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

//! Utilities for converting different tokenizer formats into Kitoken definitions.
//!
//! Additional methods for initializing from supported formats are also available in [`Definition`](crate::Definition) and [`Kitoken`](crate::Kitoken).

use alloc::string::String;

use crate::{InitializationError, RegexError};

#[cfg(feature = "convert-sentencepiece")]
mod sentencepiece;
#[cfg(feature = "convert-sentencepiece")]
pub use sentencepiece::*;

#[cfg(feature = "convert-tiktoken")]
mod tiktoken;
#[cfg(feature = "convert-tiktoken")]
pub use tiktoken::*;

#[cfg(feature = "convert-tokenizers")]
mod tokenizers;
#[cfg(feature = "convert-tokenizers")]
pub use tokenizers::*;

#[cfg(feature = "convert-tekken")]
mod tekken;
#[cfg(feature = "convert-tekken")]
pub use tekken::*;

/// Errors encountered when the conversion fails.
#[non_exhaustive]
#[derive(Debug, thiserror::Error)]
pub enum ConversionError {
    /// The data is invalid. See the error message for more information.
    #[error("invalid data: {0}")]
    InvalidData(String),
    /// The configuration is not supported.
    #[error("unsupported configuration: {0}")]
    UnsupportedConfiguration(String),
    /// A regex failed to compile.
    #[error("invalid regex: {0}")]
    InvalidRegex(String),
    /// The tokenizer failed to initialize.
    #[error("{0}")]
    InitializationError(InitializationError),
    /// Reading the data failed.
    #[cfg(feature = "std")]
    #[error("{0}")]
    IOError(#[from] std::io::Error),
}
impl From<InitializationError> for ConversionError {
    fn from(e: InitializationError) -> Self {
        Self::InitializationError(e)
    }
}
impl From<RegexError> for ConversionError {
    fn from(e: RegexError) -> Self {
        Self::InvalidRegex(e.0)
    }
}