Skip to main content

chunk_your_tools/tiktoken/
config.rs

1//! Runtime defaults for tiktoken encoding.
2
3#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
4pub enum AllowedSpecial {
5    #[default]
6    All,
7    None,
8}
9
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct TiktokenConfig {
12    pub encoding: String,
13    pub allowed_special: AllowedSpecial,
14}
15
16impl Default for TiktokenConfig {
17    fn default() -> Self {
18        Self {
19            encoding: "cl100k_base".to_string(),
20            allowed_special: AllowedSpecial::All,
21        }
22    }
23}
24
25pub fn configure(cfg: TiktokenConfig) {
26    super::configure_and_refresh(cfg);
27}
28
29#[must_use]
30pub fn snapshot() -> TiktokenConfig {
31    super::config_snapshot()
32}