ambi 0.2.6

A flexible, multi-backend, customizable AI agent framework, entirely based on Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

// src/llm/tokenizer.rs

use crate::error::{AmbiError, Result};

pub trait TokenizerTrait: Send + Sync {
    /// Pure synchronous method, ultra-fast calculation of Token consumption
    fn count_tokens(&self, text: &str) -> Result<usize>;
}

/// Default tokenizer (based on cl100k_base), a general fast approximation scheme for the vast majority of modern large models (GPT-4, Llama-3)
pub struct DefaultTokenizer {
    bpe: tiktoken_rs::CoreBPE,
}

impl DefaultTokenizer {
    pub fn make() -> Result<Self> {
        let bpe = tiktoken_rs::cl100k_base()
            .map_err(|e| AmbiError::EngineError(format!("Failed to init tokenizer: {}", e)))?;

        Ok(Self { bpe })
    }
}

impl TokenizerTrait for DefaultTokenizer {
    fn count_tokens(&self, text: &str) -> Result<usize> {
        Ok(self.bpe.encode_ordinary(text).len())
    }
}