langchainrust 0.5.0

A LangChain-inspired framework for building LLM applications in Rust. Supports OpenAI, Agents, Tools, Memory, Chains, RAG, BM25, Hybrid Retrieval, LangGraph, HyDE, Reranking, MultiQuery, and native Function Calling.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

//! Token counter and cost tracking
//!
//! Provides token counting (tiktoken), usage statistics, cost estimation,
//! and a `TokenTrackingLLM` wrapper.

pub mod counter;
pub mod tiktoken;
pub mod tracker;

pub use counter::{TokenCounter, TokenUsage};
pub use tiktoken::TiktokenCounter;
pub use tracker::{ModelPricing, TokenTrackingLLM};

use std::sync::LazyLock;
use tiktoken_rs::CoreBPE;

/// Global cached tiktoken encoder (cl100k_base, for GPT-3.5/4/4o).
static GLOBAL_ENCODER: LazyLock<CoreBPE> = LazyLock::new(|| {
    tiktoken_rs::cl100k_base().expect("Failed to load tiktoken cl100k_base encoder")
});

/// Count tokens in text using the global tiktoken encoder.
///
/// This is a convenience function that uses a lazily-initialized
/// cl100k_base encoder (suitable for GPT-3.5/4/4o models).
///
/// # Examples
/// ```
/// use langchainrust::core::token_counter::count_tokens;
///
/// let n = count_tokens("Hello, world!");
/// assert!(n > 0);
/// ```
pub fn count_tokens(text: &str) -> usize {
    GLOBAL_ENCODER.encode_with_special_tokens(text).len()
}