vil_tokenizer 0.1.0

VIL Tokenizer Engine — native Rust BPE tokenization for LLM token counting and text splitting
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

//! Semantic types for tokenizer operations (Tier B AI).

use serde::{Deserialize, Serialize};
use vil_macros::{VilAiEvent, VilAiFault, VilAiState};

#[derive(Clone, Debug, Serialize, Deserialize, VilAiEvent)]
pub struct TokenizeEvent {
    pub text_length: u32,
    pub token_count: u32,
    pub latency_us: u64,
    pub vocab: String,
}

#[derive(Clone, Debug, Serialize, Deserialize, VilAiFault)]
pub struct TokenizeFault {
    pub message: String,
}

#[derive(Clone, Debug, Default, Serialize, Deserialize, VilAiState)]
pub struct TokenizerState {
    pub total_requests: u64,
    pub total_tokens_counted: u64,
}