tekken-rs 0.1.1

Rust implementation of Mistral Tekken tokenizer with audio support
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

use tekken::tekkenizer::Tekkenizer;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("Testing tekken-rs tokenizer...");

    // Load tokenizer
    let tokenizer = Tekkenizer::from_file("tests/assets/tekken.json")?;
    println!("Tokenizer loaded successfully!");
    println!("Vocab size: {}", tokenizer.vocab_size());
    println!("Version: {:?}", tokenizer.version());

    // Test encoding
    let text = "Hello, world! This is a test.";
    let tokens = tokenizer.encode(text, true, true)?;
    println!("Encoded '{text}' -> {tokens:?}");

    // Test decoding
    let decoded = tokenizer.decode(&tokens, tekken::special_tokens::SpecialTokenPolicy::Keep)?;
    println!("Decoded {tokens:?} -> '{decoded}'");

    // Test audio support
    if tokenizer.has_audio_support() {
        println!("✅ Audio support is available");
    } else {
        println!("❌ Audio support not available");
    }

    Ok(())
}