minillm 0.1.1

A mini inference engine for running transformer language models
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

//! Tokenization example showing how to work with tokens

use minillm::inference::InferenceEngine;

fn main() -> minillm::Result<()> {
    println!("🔤 MiniLLM Tokenization Example");
    
    // Load model
    let engine = InferenceEngine::new("openai-community/gpt2")?;
    
    let texts = vec![
        "Hello, world!",
        "The quick brown fox jumps over the lazy dog.",
        "Rust is a systems programming language.",
        "Transformer models revolutionized natural language processing.",
    ];
    
    for text in texts {
        println!("\n📝 Text: \"{}\"", text);
        
        // Tokenize
        let tokens = engine.tokenize(text)?;
        println!("🔢 Token count: {}", tokens.len());
        println!("🎯 Token IDs: {:?}", tokens);
        
        // Decode back
        let decoded = engine.decode(&tokens)?;
        println!("🔄 Decoded: \"{}\"", decoded);
        
        // Verify round-trip
        if text.trim() == decoded.trim() {
            println!("✅ Round-trip successful");
        } else {
            println!("⚠️  Round-trip mismatch");
        }
    }
    
    Ok(())
}