realizar 0.3.2

Pure Rust ML inference engine built from scratch - model serving for GGUF and safetensors
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

//! Example: Tokenization with BPE and SentencePiece
//!
//! This example demonstrates:
//! 1. Creating tokenizers (Basic, BPE, SentencePiece)
//! 2. Encoding and decoding text
//! 3. Comparing different tokenization strategies
//!
//! Run with: cargo run --example tokenization

use anyhow::Result;
use realizar::tokenizer::{BPETokenizer, SentencePieceTokenizer, Tokenizer, Vocabulary};

fn demo_basic_tokenizer(text: &str) -> Result<()> {
    println!("--- Basic Tokenizer ---");
    let vocab = Vocabulary::from_tokens(vec![
        "<unk>".to_string(),
        "h".to_string(),
        "e".to_string(),
        "l".to_string(),
        "o".to_string(),
        " ".to_string(),
        "w".to_string(),
        "r".to_string(),
        "d".to_string(),
    ])?;
    let basic = Tokenizer::new(vocab, "<unk>")?;

    let tokens = basic.encode(text);
    let decoded = basic.decode(&tokens)?;
    println!("  Encoded: {:?}", tokens);
    println!("  Decoded: \"{}\"", decoded);
    println!("  Vocab size: {}\n", basic.vocab_size());
    Ok(())
}

fn demo_bpe_tokenizer(text: &str) -> Result<BPETokenizer> {
    println!("--- BPE Tokenizer ---");
    let vocab = vec![
        "<unk>".to_string(),
        "h".to_string(),
        "e".to_string(),
        "l".to_string(),
        "o".to_string(),
        " ".to_string(),
        "w".to_string(),
        "r".to_string(),
        "d".to_string(),
        "ll".to_string(), // Merge: l + l
        "he".to_string(), // Merge: h + e
        "wo".to_string(), // Merge: w + o
    ];
    let merges = vec![
        ("l".to_string(), "l".to_string()),
        ("h".to_string(), "e".to_string()),
        ("w".to_string(), "o".to_string()),
    ];
    let bpe = BPETokenizer::new(vocab, merges, "<unk>")?;

    let tokens = bpe.encode(text);
    let decoded = bpe.decode(&tokens)?;
    println!("  Encoded: {:?}", tokens);
    println!("  Decoded: \"{}\"", decoded);
    println!("  Vocab size: {}\n", bpe.vocab_size());
    Ok(bpe)
}

fn demo_sentencepiece_tokenizer(text: &str) -> Result<()> {
    println!("--- SentencePiece Tokenizer ---");
    let vocab = vec![
        ("<unk>".to_string(), 0.0),
        ("h".to_string(), -2.0),
        ("e".to_string(), -2.0),
        ("l".to_string(), -2.0),
        ("o".to_string(), -2.0),
        (" ".to_string(), -2.0),
        ("w".to_string(), -2.0),
        ("r".to_string(), -2.0),
        ("d".to_string(), -2.0),
        ("hello".to_string(), -1.0), // Higher score (more likely)
        ("world".to_string(), -1.0),
    ];
    let sp = SentencePieceTokenizer::new(vocab, "<unk>")?;

    let tokens = sp.encode(text);
    let decoded = sp.decode(&tokens)?;
    println!("  Encoded: {:?}", tokens);
    println!("  Decoded: \"{}\"", decoded);
    println!("  Vocab size: {}\n", sp.vocab_size());
    Ok(())
}

fn demo_roundtrip_test(bpe: &BPETokenizer) -> Result<()> {
    println!("--- Roundtrip Test ---");
    let test_texts = vec!["hello", "world", "hello world"];
    for test_text in test_texts {
        let tokens = bpe.encode(test_text);
        let decoded = bpe.decode(&tokens)?;
        println!(
            "  \"{}\" -> {:?} -> \"{}\" [{}]",
            test_text,
            tokens,
            decoded,
            if test_text == decoded { "✓" } else { "✗" }
        );
    }
    Ok(())
}

fn main() -> Result<()> {
    println!("=== Tokenization Example ===\n");

    let text = "hello world";
    println!("Input text: \"{}\"\n", text);

    demo_basic_tokenizer(text)?;
    let bpe = demo_bpe_tokenizer(text)?;
    demo_sentencepiece_tokenizer(text)?;
    demo_roundtrip_test(&bpe)?;

    println!("\n=== Tokenization Complete ===");
    Ok(())
}