memra-tokenizer 0.86.1

GGUF-native BPE/SPM tokenizer with chat-template support for the memra inference engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

//! tok-check: quick encode/decode sanity tool for the memra tokenizer.
//! usage: tok-check <model.gguf> "<text>"

use memra_gguf::GgufFile;
use memra_tokenizer::Tokenizer;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let path = std::env::args()
        .nth(1)
        .expect("usage: tok-check <model.gguf> \"<text>\"");
    let text = std::env::args()
        .nth(2)
        .unwrap_or_else(|| "Hello, world!".to_string());

    let g = GgufFile::open(&path)?;
    let tok = Tokenizer::from_gguf(&g)?;
    println!(
        "pre={} vocab={} eos={} bos={:?}",
        tok.pre(),
        tok.vocab_size(),
        tok.eos_id(),
        tok.bos_id()
    );

    let ids = tok.encode(&text, true);
    let ids_json: Vec<String> = ids.iter().map(|i| i.to_string()).collect();
    println!("encode({text:?}) = [{}]", ids_json.join(", "));

    let back = tok.decode(&ids);
    println!("decode = {back:?}");
    println!("round-trip {}", if back == text { "OK" } else { "DIFFERS" });

    Ok(())
}