sentencepiece-rust 0.1.1

From-scratch pure-Rust SentencePiece inference (encode/decode), differentially verified against the upstream C++/Python implementation.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

//! Load a `.model` and print a summary — proves the loader stack end-to-end.
//!
//! Usage: cargo run --example inspect -- tests/models/botchan_1000_bpe.model

use sentencepiece_rust::SentencePieceProcessor;

fn main() {
    let path = std::env::args()
        .nth(1)
        .unwrap_or_else(|| "tests/models/botchan_1000_bpe.model".to_string());

    let sp = match SentencePieceProcessor::open(&path) {
        Ok(sp) => sp,
        Err(e) => {
            eprintln!("failed to load {path}: {e}");
            std::process::exit(1);
        }
    };

    println!("model:       {path}");
    println!("model_type:  {:?}", sp.model_type());
    println!("piece_size:  {}", sp.piece_size());
    println!("first pieces:");
    for id in 0..20.min(sp.piece_size() as i32) {
        if let Some(p) = sp.id_to_piece(id) {
            println!("  {id:>4}  {p:?}");
        }
    }

    let sample = "the quick brown fox";
    match sp.encode(sample) {
        Ok(ids) => {
            println!("encode({sample:?}) = {ids:?}");
            let pieces: Vec<&str> = ids.iter().filter_map(|&i| sp.id_to_piece(i)).collect();
            println!("pieces = {pieces:?}");
            if let Ok(text) = sp.decode(&ids) {
                println!("decode = {text:?}");
            }
        }
        Err(e) => println!("encode({sample:?}) failed: {e}"),
    }
}