laya-decision 0.2.4

Pure-Rust port of the Laya non-autoregressive System-1 decision engine (Laya-Decision)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

//! Debug: encode fixed strings with a checkpoint tokenizer. `LAYA_TOK=/path/to/tokenizer.json`.
use laya::common::Tokenizer;
use laya::tokenizer::LayaTokenizer;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let path = std::env::var("LAYA_TOK").expect("set LAYA_TOK=/path/to/tokenizer.json");
    let tok = LayaTokenizer::from_file(&path)?;
    println!(
        "cls={} sep={} mask={} pad={} mask_token={:?}",
        tok.cls_id(),
        tok.sep_id(),
        tok.mask_id(),
        tok.pad_id(),
        tok.mask_token()
    );
    for s in [
        "choice question: What is this?",
        " refund: money returned",
        "私は二重に請求されました",
    ] {
        println!("{s:?} -> {:?}", tok.encode(s));
    }
    Ok(())
}