tok_debug/
tok_debug.rs

1//! Debug: encode fixed strings with a checkpoint tokenizer. `LAYA_TOK=/path/to/tokenizer.json`.
2use laya::common::Tokenizer;
3use laya::tokenizer::LayaTokenizer;
4
5fn main() -> Result<(), Box<dyn std::error::Error>> {
6    let path = std::env::var("LAYA_TOK").expect("set LAYA_TOK=/path/to/tokenizer.json");
7    let tok = LayaTokenizer::from_file(&path)?;
8    println!(
9        "cls={} sep={} mask={} pad={} mask_token={:?}",
10        tok.cls_id(),
11        tok.sep_id(),
12        tok.mask_id(),
13        tok.pad_id(),
14        tok.mask_token()
15    );
16    for s in [
17        "choice question: What is this?",
18        " refund: money returned",
19        "私は二重に請求されました",
20    ] {
21        println!("{s:?} -> {:?}", tok.encode(s));
22    }
23    Ok(())
24}
tok_debug/tok_debug.rs

tok_debug/
tok_debug.rs
