Struct LayaTokenizer 

pub struct LayaTokenizer { /* private fields */ }

A loaded checkpoint tokenizer.

Implementations

impl LayaTokenizer

pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self>

Load a tokenizer from a tokenizer.json file.

Examples found in repository

examples/tok_debug.rs (line 7)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let path = std::env::var("LAYA_TOK").expect("set LAYA_TOK=/path/to/tokenizer.json");
    let tok = LayaTokenizer::from_file(&path)?;
    println!(
        "cls={} sep={} mask={} pad={} mask_token={:?}",
        tok.cls_id(),
        tok.sep_id(),
        tok.mask_id(),
        tok.pad_id(),
        tok.mask_token()
    );
    for s in [
        "choice question: What is this?",
        " refund: money returned",
        "私は二重に請求されました",
    ] {
        println!("{s:?} -> {:?}", tok.encode(s));
    }
    Ok(())
}

pub fn encode_with_special(&self, text: &str, max_length: usize) -> Vec<u32>

Encode with special tokens (CLS/SEP), truncated to max_length. Used for encoder embeddings (mean-pooling), matching HF tok(text, truncation=True, max_length=...).

Trait Implementations

impl Tokenizer for LayaTokenizer

fn cls_id(&self) -> u32

fn sep_id(&self) -> u32

fn mask_id(&self) -> u32

fn pad_id(&self) -> u32

fn mask_token(&self) -> &str

fn encode(&self, text: &str) -> Vec<u32>