kyro 0.1.1

A high-performance ML inference engine
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

#![allow(dead_code)]

use anyhow::Result;
use std::path::Path;
use tokenizers::Tokenizer;

pub struct LuminaTokenizer {
    tokenizer: Tokenizer,
}

impl LuminaTokenizer {
    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
        let tokenizer = Tokenizer::from_file(path).map_err(|e| anyhow::anyhow!(e))?;
        Ok(Self { tokenizer })
    }

    pub fn encode(&self, text: &str) -> Result<Vec<u32>> {
        let encoding = self
            .tokenizer
            .encode(text, true)
            .map_err(|e| anyhow::anyhow!(e))?;
        Ok(encoding.get_ids().to_vec())
    }

    pub fn decode(&self, tokens: &[u32]) -> Result<String> {
        let decoded = self
            .tokenizer
            .decode(tokens, true)
            .map_err(|e| anyhow::anyhow!(e))?;
        Ok(decoded)
    }
}