use alloc::collections::BTreeMap;
use serde::Deserialize;
use super::tiktoken_util::lenient_decode_base64;
use crate::errors::{
BunsenError,
BunsenResult,
};
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct TekkenVocabulary {
spans: BTreeMap<usize, Vec<u8>>,
specials: BTreeMap<String, usize>,
num_special: usize,
vocab_size: usize,
}
#[derive(Deserialize)]
struct TekkenJson {
config: TekkenConfigJson,
vocab: Vec<TekkenTokenJson>,
special_tokens: Vec<TekkenSpecialJson>,
}
#[derive(Deserialize)]
struct TekkenConfigJson {
default_vocab_size: usize,
default_num_special_tokens: usize,
}
#[derive(Deserialize)]
struct TekkenTokenJson {
rank: usize,
token_bytes: String,
}
#[derive(Deserialize)]
struct TekkenSpecialJson {
rank: usize,
token_str: String,
}
impl TekkenVocabulary {
pub fn from_tekken_json(json: &str) -> BunsenResult<Self> {
let file: TekkenJson = serde_json::from_str(json)
.map_err(|e| BunsenError::invalid_resource("a tekken.json").with_cause(e))?;
let num_special = file.config.default_num_special_tokens;
let vocab_size = file.config.default_vocab_size;
let mut spans = BTreeMap::new();
let mut specials = BTreeMap::new();
for special in file.special_tokens {
if special.rank >= num_special {
return Err(BunsenError::invalid_resource(format!(
"special token {:?} has rank {}, not below the {num_special} specials",
special.token_str, special.rank
)));
}
spans.insert(special.rank, special.token_str.as_bytes().to_vec());
specials.insert(special.token_str, special.rank);
}
for token in file.vocab {
let id = num_special + token.rank;
if id >= vocab_size {
continue;
}
let bytes = lenient_decode_base64(&token.token_bytes).ok_or_else(|| {
BunsenError::invalid_resource(format!(
"vocabulary rank {} is not base64: {:?}",
token.rank, token.token_bytes
))
})?;
spans.insert(id, bytes);
}
Ok(Self {
spans,
specials,
num_special,
vocab_size,
})
}
pub fn len(&self) -> usize {
self.spans.len()
}
pub fn is_empty(&self) -> bool {
self.spans.is_empty()
}
pub fn vocab_size(&self) -> usize {
self.vocab_size
}
pub fn num_special(&self) -> usize {
self.num_special
}
pub fn is_special(
&self,
id: usize,
) -> bool {
id < self.num_special
}
pub fn special_id(
&self,
spelling: &str,
) -> Option<usize> {
self.specials.get(spelling).copied()
}
pub fn bytes(
&self,
id: usize,
) -> Option<&[u8]> {
self.spans.get(&id).map(Vec::as_slice)
}
pub fn spans(&self) -> impl Iterator<Item = (usize, Vec<u8>)> + '_ {
self.spans.iter().map(|(&id, b)| (id, b.clone()))
}
pub fn text(
&self,
ids: &[usize],
) -> BunsenResult<String> {
let mut bytes = Vec::new();
for &id in ids {
bytes.extend_from_slice(self.bytes(id).ok_or_else(|| {
BunsenError::invalid_resource(format!("token id {id} is outside the vocabulary"))
})?);
}
Ok(String::from_utf8_lossy(&bytes).into_owned())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::errors::{
BunsenErrorKind,
testing::ErrorMatcher,
};
const TEKKEN: &str = r#"{
"config": {"pattern": "", "num_vocab_tokens": 3, "default_vocab_size": 6,
"default_num_special_tokens": 4, "version": "v7"},
"vocab": [
{"rank": 0, "token_bytes": "IElu", "token_str": " In"},
{"rank": 1, "token_bytes": "IHRoZQ==", "token_str": " the"},
{"rank": 2, "token_bytes": "eA==", "token_str": "x"}
],
"special_tokens": [
{"rank": 0, "token_str": "<unk>", "is_control": true},
{"rank": 1, "token_str": "<s>", "is_control": true},
{"rank": 2, "token_str": "</s>", "is_control": true},
{"rank": 3, "token_str": "[STREAMING_PAD]", "is_control": true}
]
}"#;
#[test]
fn test_from_tekken_json() {
let v = TekkenVocabulary::from_tekken_json(TEKKEN).unwrap();
assert_eq!(v.num_special(), 4);
assert_eq!(v.vocab_size(), 6);
assert_eq!(v.len(), 6);
assert_eq!(v.bytes(6), None);
assert_eq!(v.special_id("[STREAMING_PAD]"), Some(3));
assert!(v.is_special(3) && !v.is_special(4));
assert_eq!(v.text(&[4, 5]).unwrap(), " In the");
assert_eq!(v.text(&[1]).unwrap(), "<s>");
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("outside the vocabulary")
.assert_err(&v.text(&[6]));
}
#[test]
fn test_refuses_a_malformed_file() {
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("tekken.json")
.assert_err(&TekkenVocabulary::from_tekken_json("{}"));
let bad = TEKKEN.replace("IElu", "I@lu");
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("not base64")
.assert_err(&TekkenVocabulary::from_tekken_json(&bad));
let bad = TEKKEN.replace(r#""rank": 3, "token_str""#, r#""rank": 9, "token_str""#);
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("not below the 4 specials")
.assert_err(&TekkenVocabulary::from_tekken_json(&bad));
}
}