use std::{
fmt::{
self,
Debug,
},
sync::Arc,
};
use wordchipper::{
TokenDecoder,
TokenType,
WCError,
decoders::TokenDictDecoder,
};
use crate::{
errors::{
BunsenError,
BunsenResult,
ConstraintError,
},
kits::tokens::Detokenizer,
};
pub struct WordchipperDetokenizer<T: TokenType> {
decoder: Arc<dyn TokenDecoder<T>>,
vocab_size: usize,
}
impl<T: TokenType> WordchipperDetokenizer<T> {
pub fn new(
decoder: Arc<dyn TokenDecoder<T>>,
vocab_size: usize,
) -> Self {
Self {
decoder,
vocab_size,
}
}
pub fn from_spans(spans: impl IntoIterator<Item = (usize, Vec<u8>)>) -> BunsenResult<Self> {
let mut table = wordchipper::hash_map_new::<T, Vec<u8>>();
let mut vocab_size = 0;
for (id, bytes) in spans {
let token = T::from_usize(id).ok_or_else(|| {
ConstraintError::out_of_range(
"WordchipperDetokenizer",
"spans",
format!("token id {id}"),
format!("the range of {}", std::any::type_name::<T>()),
)
})?;
table.insert(token, bytes);
vocab_size = vocab_size.max(id + 1);
}
Ok(Self::new(
Arc::new(TokenDictDecoder::new(table)),
vocab_size,
))
}
pub fn vocab_size(&self) -> usize {
self.vocab_size
}
}
impl<T: TokenType> Debug for WordchipperDetokenizer<T> {
fn fmt(
&self,
f: &mut fmt::Formatter<'_>,
) -> fmt::Result {
f.debug_struct("WordchipperDetokenizer")
.field("token_type", &std::any::type_name::<T>())
.field("vocab_size", &self.vocab_size)
.finish_non_exhaustive()
}
}
impl<T: TokenType> Detokenizer for WordchipperDetokenizer<T> {
fn detokenize(
&self,
ids: &[i64],
) -> BunsenResult<String> {
let narrowed = ids
.iter()
.map(|&id| {
T::from_i64(id)
.filter(|t| t.to_usize().is_some_and(|t| t < self.vocab_size))
.ok_or_else(|| {
BunsenError::invalid_resource(format!(
"token id {id} is outside the {}-id vocabulary",
self.vocab_size,
))
})
})
.collect::<BunsenResult<Vec<T>>>()?;
self.decoder
.try_decode_to_string(&narrowed)
.map_err(BunsenError::other)?
.try_result()
.map_err(|e| {
let message = match e {
WCError::IncompleteDecode { remaining } if remaining <= ids.len() => format!(
"token id {} is not in the {}-id vocabulary",
ids[ids.len() - remaining],
self.vocab_size,
),
_ => format!(
"the {}-id vocabulary does not cover every id",
self.vocab_size
),
};
BunsenError::invalid_resource(message).with_cause(e)
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::errors::{
BunsenErrorKind,
testing::ErrorMatcher,
};
fn tiny<T: TokenType>() -> WordchipperDetokenizer<T> {
WordchipperDetokenizer::from_spans([
(0, b"ok".to_vec()),
(1, b" \xf0\x9f\x8e".to_vec()),
(2, b"\x89".to_vec()),
(3, b" done".to_vec()),
(4, Vec::new()),
(6, b"<|special|>".to_vec()),
])
.unwrap()
}
#[test]
fn test_detokenizes() {
let detok = tiny::<u16>();
assert_eq!(detok.vocab_size(), 7);
assert_eq!(detok.detokenize(&[]).unwrap(), "");
assert_eq!(detok.detokenize(&[0, 3]).unwrap(), "ok done");
assert_eq!(detok.detokenize(&[6]).unwrap(), "<|special|>");
}
#[test]
fn test_joins_bytes_before_utf8() {
let detok = tiny::<u16>();
assert_eq!(detok.detokenize(&[0, 1, 2, 3]).unwrap(), "ok 🎉 done");
assert_eq!(detok.detokenize(&[1]).unwrap(), " \u{fffd}");
}
#[test]
fn test_empty_token_is_not_a_gap() {
let detok = tiny::<u16>();
assert_eq!(detok.detokenize(&[0, 4, 3]).unwrap(), "ok done");
assert_eq!(detok.detokenize(&[4]).unwrap(), "");
}
#[test]
fn test_rejects_ids_outside_the_vocabulary() {
let detok = tiny::<u16>();
for bad in [-1, 7, i64::from(u16::MAX) + 1, i64::MAX, i64::MIN] {
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains(&format!("token id {bad} "))
.assert_err(&detok.detokenize(&[0, bad, 3]));
}
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("token id 5 ")
.has_cause::<WCError>()
.assert_err(&detok.detokenize(&[0, 5, 3]));
}
#[test]
fn test_generic_over_token_type() {
assert_eq!(
tiny::<u32>().detokenize(&[0, 1, 2, 3]).unwrap(),
"ok 🎉 done"
);
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.has_cause::<ConstraintError>()
.message_contains("token id 70000")
.assert_err(&WordchipperDetokenizer::<u16>::from_spans([(
70_000,
b"x".to_vec(),
)]));
}
#[test]
fn test_debug_and_trait_object() {
let detok: Arc<dyn Detokenizer> = Arc::new(tiny::<u16>());
let shown = format!("{detok:?}");
assert!(shown.starts_with("WordchipperDetokenizer"), "{shown}");
assert!(shown.contains("token_type: \"u16\""), "{shown}");
assert!(shown.contains("vocab_size: 7"), "{shown}");
assert_eq!(detok.detokenize(&[0]).unwrap(), "ok");
}
}