use std::{
collections::HashMap,
path::Path,
};
use crate::errors::{
BunsenError,
BunsenErrorKind,
BunsenResult,
ParseError,
sys_at,
};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TiktokenRanks {
spans: Vec<Vec<u8>>,
}
impl TiktokenRanks {
pub fn parse(text: &str) -> BunsenResult<Self> {
Self::parse_from(text, None)
}
fn parse_from(
text: &str,
origin: Option<&Path>,
) -> BunsenResult<Self> {
let invalid =
|error: ParseError| BunsenError::from_cause(BunsenErrorKind::InvalidResource, error);
let at_line = |number: usize| match origin {
Some(path) => format!("{}:{number}", path.display()),
None => format!("line {number}"),
};
let in_file = |error: ParseError| match origin {
Some(path) => error.at(path.display()),
None => error,
};
let mut entries: Vec<(usize, Vec<u8>)> = Vec::new();
for (i, line) in text.lines().enumerate() {
let line = line.trim();
if line.is_empty() {
continue;
}
let number = i + 1;
let mut fields = line.split_whitespace();
let (Some(b64), Some(rank), None) = (fields.next(), fields.next(), fields.next())
else {
return Err(invalid(
ParseError::new("tiktoken rank line")
.at(at_line(number))
.input(line)
.because("expected `<base64> <rank>`"),
));
};
let span = lenient_decode_base64(b64).ok_or_else(|| {
invalid(
ParseError::new("tiktoken base64 field")
.at(at_line(number))
.input(b64)
.because("not base64"),
)
})?;
let rank: usize = rank.parse().map_err(|e| {
invalid(
ParseError::new("tiktoken rank")
.at(at_line(number))
.input(rank)
.with_source(e),
)
})?;
entries.push((rank, span));
}
if entries.is_empty() {
return Err(invalid(in_file(
ParseError::new("tiktoken rank file").because("it has no ranks"),
)));
}
let count = entries.len();
let mut spans: Vec<Option<Vec<u8>>> = vec![None; count];
for (rank, span) in entries {
let slot = spans.get_mut(rank).ok_or_else(|| {
invalid(in_file(ParseError::new("tiktoken rank file").because(
format!(
"rank {rank} is beyond the {count} entries: ranks must be contiguous from 0"
),
)))
})?;
if slot.is_some() {
return Err(invalid(in_file(
ParseError::new("tiktoken rank file")
.because(format!("rank {rank} appears more than once")),
)));
}
*slot = Some(span);
}
let spans = spans
.into_iter()
.map(|span| span.expect("every rank is present"))
.collect();
Ok(Self { spans })
}
pub fn load(path: impl AsRef<Path>) -> BunsenResult<Self> {
let path = path.as_ref();
let text = std::fs::read_to_string(path).map_err(sys_at("read", path))?;
Self::parse_from(&text, Some(path))
}
pub fn len(&self) -> usize {
self.spans.len()
}
pub fn is_empty(&self) -> bool {
self.spans.is_empty()
}
pub fn get(
&self,
rank: usize,
) -> Option<&[u8]> {
self.spans.get(rank).map(Vec::as_slice)
}
pub fn iter(&self) -> impl ExactSizeIterator<Item = &[u8]> {
self.spans.iter().map(Vec::as_slice)
}
pub fn into_spans(self) -> Vec<Vec<u8>> {
self.spans
}
}
fn lenient_decode_base64(field: &str) -> Option<Vec<u8>> {
let data = field.trim_end_matches('=');
if data.len() % 4 == 1 {
return None;
}
let mut out = Vec::with_capacity(data.len() * 3 / 4);
let mut acc: u32 = 0;
let mut bits = 0;
for c in data.bytes() {
let v = match c {
b'A'..=b'Z' => c - b'A',
b'a'..=b'z' => c - b'a' + 26,
b'0'..=b'9' => c - b'0' + 52,
b'+' => 62,
b'/' => 63,
_ => return None,
};
acc = (acc << 6) | u32::from(v);
bits += 6;
if bits >= 8 {
bits -= 8;
out.push((acc >> bits) as u8);
acc &= (1 << bits) - 1;
}
}
Some(out)
}
pub const SYMBOLS: &[&str] = &[
"\"",
"#",
"(",
")",
"*",
"+",
"/",
":",
";",
"<",
"=",
">",
"@",
"[",
"\\",
"]",
"^",
"_",
"`",
"{",
"|",
"}",
"~",
"\u{300c}",
"\u{300d}",
"\u{300e}",
"\u{300f}",
"<<",
">>",
"<<<",
">>>",
"--",
"---",
"-(",
"-[",
"('",
"(\"",
"((",
"))",
"(((",
")))",
"[[",
"]]",
"{{",
"}}",
"\u{266a}\u{266a}",
"\u{266a}\u{266a}\u{266a}",
];
pub const MISCELLANEOUS: &[&str] = &[
"\u{2669}", "\u{266a}", "\u{266b}", "\u{266c}", "\u{266d}", "\u{266e}", "\u{266f}",
];
pub fn lookup(ranks: &TiktokenRanks) -> HashMap<&[u8], i64> {
ranks
.iter()
.enumerate()
.map(|(id, bytes)| (bytes, id as i64))
.collect()
}
fn first_token(
table: &HashMap<&[u8], i64>,
bytes: &[u8],
) -> Option<i64> {
(1..=bytes.len())
.rev()
.find_map(|n| table.get(&bytes[..n]).copied())
}
pub fn non_speech_tokens(ranks: &TiktokenRanks) -> Vec<i64> {
let table = lookup(ranks);
let mut ids: Vec<i64> = [" -", " '"]
.iter()
.filter_map(|s| table.get(s.as_bytes()).copied())
.collect();
for symbol in SYMBOLS {
for candidate in [symbol.to_string(), format!(" {symbol}")] {
if let Some(&id) = table.get(candidate.as_bytes()) {
ids.push(id);
}
}
}
for symbol in MISCELLANEOUS {
for candidate in [symbol.to_string(), format!(" {symbol}")] {
if let Some(id) = first_token(&table, candidate.as_bytes()) {
ids.push(id);
}
}
}
ids.sort_unstable();
ids.dedup();
ids
}
pub fn blank_token(ranks: &TiktokenRanks) -> Option<i64> {
lookup(ranks).get(b" ".as_slice()).copied()
}
#[cfg(test)]
mod tests {
use burn::prelude::*;
use super::*;
use crate::{
errors::{
LookupError,
LookupProblem,
testing::{
ErrorMatcher,
predicate,
},
},
kits::speech::whisper::logit_filters::{
LogitFilter,
SuppressBlank,
},
support::testing::cpu_device,
};
#[test]
fn test_decode_base64() {
assert_eq!(lenient_decode_base64("IQ=="), Some(b"!".to_vec()));
assert_eq!(
lenient_decode_base64("IQ"),
Some(b"!".to_vec()),
"padding is optional"
);
assert_eq!(lenient_decode_base64("SGVsbG8="), Some(b"Hello".to_vec()));
assert_eq!(lenient_decode_base64("IGdhemVk"), Some(b" gazed".to_vec()));
assert_eq!(
lenient_decode_base64("IPCfjg=="),
Some(b" \xf0\x9f\x8e".to_vec())
);
assert_eq!(lenient_decode_base64("+/8="), Some(vec![0xfb, 0xff]));
assert_eq!(lenient_decode_base64("="), Some(Vec::new()));
assert_eq!(lenient_decode_base64("=="), Some(Vec::new()));
assert_eq!(lenient_decode_base64(""), Some(Vec::new()));
assert_eq!(lenient_decode_base64("I%=="), None, "outside the alphabet");
assert_eq!(
lenient_decode_base64("I"),
None,
"no encoding is one symbol long"
);
assert_eq!(
lenient_decode_base64("IQ =="),
None,
"whitespace is not skipped"
);
}
#[test]
fn test_parse() {
let ranks = TiktokenRanks::parse("IQ== 0\nIg== 1\n\n= 2\n").unwrap();
assert_eq!(ranks.len(), 3);
assert!(!ranks.is_empty());
assert_eq!(ranks.get(0), Some(&b"!"[..]));
assert_eq!(ranks.get(1), Some(&b"\""[..]));
assert_eq!(ranks.get(2), Some(&b""[..]), "the empty token is present");
assert_eq!(ranks.get(3), None);
assert_eq!(ranks.iter().count(), 3);
assert_eq!(
ranks.into_spans(),
vec![b"!".to_vec(), b"\"".to_vec(), Vec::new()]
);
}
#[test]
fn test_parse_orders_by_rank() {
let ranks = TiktokenRanks::parse("Ig== 1\nIQ== 0\n").unwrap();
assert_eq!(ranks.get(0), Some(&b"!"[..]));
assert_eq!(ranks.get(1), Some(&b"\""[..]));
}
#[test]
fn test_parse_rejects_malformed_input() {
for (text, why) in [
("", "empty"),
("\n\n", "only blank lines"),
("IQ==", "no rank"),
("IQ== 0 extra", "three fields"),
("IQ== x", "rank not a number"),
("IQ== -1", "negative rank"),
("I%== 0", "bad base64"),
("IQ== 0\nIg== 0", "duplicate rank"),
("IQ== 0\nIg== 2", "gap in the ranks"),
("IQ== 1", "does not start at zero"),
] {
let err = TiktokenRanks::parse(text).expect_err(why);
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.has_cause::<ParseError>()
.assert(&err);
}
}
#[test]
fn test_parse_errors_name_the_line() {
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.cause(predicate("at line 2", |e: &ParseError| {
e.at.as_deref() == Some("line 2")
}))
.assert_err(&TiktokenRanks::parse(
"IQ== 0
I%== 1",
));
}
#[test]
fn test_load_names_the_file_and_line() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("bad.tiktoken");
std::fs::write(
&path,
"IQ== 0
Ig== x
",
)
.unwrap();
let at = format!("{}:2", path.display());
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.cause(predicate("at the file's line 2", move |e: &ParseError| {
e.at.as_deref() == Some(at.as_str())
}))
.assert_err(&TiktokenRanks::load(&path));
}
#[test]
fn test_load_reports_a_missing_file() {
ErrorMatcher::kind(BunsenErrorKind::Lookup)
.cause(predicate("a missing path", |e: &LookupError| {
e.problem == LookupProblem::Missing && e.key == "/nonexistent/whisper.tiktoken"
}))
.assert_err(&TiktokenRanks::load("/nonexistent/whisper.tiktoken"));
}
fn logits(
rows: &[&[f32]],
device: &Device,
) -> Tensor<2> {
let vocab = rows[0].len();
let flat: Vec<f32> = rows.iter().flat_map(|r| r.iter().copied()).collect();
Tensor::from_data(TensorData::new(flat, [rows.len(), vocab]), device)
}
fn to_rows(t: Tensor<2>) -> Vec<Vec<f32>> {
let [rows, vocab] = t.dims();
let flat = t.to_data().try_to_vec_as::<f32>().unwrap();
flat.chunks(vocab).map(|c| c.to_vec()).collect::<Vec<_>>()[..rows].to_vec()
}
#[test]
fn test_suppress_blank() {
let device = cpu_device();
let filter = SuppressBlank::new(2, 4);
let first = to_rows(LogitFilter::apply(
&filter,
logits(&[&[0.0, 1.0, 2.0, 3.0, 4.0]], &device),
&[vec![9, 9]],
2,
));
assert_eq!(
first[0],
vec![0.0, 1.0, f32::NEG_INFINITY, 3.0, f32::NEG_INFINITY]
);
let later = to_rows(LogitFilter::apply(
&filter,
logits(&[&[0.0, 1.0, 2.0, 3.0, 4.0]], &device),
&[vec![9, 9, 1]],
2,
));
assert_eq!(later[0], vec![0.0, 1.0, 2.0, 3.0, 4.0]);
}
}