use crate::incremental::MergeState;
use crate::pretokenize::{Span, SplitPattern, pretokens, spans};
use crate::reference::MergeTable;
use crate::vocab::Vocabulary;
use crate::Encoding;
use crate::{TokenId, ExatokError};
const BYTES_PER_TOKEN_RESERVE: usize = 3;
#[derive(Debug, Clone)]
pub struct Tokenizer {
encoding: Encoding,
vocabulary: Vocabulary,
byte_ids: [TokenId; 256],
}
impl Tokenizer {
pub fn new(vocabulary: Vocabulary, encoding: Encoding) -> Result<Self, ExatokError> {
let byte_ids = crate::byte_token_ids(encoding.artifact(), &vocabulary)?;
Ok(Self {
encoding,
vocabulary,
byte_ids,
})
}
#[must_use]
pub const fn encoding(&self) -> &Encoding {
&self.encoding
}
#[must_use]
pub const fn vocabulary(&self) -> &Vocabulary {
&self.vocabulary
}
#[must_use]
pub const fn byte_ids(&self) -> &[TokenId; 256] {
&self.byte_ids
}
#[must_use]
pub fn encode(&self, input: &[u8]) -> Vec<TokenId> {
encode_bytes(
self.encoding.pattern(),
input,
&self.byte_ids,
self.vocabulary.merge_table(),
)
}
#[must_use]
pub fn split(&self, input: &[u8]) -> Vec<Span> {
spans(self.encoding.pattern(), input)
}
#[must_use]
pub fn bytes_of(&self, id: TokenId) -> Option<&[u8]> {
if let Some(bytes) = self.vocabulary.bytes_of(id) {
return Some(bytes);
}
self.vocabulary
.specials()
.find(|(_, special)| *special == id)
.map(|(text, _)| text.as_bytes())
}
pub fn decode(&self, ids: &[TokenId]) -> Result<Vec<u8>, UnknownTokenId> {
let mut bytes = Vec::with_capacity(ids.len() * BYTES_PER_TOKEN_RESERVE);
for (index, id) in ids.iter().enumerate() {
let Some(token) = self.bytes_of(*id) else {
return Err(UnknownTokenId {
id: *id,
index,
n_vocab: self.vocabulary.n_vocab(),
highest_ordinary: self.vocabulary.max_token_id().map_or(0, |id| id.0),
});
};
bytes.extend_from_slice(token);
}
Ok(bytes)
}
}
#[must_use]
pub fn encode_bytes(
pattern: SplitPattern,
input: &[u8],
byte_ids: &[TokenId; 256],
table: &impl MergeTable,
) -> Vec<TokenId> {
let mut ids = Vec::with_capacity(input.len() / BYTES_PER_TOKEN_RESERVE + 1);
let mut start = Vec::new();
let mut state = MergeState::new();
for span in pretokens(pattern, input) {
let pretoken = &input[span.start..span.end];
start.clear();
start.extend(pretoken.iter().map(|&byte| byte_ids[byte as usize]));
state.load(&start, table);
state.run(table);
ids.extend_from_slice(state.tokens());
}
ids
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct UnknownTokenId {
pub id: TokenId,
pub index: usize,
pub n_vocab: usize,
pub highest_ordinary: u32,
}
impl std::fmt::Display for UnknownTokenId {
fn fmt(&self, out: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
out,
"token id {} at position {} is not in this vocabulary, whose highest decodable id is \
{} ({} ordinary, the rest special)",
self.id.0,
self.index,
self.n_vocab.saturating_sub(1),
self.highest_ordinary
)
}
}
impl std::error::Error for UnknownTokenId {}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::{Tokenizer, UnknownTokenId};
use crate::Encoding;
use crate::pretokenize::SplitPattern;
use crate::vocab::{Source, Vocabulary};
use crate::{Rank, TokenId};
fn tiny() -> Vocabulary {
let mut text = String::new();
for byte in 0..=u8::MAX {
text.push_str(&base64(&[byte]));
text.push(' ');
text.push_str(&format!("{byte}\n"));
}
text.push_str(&base64(b"hi"));
text.push_str(" 256\n");
Vocabulary::load_tiktoken(Source::new("tiny", &text)).expect("tiny artifact must load")
}
fn base64(bytes: &[u8]) -> String {
const ALPHABET: &[u8; 64] =
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut out = String::new();
for chunk in bytes.chunks(3) {
let mut word = 0_u32;
for (index, byte) in chunk.iter().enumerate() {
word |= u32::from(*byte) << (16 - 8 * index);
}
for index in 0..4 {
if index <= chunk.len() {
let slot = (word >> (18 - 6 * index)) & 0x3F;
out.push(char::from(ALPHABET[slot as usize]));
} else {
out.push('=');
}
}
}
out
}
fn tokenizer() -> Tokenizer {
Tokenizer::new(tiny(), Encoding::new("tiny", SplitPattern::Gpt2))
.expect("every single byte is present")
}
#[test]
fn encode_merges_the_pair_the_table_holds_and_not_its_reverse() {
let tokenizer = tokenizer();
assert_eq!(tokenizer.encode(b"hi"), vec![TokenId(256)]);
assert_eq!(
tokenizer.encode(b"ih"),
vec![TokenId(u32::from(b'i')), TokenId(u32::from(b'h'))]
);
}
#[test]
fn a_pair_split_across_pretokens_does_not_merge() {
let tokenizer = tokenizer();
let ids = tokenizer.encode(b"h i");
assert_eq!(
ids,
vec![
TokenId(u32::from(b'h')),
TokenId(u32::from(b' ')),
TokenId(u32::from(b'i')),
]
);
}
#[test]
fn decode_recovers_the_exact_bytes_encode_consumed() {
let tokenizer = tokenizer();
let input = b"hi h i hi";
let ids = tokenizer.encode(input);
assert_eq!(tokenizer.decode(&ids).unwrap(), input.to_vec());
}
#[test]
fn decode_names_the_offending_id_and_where_it_sat() {
let tokenizer = tokenizer();
let error = tokenizer
.decode(&[TokenId(0), TokenId(9999), TokenId(1)])
.expect_err("9999 is not a token in the tiny vocabulary");
assert_eq!(
error,
UnknownTokenId {
id: TokenId(9999),
index: 1,
n_vocab: 257,
highest_ordinary: 256,
}
);
assert!(
error.to_string().contains("at position 1"),
"the message must carry the position: {error}"
);
}
#[test]
fn decode_resolves_a_registered_special_token() {
let mut vocabulary = tiny();
vocabulary
.register_special("<|endoftext|>", TokenId(300))
.expect("300 is representable and unused");
let tokenizer = Tokenizer::new(vocabulary, Encoding::new("tiny", SplitPattern::Gpt2))
.expect("every single byte is present");
assert_eq!(
tokenizer.decode(&[TokenId(300)]).unwrap(),
b"<|endoftext|>".to_vec()
);
assert_eq!(tokenizer.bytes_of(TokenId(300)), Some(&b"<|endoftext|>"[..]));
}
#[test]
fn ordinary_encoding_of_a_special_literal_yields_its_byte_ids() {
let mut vocabulary = tiny();
vocabulary
.register_special("<|endoftext|>", TokenId(300))
.expect("300 is representable and unused");
let tokenizer = Tokenizer::new(vocabulary, Encoding::new("tiny", SplitPattern::Gpt2))
.expect("every single byte is present");
let ids = tokenizer.encode(b"<|endoftext|>");
assert!(
!ids.contains(&TokenId(300)),
"ordinary encoding must not emit a special id: {ids:?}"
);
assert_eq!(tokenizer.decode(&ids).unwrap(), b"<|endoftext|>".to_vec());
}
#[test]
fn byte_ids_are_the_byte_values_for_a_rank_ordered_artifact() {
let tokenizer = tokenizer();
for byte in 0..=u8::MAX {
assert_eq!(
tokenizer.byte_ids()[byte as usize],
TokenId(u32::from(byte)),
"byte {byte}"
);
}
}
#[test]
fn a_vocabulary_without_every_single_byte_is_refused_at_construction() {
let mut text = String::new();
for byte in 0..=254_u8 {
text.push_str(&base64(&[byte]));
text.push(' ');
text.push_str(&format!("{byte}\n"));
}
let vocabulary =
Vocabulary::load_tiktoken(Source::new("holed", &text)).expect("artifact must load");
let error = Tokenizer::new(vocabulary, Encoding::new("holed", SplitPattern::Gpt2))
.expect_err("byte 0xff has no token");
let message = error.to_string();
assert!(message.contains("0xff"), "must name the byte: {message}");
assert!(message.contains("Fix:"), "must say what to do: {message}");
}
#[test]
fn a_merge_at_the_absent_sentinel_rank_is_rejected_by_the_loader() {
let mut table = crate::reference::HostMergeTable::new();
let error = table
.insert(
"sentinel.tiktoken",
crate::PairKey::new(TokenId(1), TokenId(2)),
TokenId(3),
Rank::ABSENT,
)
.expect_err("the sentinel rank must be refused");
assert!(
error.to_string().contains("Fix:"),
"must say what to do: {error}"
);
}
}