use std::collections::HashMap;
use std::collections::hash_map::Entry;
use std::fs;
use std::path::{Path, PathBuf};
use crate::byte_encoder;
use crate::pretokenize::SplitPattern;
use crate::reference::{HostMergeTable, MergeEntry, MergeTable};
use crate::{PairKey, Rank, TokenId, ExatokError};
pub const MAX_TOKEN_ID: u32 = u32::MAX - 2;
const MAX_ID_SPARSITY: usize = 4;
const SPARSITY_FLOOR: usize = 1024;
#[derive(Debug, Clone, Copy)]
pub struct Source<'a> {
pub name: &'a str,
pub text: &'a str,
}
impl<'a> Source<'a> {
#[must_use]
pub const fn new(name: &'a str, text: &'a str) -> Self {
Self { name, text }
}
}
#[derive(Debug, Clone, Default)]
pub struct Vocabulary {
by_id: Vec<Vec<u8>>,
by_bytes: HashMap<Vec<u8>, TokenId>,
merges: HostMergeTable,
specials: Vec<(String, TokenId)>,
}
fn artifact_error(artifact: &str, location: String, detail: String, fix: &str) -> ExatokError {
ExatokError::Artifact {
artifact: artifact.to_owned(),
location,
detail,
fix: fix.to_owned(),
}
}
fn line_error(artifact: &str, line: usize, detail: String, fix: &str) -> ExatokError {
artifact_error(artifact, format!("line {line}"), detail, fix)
}
fn insert_merge(
table: &mut HostMergeTable,
artifact: &str,
line: usize,
pair: PairKey,
merged: TokenId,
rank: Rank,
) -> Result<(), ExatokError> {
table
.insert(artifact, pair, merged, rank)
.map_err(|error| match error {
ExatokError::Artifact {
artifact,
location,
detail,
fix,
} => ExatokError::Artifact {
artifact,
location: format!("line {line}, {location}"),
detail,
fix,
},
other => other,
})
}
fn resolve_halves(
by_bytes: &HashMap<Vec<u8>, TokenId>,
artifact: &str,
line: usize,
left: &[u8],
right: &[u8],
) -> Result<PairKey, ExatokError> {
let lookup = |half: &[u8], side: &str| -> Result<TokenId, ExatokError> {
by_bytes.get(half).copied().ok_or_else(|| {
line_error(
artifact,
line,
format!(
"{side} half {:?} is not in the vocabulary",
byte_encoder::encode(half)
),
"every merge half must itself be a vocabulary token; the \
vocabulary and the merge list are out of sync",
)
})
};
Ok(PairKey::new(lookup(left, "left")?, lookup(right, "right")?))
}
fn index_tokens(
artifact: &str,
tokens: &[(TokenId, Vec<u8>)],
) -> Result<(Vec<Vec<u8>>, HashMap<Vec<u8>, TokenId>), ExatokError> {
let highest = tokens.iter().map(|(id, _)| id.0).max().unwrap_or(0);
let needed = highest as usize + 1;
let allowed = tokens
.len()
.saturating_mul(MAX_ID_SPARSITY)
.saturating_add(SPARSITY_FLOOR);
if needed > allowed {
return Err(artifact_error(
artifact,
format!("token id {highest}"),
format!(
"{} tokens span {needed} ids, more than the {allowed} an id space \
this dense may occupy",
tokens.len()
),
"check the id column: real vocabularies are dense apart from a few \
reserved holes",
));
}
let mut by_id: Vec<Vec<u8>> = vec![Vec::new(); needed];
let mut by_bytes: HashMap<Vec<u8>, TokenId> = HashMap::with_capacity(tokens.len());
for (id, bytes) in tokens {
match by_bytes.entry(bytes.clone()) {
Entry::Occupied(occupied) => {
return Err(artifact_error(
artifact,
format!("token id {}", id.0),
format!(
"token {:?} is already defined with id {}",
byte_encoder::encode(bytes),
occupied.get().0
),
"each token may appear once; drop the duplicate entry",
));
}
Entry::Vacant(vacant) => {
vacant.insert(*id);
}
}
by_id[id.0 as usize] = bytes.clone();
}
Ok((by_id, by_bytes))
}
impl Vocabulary {
pub fn load_tiktoken(source: Source<'_>) -> Result<Self, ExatokError> {
let mut tokens: Vec<(TokenId, Vec<u8>)> = Vec::new();
let mut rank_lines: HashMap<u32, usize> = HashMap::new();
let mut token_lines: HashMap<Vec<u8>, usize> = HashMap::new();
for (index, line) in source.text.lines().enumerate() {
let number = index + 1;
if line.is_empty() {
return Err(line_error(
source.name,
number,
"line is empty".to_owned(),
"delete the blank line; every line must be `base64(token) rank`",
));
}
let Some((encoded, rank_text)) = line.split_once(' ') else {
return Err(line_error(
source.name,
number,
format!("no space separator in {line:?}, so the rank column is missing"),
"write the line as `base64(token) rank` with exactly one space",
));
};
if rank_text.contains(' ') {
return Err(line_error(
source.name,
number,
format!("more than two columns: rank field is {rank_text:?}"),
"a token line has exactly two space separated columns",
));
}
let token = decode_base64(encoded).map_err(|detail| {
line_error(
source.name,
number,
detail,
"re-export the artifact; the token column must be standard padded base64",
)
})?;
if token.is_empty() {
return Err(line_error(
source.name,
number,
"token decodes to zero bytes".to_owned(),
"remove the entry; a vocabulary token must have at least one byte",
));
}
let rank: u32 = rank_text.parse().map_err(|_| {
line_error(
source.name,
number,
format!("rank {rank_text:?} is not a 32-bit unsigned integer"),
"the rank column must be a plain non-negative decimal integer",
)
})?;
if rank > MAX_TOKEN_ID {
return Err(line_error(
source.name,
number,
format!("rank {rank} exceeds the maximum token id {MAX_TOKEN_ID}"),
"token ids must stay below the reserved device hash-table sentinels",
));
}
if let Some(previous) = rank_lines.insert(rank, number) {
return Err(line_error(
source.name,
number,
format!("rank {rank} was already used on line {previous}"),
"ranks are token ids and must be unique; drop the duplicate entry",
));
}
if let Some(previous) = token_lines.insert(token.clone(), number) {
return Err(line_error(
source.name,
number,
format!(
"token {:?} was already defined on line {previous}",
byte_encoder::encode(&token)
),
"each token may appear once; drop the duplicate entry",
));
}
tokens.push((TokenId(rank), token));
}
if tokens.is_empty() {
return Err(artifact_error(
source.name,
"whole file".to_owned(),
"no token lines found".to_owned(),
"supply a rank file with at least one `base64(token) rank` line",
));
}
let (by_id, by_bytes) = index_tokens(source.name, &tokens)?;
tokens.sort_unstable_by_key(|(id, _)| id.0);
let mut merges = HostMergeTable::new();
for (id, token) in &tokens {
if token.len() < 2 {
continue;
}
let line = token_lines.get(token).copied().unwrap_or(0);
let mut splits = 0usize;
for cut in 1..token.len() {
let (left, right) = token.split_at(cut);
let (Some(&left_id), Some(&right_id)) =
(by_bytes.get(left), by_bytes.get(right))
else {
continue;
};
insert_merge(
&mut merges,
source.name,
line,
PairKey::new(left_id, right_id),
*id,
Rank(id.0),
)?;
splits += 1;
}
if splits == 0 {
return Err(line_error(
source.name,
line,
format!(
"token {:?} has no split whose halves are both in the vocabulary",
byte_encoder::encode(token)
),
"the file is incomplete: a multi-byte token BPE can never build is \
unreachable, so a cheaper token it needs is missing",
));
}
}
Ok(Self {
by_id,
by_bytes,
merges,
specials: Vec::new(),
})
}
pub fn load_tiktoken_path(path: &Path) -> Result<Self, ExatokError> {
let name = path.display().to_string();
let text = read_artifact(&name, path)?;
Self::load_tiktoken(Source::new(&name, &text))
}
pub fn load_huggingface(
vocab: Source<'_>,
merges: Source<'_>,
specials: &[&str],
) -> Result<Self, ExatokError> {
let parsed = parse_vocab_json(vocab)?;
let merge_lines = split_merges_txt(merges)?;
let specials: Vec<(String, Option<TokenId>)> = specials
.iter()
.map(|name| ((*name).to_owned(), None))
.collect();
Self::from_huggingface_parts(vocab.name, &parsed, merges.name, &merge_lines, &specials)
}
pub fn load_tokenizer_json(source: Source<'_>) -> Result<Self, ExatokError> {
let root: serde_json::Value = serde_json::from_str(source.text).map_err(|error| {
artifact_error(
source.name,
format!("line {}, column {}", error.line(), error.column()),
format!("invalid JSON: {error}"),
"repair the JSON; tokenizer.json must be a single JSON object",
)
})?;
let missing = |field: &str| {
artifact_error(
source.name,
format!("field {field}"),
format!("{field} is missing or has the wrong type"),
"supply a BPE tokenizer.json with model.vocab and model.merges",
)
};
let model = root.get("model").ok_or_else(|| missing("model"))?;
let vocab_object = model
.get("vocab")
.and_then(serde_json::Value::as_object)
.ok_or_else(|| missing("model.vocab"))?;
let merge_array = model
.get("merges")
.and_then(serde_json::Value::as_array)
.ok_or_else(|| missing("model.merges"))?;
let mut parsed = Vec::with_capacity(vocab_object.len());
for (token, id) in vocab_object {
parsed.push((token.clone(), vocab_id(source.name, token, id)?));
}
let mut merge_lines = Vec::with_capacity(merge_array.len());
for (index, entry) in merge_array.iter().enumerate() {
let number = index + 1;
let line = match entry {
serde_json::Value::String(text) => text.clone(),
serde_json::Value::Array(halves) => {
let [left, right] = halves.as_slice() else {
return Err(line_error(
source.name,
number,
format!(
"model.merges entry {index} is an array of {} elements, \
expected exactly 2",
halves.len()
),
"an array-form merge must be [\"left\", \"right\"]",
));
};
let (Some(left), Some(right)) = (left.as_str(), right.as_str()) else {
return Err(line_error(
source.name,
number,
format!(
"model.merges entry {index} has a non-string half: \
[{left}, {right}]"
),
"both halves of an array-form merge must be strings",
));
};
format!("{left} {right}")
}
other => {
return Err(line_error(
source.name,
number,
format!(
"model.merges entry {index} is {other}, expected \
\"left right\" or [\"left\", \"right\"]"
),
"each merge must be a space separated string or a two element array",
));
}
};
merge_lines.push((number, line));
}
let mut specials: Vec<(String, Option<TokenId>)> = Vec::new();
if let Some(added) = root.get("added_tokens").and_then(serde_json::Value::as_array) {
for entry in added {
let Some(content) = entry.get("content").and_then(serde_json::Value::as_str)
else {
continue;
};
let declared = match entry.get("id") {
Some(value) => Some(TokenId(vocab_id(source.name, content, value)?)),
None => None,
};
specials.push((content.to_owned(), declared));
}
}
Self::from_huggingface_parts(
source.name,
&parsed,
source.name,
&merge_lines,
&specials,
)
}
fn from_huggingface_parts(
vocab_name: &str,
parsed: &[(String, u32)],
merges_name: &str,
merge_lines: &[(usize, String)],
specials: &[(String, Option<TokenId>)],
) -> Result<Self, ExatokError> {
let mut tokens: Vec<(TokenId, Vec<u8>)> = Vec::with_capacity(parsed.len());
let mut special_vocab_ids: Vec<(String, TokenId)> = Vec::new();
for (token, id) in parsed {
if *id > MAX_TOKEN_ID {
return Err(artifact_error(
vocab_name,
format!("token {token:?}"),
format!("id {id} exceeds the maximum token id {MAX_TOKEN_ID}"),
"token ids must stay below the reserved device hash-table sentinels",
));
}
if specials.iter().any(|(name, _)| name == token) {
special_vocab_ids.push((token.clone(), TokenId(*id)));
continue;
}
let bytes = byte_encoder::decode(token).map_err(|bad| {
artifact_error(
vocab_name,
format!("token {token:?}, character {}", bad.char_index),
format!(
"character {:?} is not in the GPT-2 byte alphabet",
bad.character
),
"token strings must be byte-encoded, so a literal space is `Ä `; \
list a genuine special token in the specials argument instead",
)
})?;
if bytes.is_empty() {
return Err(artifact_error(
vocab_name,
format!("token {token:?}"),
"token decodes to zero bytes".to_owned(),
"remove the entry; a vocabulary token must have at least one byte",
));
}
tokens.push((TokenId(*id), bytes));
}
if tokens.is_empty() {
return Err(artifact_error(
vocab_name,
"whole file".to_owned(),
"no ordinary tokens found".to_owned(),
"vocab.json must map at least one token string to an id",
));
}
let (by_id, by_bytes) = index_tokens(vocab_name, &tokens)?;
let mut merges = HostMergeTable::new();
for (rank, (number, line)) in merge_lines.iter().enumerate() {
let Some((left_text, right_text)) = line.split_once(' ') else {
return Err(line_error(
merges_name,
*number,
format!("no space separator in {line:?}, so the pair is incomplete"),
"write the line as `left right` with exactly one space",
));
};
if right_text.contains(' ') {
return Err(line_error(
merges_name,
*number,
format!("more than two columns: right half is {right_text:?}"),
"a merge line has exactly two space separated halves; \
a literal space inside a half must be written `Ä `",
));
}
let decode_half = |text: &str| -> Result<Vec<u8>, ExatokError> {
byte_encoder::decode(text).map_err(|bad| {
line_error(
merges_name,
*number,
format!(
"character {:?} at position {} of {text:?} is not in the \
GPT-2 byte alphabet",
bad.character, bad.char_index
),
"merge halves must be byte-encoded token strings",
)
})
};
let left = decode_half(left_text)?;
let right = decode_half(right_text)?;
let pair = resolve_halves(&by_bytes, merges_name, *number, &left, &right)?;
let mut merged_bytes = Vec::with_capacity(left.len() + right.len());
merged_bytes.extend_from_slice(&left);
merged_bytes.extend_from_slice(&right);
let Some(&merged) = by_bytes.get(merged_bytes.as_slice()) else {
return Err(line_error(
merges_name,
*number,
format!(
"merge result {:?} is not in the vocabulary",
byte_encoder::encode(&merged_bytes)
),
"vocab.json and merges.txt are out of sync: every merge must \
produce a token that vocab.json defines",
));
};
let rank = u32::try_from(rank).map_err(|_| {
line_error(
merges_name,
*number,
"more merges than a 32-bit rank can number".to_owned(),
"ranks must fit in u32 to reach the device",
)
})?;
insert_merge(
&mut merges,
merges_name,
*number,
pair,
merged,
Rank(rank),
)?;
}
let mut vocabulary = Self {
by_id,
by_bytes,
merges,
specials: Vec::new(),
};
for (name, declared) in specials {
let from_vocab = special_vocab_ids
.iter()
.find(|(other, _)| other == name)
.map(|(_, id)| *id);
let resolved = match (*declared, from_vocab) {
(Some(added), Some(in_vocab)) if added != in_vocab => {
return Err(artifact_error(
vocab_name,
format!("special token {name:?}"),
format!(
"added_tokens gives id {} but the vocabulary gives id {}",
added.0, in_vocab.0
),
"make the two ids agree; a special token has exactly one id",
));
}
(Some(id), _) | (None, Some(id)) => id,
(None, None) => {
return Err(artifact_error(
vocab_name,
format!("special token {name:?}"),
"has no id: absent from the vocabulary and no added_tokens id"
.to_owned(),
"give the special token an id in added_tokens, or name only \
specials that the vocabulary defines",
));
}
};
vocabulary.register_special(name, resolved)?;
}
Ok(vocabulary)
}
pub fn load_tokenizer_json_path(path: &Path) -> Result<Self, ExatokError> {
let name = path.display().to_string();
let text = read_artifact(&name, path)?;
Self::load_tokenizer_json(Source::new(&name, &text))
}
pub fn register_special(&mut self, text: &str, id: TokenId) -> Result<(), ExatokError> {
let location = || format!("special token {text:?}");
if id.0 > MAX_TOKEN_ID {
return Err(artifact_error(
"special tokens",
location(),
format!("id {} exceeds the maximum token id {MAX_TOKEN_ID}", id.0),
"token ids must stay below the reserved device hash-table sentinels",
));
}
if let Some((existing, _)) = self.specials.iter().find(|(name, _)| name == text) {
return Err(artifact_error(
"special tokens",
location(),
format!("{existing:?} is already registered"),
"register each special token once",
));
}
if let Some((name, _)) = self.specials.iter().find(|(_, other)| *other == id) {
return Err(artifact_error(
"special tokens",
location(),
format!("id {} is already held by special token {name:?}", id.0),
"give each special token a distinct id",
));
}
if let Some(bytes) = self.bytes_of(id) {
return Err(artifact_error(
"special tokens",
location(),
format!(
"id {} is already held by ordinary token {:?}",
id.0,
byte_encoder::encode(bytes)
),
"special token ids must not collide with the ordinary vocabulary",
));
}
self.specials.push((text.to_owned(), id));
Ok(())
}
#[must_use]
pub fn len(&self) -> usize {
self.by_bytes.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.by_bytes.is_empty()
}
#[must_use]
pub fn special_len(&self) -> usize {
self.specials.len()
}
#[must_use]
pub fn n_vocab(&self) -> usize {
let highest_special = self.specials.iter().map(|(_, id)| id.0 as usize + 1).max();
self.by_id.len().max(highest_special.unwrap_or(0))
}
#[must_use]
pub fn id_of(&self, bytes: &[u8]) -> Option<TokenId> {
self.by_bytes.get(bytes).copied()
}
#[must_use]
pub fn bytes_of(&self, id: TokenId) -> Option<&[u8]> {
self.by_id
.get(id.0 as usize)
.filter(|bytes| !bytes.is_empty())
.map(Vec::as_slice)
}
#[must_use]
pub fn special_id(&self, text: &str) -> Option<TokenId> {
self.specials
.iter()
.find(|(name, _)| name == text)
.map(|(_, id)| *id)
}
pub fn specials(&self) -> impl Iterator<Item = (&str, TokenId)> {
self.specials.iter().map(|(name, id)| (name.as_str(), *id))
}
#[must_use]
pub const fn merge_table(&self) -> &HostMergeTable {
&self.merges
}
#[must_use]
pub fn merge_count(&self) -> usize {
self.merges.len()
}
#[must_use]
pub fn max_token_id(&self) -> Option<TokenId> {
self.by_id
.iter()
.enumerate()
.rev()
.find(|(_, bytes)| !bytes.is_empty())
.and_then(|(index, _)| u32::try_from(index).ok())
.map(TokenId)
}
}
impl MergeTable for Vocabulary {
fn lookup(&self, pair: PairKey) -> Option<MergeEntry> {
self.merges.lookup(pair)
}
}
const BASE64_INVALID: u8 = 0xFF;
const fn build_base64_reverse() -> [u8; 256] {
let alphabet = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut table = [BASE64_INVALID; 256];
let mut index = 0usize;
while index < alphabet.len() {
table[alphabet[index] as usize] = index as u8;
index += 1;
}
table
}
const BASE64_REVERSE: [u8; 256] = build_base64_reverse();
fn decode_base64(text: &str) -> Result<Vec<u8>, String> {
let bytes = text.as_bytes();
if bytes.is_empty() {
return Err("token column is empty".to_owned());
}
if bytes.len() % 4 != 0 {
return Err(format!(
"base64 length {} is not a multiple of 4",
bytes.len()
));
}
let blocks = bytes.len() / 4;
let mut out = Vec::with_capacity(blocks * 3);
for block in 0..blocks {
let start = block * 4;
let is_last = block + 1 == blocks;
let mut sextets = [0u8; 4];
let mut padding = 0usize;
for slot in 0..4 {
let at = start + slot;
let character = bytes[at];
if character == b'=' {
if !is_last {
return Err(format!(
"base64 padding '=' at index {at} is not in the final block"
));
}
if slot < 2 {
return Err(format!(
"base64 padding '=' at index {at} would leave no data byte"
));
}
padding += 1;
continue;
}
if padding > 0 {
return Err(format!(
"base64 byte 0x{character:02x} at index {at} follows padding"
));
}
let value = BASE64_REVERSE[character as usize];
if value == BASE64_INVALID {
return Err(format!(
"invalid base64 byte 0x{character:02x} at index {at}"
));
}
sextets[slot] = value;
}
let triple = (u32::from(sextets[0]) << 18)
| (u32::from(sextets[1]) << 12)
| (u32::from(sextets[2]) << 6)
| u32::from(sextets[3]);
out.push((triple >> 16) as u8);
if padding < 2 {
out.push((triple >> 8) as u8);
}
if padding < 1 {
out.push(triple as u8);
}
}
Ok(out)
}
fn vocab_id(artifact: &str, token: &str, value: &serde_json::Value) -> Result<u32, ExatokError> {
let Some(number) = value.as_u64() else {
return Err(artifact_error(
artifact,
format!("token {token:?}"),
format!("id {value} is not a non-negative integer"),
"vocab.json must map every token string to a non-negative integer id",
));
};
u32::try_from(number).map_err(|_| {
artifact_error(
artifact,
format!("token {token:?}"),
format!("id {number} does not fit in 32 bits"),
"token ids must fit in u32 to reach the device",
)
})
}
fn parse_vocab_json(source: Source<'_>) -> Result<Vec<(String, u32)>, ExatokError> {
let root: serde_json::Value = serde_json::from_str(source.text).map_err(|error| {
artifact_error(
source.name,
format!("line {}, column {}", error.line(), error.column()),
format!("invalid JSON: {error}"),
"repair the JSON; vocab.json must be one object mapping token strings to ids",
)
})?;
let Some(object) = root.as_object() else {
return Err(artifact_error(
source.name,
"document root".to_owned(),
"top level value is not a JSON object".to_owned(),
"vocab.json must be a single object mapping token strings to integer ids",
));
};
let mut out = Vec::with_capacity(object.len());
for (token, value) in object {
out.push((token.clone(), vocab_id(source.name, token, value)?));
}
Ok(out)
}
fn split_merges_txt(source: Source<'_>) -> Result<Vec<(usize, String)>, ExatokError> {
let mut out = Vec::new();
for (index, line) in source.text.lines().enumerate() {
let number = index + 1;
if number == 1 && line.starts_with("#version") {
continue;
}
if line.is_empty() {
return Err(line_error(
source.name,
number,
"line is empty".to_owned(),
"delete the blank line; every line must be `left right`",
));
}
out.push((number, line.to_owned()));
}
Ok(out)
}
fn read_artifact(name: &str, path: &Path) -> Result<String, ExatokError> {
fs::read_to_string(path).map_err(|error| {
artifact_error(
name,
"file read".to_owned(),
format!("could not read the artifact: {error}"),
"check the path and read permissions",
)
})
}
pub const ARTIFACT_DIR_ENV: &str = "EXATOK_VOCAB_DIR";
pub const ARTIFACT_DIR_DEFAULT: &str = ".cache/exatok-vocab";
#[must_use]
pub fn artifact_dir() -> PathBuf {
match std::env::var_os(ARTIFACT_DIR_ENV) {
Some(dir) => PathBuf::from(dir),
None => {
let home = std::env::var_os("HOME").map_or_else(PathBuf::new, PathBuf::from);
home.join(ARTIFACT_DIR_DEFAULT)
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct NamedVocabulary {
pub name: &'static str,
pub pattern: SplitPattern,
pub tokens: usize,
pub pairs: usize,
pub text_ids: usize,
}
impl NamedVocabulary {
#[must_use]
pub fn artifact_path(self) -> PathBuf {
artifact_dir().join(format!("{}.tiktoken", self.name))
}
}
pub const CL100K_BASE: NamedVocabulary = NamedVocabulary {
name: "cl100k_base",
pattern: SplitPattern::Cl100kBase,
tokens: 100_256,
pairs: 233_378,
text_ids: 1_211,
};
pub const R50K_BASE: NamedVocabulary = NamedVocabulary {
name: "r50k_base",
pattern: SplitPattern::Gpt2,
tokens: 50_256,
pairs: 108_299,
text_ids: 1_500,
};
pub const P50K_BASE: NamedVocabulary = NamedVocabulary {
name: "p50k_base",
pattern: SplitPattern::Gpt2,
tokens: 50_280,
pairs: 108_599,
text_ids: 1_331,
};
pub const O200K_BASE: NamedVocabulary = NamedVocabulary {
name: "o200k_base",
pattern: SplitPattern::O200kBase,
tokens: 199_998,
pairs: 446_189,
text_ids: 1_015,
};
pub const NAMED_VOCABULARIES: [NamedVocabulary; 4] =
[CL100K_BASE, R50K_BASE, P50K_BASE, O200K_BASE];
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Artifact {
Present(PathBuf),
NoCache {
dir: PathBuf,
path: PathBuf,
},
Incomplete {
dir: PathBuf,
path: PathBuf,
},
}
impl Artifact {
#[must_use]
pub fn report(&self) -> String {
match self {
Self::Present(path) => format!("rank artifact present: {}", path.display()),
Self::NoCache { dir, path } => format!(
"nothing is provisioned: {} holds no tiktoken rank table at all, and none is at \
{}. Populate that directory, creating it if it is absent, or set \
{ARTIFACT_DIR_ENV} to a directory that already holds the tiktoken rank tables.",
dir.display(),
path.display()
),
Self::Incomplete { dir, path } => format!(
"the vocabulary cache is incomplete rather than absent: there is no usable rank \
artifact at {}, where a zero-length file counts as none. Fix: re-fetch that one \
artifact into {}, or point {ARTIFACT_DIR_ENV} at a complete directory.",
path.display(),
dir.display()
),
}
}
}
fn usable_artifact(path: &Path) -> bool {
std::fs::metadata(path).is_ok_and(|found| found.is_file() && found.len() > 0)
}
#[must_use]
pub fn locate_in(dir: &Path, vocabulary: NamedVocabulary) -> Artifact {
let path = dir.join(format!("{}.tiktoken", vocabulary.name));
if usable_artifact(&path) {
return Artifact::Present(path);
}
let populated = NAMED_VOCABULARIES
.iter()
.any(|named| dir.join(format!("{}.tiktoken", named.name)).exists());
let dir = dir.to_path_buf();
if populated {
Artifact::Incomplete { dir, path }
} else {
Artifact::NoCache { dir, path }
}
}
#[must_use]
pub fn locate(vocabulary: NamedVocabulary) -> Artifact {
locate_in(&artifact_dir(), vocabulary)
}
pub fn load(vocabulary: NamedVocabulary, path: &Path) -> Result<Vocabulary, ExatokError> {
let loaded = Vocabulary::load_tiktoken_path(path)?;
let mismatch = |location: &str, detail: String| ExatokError::Artifact {
artifact: vocabulary.name.to_owned(),
location: location.to_owned(),
detail,
fix: "load the published rank file for this vocabulary, and derive a pair for EVERY \
split whose halves are both in the vocabulary, not only the canonical one"
.to_owned(),
};
if loaded.len() != vocabulary.tokens {
return Err(mismatch(
"token count",
format!("expected {}, loaded {}", vocabulary.tokens, loaded.len()),
));
}
if loaded.merge_count() != vocabulary.pairs {
return Err(mismatch(
"derived pair count",
format!(
"expected {}, derived {}",
vocabulary.pairs,
loaded.merge_count()
),
));
}
Ok(loaded)
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
#[test]
fn base64_decodes_known_vectors() {
assert_eq!(decode_base64("IQ=="), Ok(b"!".to_vec()));
assert_eq!(decode_base64("YQ=="), Ok(b"a".to_vec()));
assert_eq!(decode_base64("Yg=="), Ok(b"b".to_vec()));
assert_eq!(decode_base64("Yw=="), Ok(b"c".to_vec()));
assert_eq!(decode_base64("IA=="), Ok(b" ".to_vec()));
assert_eq!(decode_base64("YWI="), Ok(b"ab".to_vec()));
assert_eq!(decode_base64("IGE="), Ok(b" a".to_vec()));
assert_eq!(decode_base64("IGFi"), Ok(b" ab".to_vec()));
assert_eq!(decode_base64("IENvbnZleW9y"), Ok(b" Conveyor".to_vec()));
}
#[test]
fn base64_round_trips_all_256_single_byte_tokens() {
const ALPHABET: &[u8; 64] =
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
for byte in 0..=u8::MAX {
let encoded = format!(
"{}{}==",
ALPHABET[(byte >> 2) as usize] as char,
ALPHABET[((byte & 0b11) << 4) as usize] as char,
);
assert_eq!(
decode_base64(&encoded),
Ok(vec![byte]),
"byte 0x{byte:02x} encoded as {encoded}"
);
}
}
#[test]
fn base64_rejects_malformed_input() {
assert!(decode_base64("").is_err(), "empty");
assert!(decode_base64("YQ=").is_err(), "length 3");
assert!(decode_base64("YQ").is_err(), "length 2");
assert!(decode_base64("YQ===").is_err(), "length 5");
assert!(decode_base64("Y!==").is_err(), "invalid character");
assert!(decode_base64("YQ==YQ==").is_err(), "padding mid stream");
assert!(decode_base64("====").is_err(), "padding in slot 0");
assert!(decode_base64("Y===").is_err(), "padding in slot 1");
assert!(decode_base64("YQ=A").is_err(), "data after padding");
let message = decode_base64("Y!==").unwrap_err();
assert!(
message.contains("index 1"),
"error must locate the bad byte, got {message:?}"
);
}
#[test]
fn merges_header_is_only_skipped_on_line_one() {
let parsed =
split_merges_txt(Source::new("merges.txt", "#version: 0.2\na b\n# Ä \n")).unwrap();
assert_eq!(
parsed,
vec![(2, "a b".to_owned()), (3, "# Ä ".to_owned())],
"only line 1 is a header; line 3 is a merge whose left half is '#'"
);
let no_header = split_merges_txt(Source::new("merges.txt", "a b\n")).unwrap();
assert_eq!(no_header, vec![(1, "a b".to_owned())]);
let later_version =
split_merges_txt(Source::new("m.txt", "a b\n#version: 0.2\n")).unwrap();
assert_eq!(
later_version.len(),
2,
"a #version line below line 1 is data, not a header"
);
}
#[test]
fn blank_merge_line_is_reported_with_its_number() {
let error = split_merges_txt(Source::new("merges.txt", "a b\n\nc d\n")).unwrap_err();
let ExatokError::Artifact { location, .. } = &error else {
panic!("expected an Artifact error, got {error:?}");
};
assert_eq!(location, "line 2");
}
#[test]
fn artifact_states_distinguish_absent_incomplete_and_present() {
let dir = std::env::temp_dir().join("exatok-vocab-artifact-states");
let _ = std::fs::remove_dir_all(&dir);
let file = dir.join("cl100k_base.tiktoken");
let sibling = dir.join("r50k_base.tiktoken");
let no_directory = locate_in(&dir, CL100K_BASE);
std::fs::create_dir_all(&dir).unwrap();
let empty_directory = locate_in(&dir, CL100K_BASE);
std::fs::write(&sibling, "YQ== 0\n").unwrap();
let sibling_only = locate_in(&dir, CL100K_BASE);
std::fs::write(&file, "").unwrap();
let zero_length = locate_in(&dir, CL100K_BASE);
std::fs::write(&file, "YQ== 0\n").unwrap();
let present = locate_in(&dir, CL100K_BASE);
let _ = std::fs::remove_dir_all(&dir);
let absent = Artifact::NoCache {
dir: dir.clone(),
path: file.clone(),
};
let partial = Artifact::Incomplete {
dir: dir.clone(),
path: file.clone(),
};
assert_eq!(no_directory, absent);
assert_eq!(
empty_directory, absent,
"a directory that holds no rank table was never provisioned, so it is not partial"
);
assert_eq!(
sibling_only, partial,
"a directory holding one rank table and not this one is partial"
);
assert_eq!(
zero_length, partial,
"an empty rank file is a broken cache, not an absent one"
);
assert_eq!(present, Artifact::Present(file));
for report in [absent.report(), partial.report()] {
assert!(
report.contains("cl100k_base.tiktoken") && report.contains(ARTIFACT_DIR_ENV),
"a report must name the path and the override, got {report}"
);
}
}
#[test]
fn artifact_reports_describe_the_artifact_and_not_a_gate() {
let dir = std::env::temp_dir().join("exatok-vocab-report-wording");
let _ = std::fs::remove_dir_all(&dir);
let absent = locate_in(&dir, CL100K_BASE).report();
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("r50k_base.tiktoken"), "YQ== 0\n").unwrap();
let broken = locate_in(&dir, CL100K_BASE).report();
let _ = std::fs::remove_dir_all(&dir);
for report in [&absent, &broken] {
for forbidden in ["SKIPPED", "skipped", "gate", "parity"] {
assert!(
!report.contains(forbidden),
"an artifact report must not speak as a test harness, found {forbidden:?} \
in: {report}"
);
}
assert!(
report.contains("Fix:") || report.contains("Populate"),
"a report must say what to do about it, got: {report}"
);
}
}
#[test]
fn named_vocabularies_are_distinct_and_their_counts_are_pinned() {
let mut names: Vec<&str> = NAMED_VOCABULARIES.iter().map(|v| v.name).collect();
names.sort_unstable();
let count = names.len();
names.dedup();
assert_eq!(names.len(), count, "two descriptors share a name: {names:?}");
assert_eq!(
names,
vec!["cl100k_base", "o200k_base", "p50k_base", "r50k_base"],
"the shipped set changed"
);
assert_eq!(CL100K_BASE.tokens, 100_256);
assert_eq!(CL100K_BASE.pairs, 233_378);
assert_eq!(CL100K_BASE.text_ids, 1_211);
assert_eq!(R50K_BASE.tokens, 50_256);
assert_eq!(R50K_BASE.pairs, 108_299);
assert_eq!(P50K_BASE.tokens, 50_280);
assert_eq!(P50K_BASE.pairs, 108_599);
assert_eq!(O200K_BASE.tokens, 199_998);
assert_eq!(O200K_BASE.pairs, 446_189);
assert!(
CL100K_BASE.artifact_path().ends_with("cl100k_base.tiktoken"),
"the artifact path must name the stem"
);
}
#[test]
fn the_artifact_dir_override_takes_precedence_over_home() {
assert_eq!(ARTIFACT_DIR_ENV, "EXATOK_VOCAB_DIR");
assert_eq!(ARTIFACT_DIR_DEFAULT, ".cache/exatok-vocab");
let dir = artifact_dir();
match std::env::var_os(ARTIFACT_DIR_ENV) {
Some(override_dir) => assert_eq!(
dir,
PathBuf::from(override_dir),
"the override must be used verbatim"
),
None => assert!(
dir.ends_with(ARTIFACT_DIR_DEFAULT),
"without the override the path must end with the default cache, got {}",
dir.display()
),
}
}
}