use crate::vocab::Vocabulary;
use crate::{TokenId, ExatokError};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct NamedSpecials {
pub name: &'static str,
pub tokens: &'static [(&'static str, u32)],
pub n_vocab: usize,
pub special_topped: bool,
}
pub const CL100K_BASE_SPECIALS: NamedSpecials = NamedSpecials {
name: "cl100k_base",
tokens: &[
("<|endoftext|>", 100_257),
("<|fim_prefix|>", 100_258),
("<|fim_middle|>", 100_259),
("<|fim_suffix|>", 100_260),
("<|endofprompt|>", 100_276),
],
n_vocab: 100_277,
special_topped: true,
};
pub const R50K_BASE_SPECIALS: NamedSpecials = NamedSpecials {
name: "r50k_base",
tokens: &[("<|endoftext|>", 50_256)],
n_vocab: 50_257,
special_topped: true,
};
pub const P50K_BASE_SPECIALS: NamedSpecials = NamedSpecials {
name: "p50k_base",
tokens: &[("<|endoftext|>", 50_256)],
n_vocab: 50_281,
special_topped: false,
};
pub const O200K_BASE_SPECIALS: NamedSpecials = NamedSpecials {
name: "o200k_base",
tokens: &[("<|endoftext|>", 199_999), ("<|endofprompt|>", 200_018)],
n_vocab: 200_019,
special_topped: true,
};
pub const NAMED_SPECIALS: [NamedSpecials; 4] = [
CL100K_BASE_SPECIALS,
R50K_BASE_SPECIALS,
P50K_BASE_SPECIALS,
O200K_BASE_SPECIALS,
];
#[must_use]
pub fn specials_for(name: &str) -> Option<NamedSpecials> {
NAMED_SPECIALS
.into_iter()
.find(|specials| specials.name == name)
}
pub fn register(vocabulary: &mut Vocabulary, name: &str) -> Result<(), ExatokError> {
let Some(specials) = specials_for(name) else {
return Ok(());
};
for (text, id) in specials.tokens {
vocabulary.register_special(text, TokenId(*id))?;
}
Ok(())
}
impl NamedSpecials {
#[must_use]
pub fn allowed(&self) -> Vec<(&'static str, TokenId)> {
self.tokens
.iter()
.map(|(text, id)| (*text, TokenId(*id)))
.collect()
}
#[must_use]
pub fn partition(
&self,
allow: &[&str],
) -> (Vec<(&'static str, TokenId)>, Vec<(&'static str, TokenId)>) {
self.tokens
.iter()
.map(|(text, id)| (*text, TokenId(*id)))
.partition(|(text, _)| allow.contains(text))
}
}
fn next_special<'t>(
input: &[u8],
from: usize,
candidates: &[(&'t str, TokenId)],
) -> Option<(usize, &'t str, TokenId)> {
(from..input.len()).find_map(|start| {
candidates
.iter()
.filter(|(text, _)| input[start..].starts_with(text.as_bytes()))
.max_by_key(|(text, _)| text.len())
.map(|(text, id)| (start, *text, *id))
})
}
fn next_allowed_special(
input: &[u8],
from: usize,
allowed: &[(&str, TokenId)],
) -> Option<(usize, usize, TokenId)> {
next_special(input, from, allowed)
.map(|(start, text, id)| (start, start + text.len(), id))
}
#[must_use]
pub fn encode_allowed(
pattern: crate::pretokenize::SplitPattern,
input: &[u8],
bytes: &[TokenId; 256],
table: &impl crate::reference::MergeTable,
allowed: &[(&str, TokenId)],
) -> Vec<TokenId> {
let mut out = Vec::new();
let mut at = 0;
while let Some((start, end, id)) = next_allowed_special(input, at, allowed) {
if start > at {
out.extend(crate::tokenizer::encode_bytes(
pattern,
&input[at..start],
bytes,
table,
));
}
out.push(id);
at = end;
}
if at < input.len() {
out.extend(crate::tokenizer::encode_bytes(
pattern,
&input[at..],
bytes,
table,
));
}
out
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DisallowedSpecial {
pub text: String,
pub id: TokenId,
pub at: usize,
}
impl std::fmt::Display for DisallowedSpecial {
fn fmt(&self, out: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
out,
"input carries disallowed special token {:?} (id {}) at byte offset {}. Fix: move it \
into `allowed` to encode it as that id, drop it from `disallowed` to encode it as \
ordinary text, or reject this input",
self.text, self.id.0, self.at
)
}
}
impl std::error::Error for DisallowedSpecial {}
pub fn encode_checked(
pattern: crate::pretokenize::SplitPattern,
input: &[u8],
bytes: &[TokenId; 256],
table: &impl crate::reference::MergeTable,
allowed: &[(&str, TokenId)],
disallowed: &[(&str, TokenId)],
) -> Result<Vec<TokenId>, DisallowedSpecial> {
if let Some((at, text, id)) = next_special(input, 0, disallowed) {
return Err(DisallowedSpecial {
text: text.to_owned(),
id,
at,
});
}
Ok(encode_allowed(pattern, input, bytes, table, allowed))
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
#[test]
fn every_named_vocabulary_has_a_specials_entry() {
for vocabulary in crate::vocab::NAMED_VOCABULARIES {
assert!(
specials_for(vocabulary.name).is_some(),
"{} has no specials entry",
vocabulary.name
);
}
assert_eq!(NAMED_SPECIALS.len(), crate::vocab::NAMED_VOCABULARIES.len());
}
#[test]
fn special_ids_sit_below_n_vocab_and_top_it_exactly_where_they_should() {
for specials in NAMED_SPECIALS {
let highest = specials
.tokens
.iter()
.map(|(_, id)| *id as usize)
.max()
.expect("every entry has at least one special");
assert!(
highest < specials.n_vocab,
"{}: special {highest} is at or above n_vocab {}, so it is unrepresentable",
specials.name,
specials.n_vocab
);
if specials.special_topped {
assert_eq!(
highest + 1,
specials.n_vocab,
"{}: specials top the range, so the highest must be n_vocab - 1",
specials.name
);
} else {
assert!(
highest + 1 < specials.n_vocab,
"{}: specials do not top the range, so n_vocab must exceed the highest \
special by more than one",
specials.name
);
}
}
assert_eq!(
NAMED_SPECIALS
.iter()
.filter(|specials| !specials.special_topped)
.map(|specials| specials.name)
.collect::<Vec<&str>>(),
vec!["p50k_base"],
"p50k_base is the only vocabulary whose specials do not top the id range"
);
}
#[test]
fn special_ids_and_texts_are_unique_within_each_vocabulary() {
for specials in NAMED_SPECIALS {
let mut ids: Vec<u32> = specials.tokens.iter().map(|(_, id)| *id).collect();
let count = ids.len();
ids.sort_unstable();
ids.dedup();
assert_eq!(ids.len(), count, "{} has a duplicate id", specials.name);
let mut texts: Vec<&str> = specials.tokens.iter().map(|(text, _)| *text).collect();
texts.sort_unstable();
texts.dedup();
assert_eq!(texts.len(), count, "{} has a duplicate text", specials.name);
}
}
#[test]
fn every_special_is_delimited() {
for specials in NAMED_SPECIALS {
for (text, _) in specials.tokens {
assert!(
text.starts_with("<|") && text.ends_with("|>"),
"{}: {text} is not delimited",
specials.name
);
}
}
}
#[test]
fn an_unknown_name_registers_nothing() {
let mut vocabulary = Vocabulary::default();
register(&mut vocabulary, "not_a_vocabulary").unwrap();
assert_eq!(vocabulary.special_len(), 0);
}
#[test]
fn no_declared_special_is_a_prefix_of_another() {
for specials in NAMED_SPECIALS {
for (outer, _) in specials.tokens {
for (inner, _) in specials.tokens {
if outer == inner {
continue;
}
assert!(
!outer.starts_with(inner),
"{}: {inner} is a prefix of {outer}, so leftmost-longest and \
leftmost-first disagree",
specials.name
);
}
}
}
}
#[test]
fn allowed_carries_every_declared_special_with_its_id() {
assert_eq!(
CL100K_BASE_SPECIALS.allowed(),
vec![
("<|endoftext|>", TokenId(100_257)),
("<|fim_prefix|>", TokenId(100_258)),
("<|fim_middle|>", TokenId(100_259)),
("<|fim_suffix|>", TokenId(100_260)),
("<|endofprompt|>", TokenId(100_276)),
]
);
assert_eq!(
P50K_BASE_SPECIALS.allowed(),
vec![("<|endoftext|>", TokenId(50_256))]
);
for specials in NAMED_SPECIALS {
assert_eq!(
specials.allowed().len(),
specials.tokens.len(),
"{}: allow-list must not change length",
specials.name
);
}
}
#[test]
fn partition_splits_the_declared_specials_into_exact_disjoint_lists() {
let (allowed, disallowed) =
CL100K_BASE_SPECIALS.partition(&["<|fim_prefix|>", "<|endofprompt|>"]);
assert_eq!(
allowed,
vec![
("<|fim_prefix|>", TokenId(100_258)),
("<|endofprompt|>", TokenId(100_276)),
]
);
assert_eq!(
disallowed,
vec![
("<|endoftext|>", TokenId(100_257)),
("<|fim_middle|>", TokenId(100_259)),
("<|fim_suffix|>", TokenId(100_260)),
],
"everything not named must land in the disallow-list, in declaration order"
);
}
#[test]
fn partition_at_both_extremes_puts_every_special_on_the_expected_side() {
for specials in NAMED_SPECIALS {
let (none_allowed, all_disallowed) = specials.partition(&[]);
assert_eq!(
none_allowed,
vec![],
"{}: an empty allow-list must allow nothing",
specials.name
);
assert_eq!(
all_disallowed,
specials.allowed(),
"{}: an empty allow-list must disallow every declared special",
specials.name
);
let every: Vec<&str> = specials.tokens.iter().map(|(text, _)| *text).collect();
let (all_allowed, none_disallowed) = specials.partition(&every);
assert_eq!(
all_allowed,
specials.allowed(),
"{}: naming every special must allow every special",
specials.name
);
assert_eq!(
none_disallowed,
vec![],
"{}: naming every special must leave nothing disallowed",
specials.name
);
}
}
#[test]
fn partition_ignores_a_name_the_vocabulary_does_not_declare() {
let (allowed, disallowed) = CL100K_BASE_SPECIALS.partition(&["<|endoftext>", "<|nope|>"]);
assert_eq!(allowed, vec![], "no undeclared name may become allowed");
assert_eq!(
disallowed,
CL100K_BASE_SPECIALS.allowed(),
"a typo must leave every real special disallowed"
);
}
#[test]
fn the_scanner_reports_the_matched_text_with_its_offset_and_id() {
let candidates = [("<|b|>", TokenId(2)), ("<|a|>", TokenId(1))];
assert_eq!(
next_special(b"zz<|b|>yy<|a|>", 0, &candidates),
Some((2, "<|b|>", TokenId(2)))
);
assert_eq!(
next_special(b"zz<|b|>yy<|a|>", 7, &candidates),
Some((9, "<|a|>", TokenId(1))),
"resuming past the first match must report the second, not the first"
);
let prefix_pair = [("<|x|>", TokenId(1)), ("<|x|>y", TokenId(2))];
assert_eq!(
next_special(b"<|x|>y", 0, &prefix_pair),
Some((0, "<|x|>y", TokenId(2))),
"the longest match at one offset must report its own text"
);
assert_eq!(next_special(b"nothing", 0, &candidates), None);
}
#[test]
fn the_scanner_reports_the_leftmost_special_not_the_first_listed() {
let allowed = [("<|b|>", TokenId(2)), ("<|a|>", TokenId(1))];
assert_eq!(
next_allowed_special(b"<|a|>xx<|b|>", 0, &allowed),
Some((0, 5, TokenId(1)))
);
assert_eq!(
next_allowed_special(b"zz<|b|>yy<|a|>", 0, &allowed),
Some((2, 7, TokenId(2)))
);
}
#[test]
fn the_scanner_prefers_the_longest_match_at_one_position() {
let allowed = [("<|x|>", TokenId(1)), ("<|x|>y", TokenId(2))];
assert_eq!(
next_allowed_special(b"<|x|>y", 0, &allowed),
Some((0, 6, TokenId(2))),
"the 6-byte special must beat its own 5-byte prefix"
);
assert_eq!(
next_allowed_special(b"<|x|>z", 0, &allowed),
Some((0, 5, TokenId(1))),
"without the trailing y only the short special matches"
);
}
#[test]
fn the_scanner_honours_the_start_offset_and_reports_absence() {
let allowed = [("<|a|>", TokenId(1))];
assert_eq!(
next_allowed_special(b"<|a|>tail", 0, &allowed),
Some((0, 5, TokenId(1)))
);
assert_eq!(next_allowed_special(b"<|a|>tail", 5, &allowed), None);
assert_eq!(next_allowed_special(b"nothing here", 0, &allowed), None);
assert_eq!(next_allowed_special(b"", 0, &allowed), None);
assert_eq!(
next_allowed_special(b"<|a|>", 0, &[]),
None,
"an empty allow-list recognizes nothing"
);
}
#[test]
fn a_truncated_special_is_not_matched() {
let allowed = CL100K_BASE_SPECIALS.allowed();
assert_eq!(next_allowed_special(b"<|endoftext|", 0, &allowed), None);
assert_eq!(next_allowed_special(b"<|endoftext", 0, &allowed), None);
assert_eq!(
next_allowed_special(b"<|endoftext|>", 0, &allowed),
Some((0, 13, TokenId(100_257))),
"the complete form is 13 bytes and must match"
);
}
}