#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use std::process::Command;
use exatok::parity;
use exatok::pretokenize::pretokens;
use exatok::specials::specials_for;
use exatok::vocab::{
self, Artifact, CL100K_BASE, NamedVocabulary, O200K_BASE, P50K_BASE, R50K_BASE, Vocabulary,
};
use exatok::{Encoding, TokenId, Tokenizer};
const ENCODINGS: [NamedVocabulary; 4] = [CL100K_BASE, R50K_BASE, P50K_BASE, O200K_BASE];
struct Loaded {
vocab: Vocabulary,
tokenizer: Tokenizer,
bytes: [TokenId; 256],
}
const ALLOW_UNPROVISIONED: &str = "EXATOK_ALLOW_UNPROVISIONED";
fn unprovisioned_is_allowed() -> bool {
std::env::var_os(ALLOW_UNPROVISIONED).is_some_and(|value| value == "1")
}
fn load(encoding: &NamedVocabulary) -> Option<Loaded> {
let located = vocab::locate(*encoding);
let path = match &located {
Artifact::Present(path) => path.clone(),
Artifact::Incomplete { .. } => panic!("{}", located.report()),
Artifact::NoCache { .. } => {
assert!(
unprovisioned_is_allowed(),
"{}\nFix: this suite pins every public route to a token id against the \
full-rescan oracle and must not be skipped past. Populate {} or point {} at a \
directory holding the rank tables. Set {}=1 only to declare a deliberately \
unprovisioned machine, which makes these comparisons skip and report nothing.",
located.report(),
vocab::artifact_dir().display(),
vocab::ARTIFACT_DIR_ENV,
ALLOW_UNPROVISIONED
);
eprintln!("{}", located.report());
return None;
}
};
let vocab = vocab::load(*encoding, &path).expect("the pinned counts must hold");
let bytes = exatok::byte_token_ids(encoding.name, &vocab).expect("every byte must resolve");
let tokenizer = Tokenizer::new(vocab.clone(), Encoding::new(encoding.name, encoding.pattern))
.unwrap_or_else(|error| panic!("{} must carry every byte: {error}", encoding.name));
Some(Loaded {
vocab,
tokenizer,
bytes,
})
}
fn all_cases() -> Vec<Vec<u8>> {
let mut cases: Vec<Vec<u8>> = parity::corpus()
.into_iter()
.map(|case| case.text.into_bytes())
.collect();
cases.extend(parity::byte_corpus().into_iter().map(|case| case.bytes));
cases
}
#[test]
fn every_host_route_produces_identical_ids_on_the_whole_corpus() {
let cases = all_cases();
let mut compared = 0_usize;
for encoding in &ENCODINGS {
let Some(loaded) = load(encoding) else {
continue;
};
let table = loaded.vocab.merge_table();
for (index, case) in cases.iter().enumerate() {
let oracle = parity::encode_bytes(encoding.pattern, case, &loaded.bytes, table);
let shipping = loaded.tokenizer.encode(case);
let free = exatok::tokenizer::encode_bytes(
encoding.pattern,
case,
&loaded.bytes,
table,
);
assert_eq!(
shipping, oracle,
"{} case {index}: Tokenizer::encode diverged from the full-rescan oracle on \
{case:?}",
encoding.name
);
assert_eq!(
free, oracle,
"{} case {index}: tokenizer::encode_bytes diverged from the full-rescan oracle \
on {case:?}",
encoding.name
);
let mut joined = Vec::new();
for span in pretokens(encoding.pattern, case) {
let pretoken = span
.bytes(case)
.expect("a span this module produced must lie inside its own input");
let start: Vec<TokenId> = pretoken
.iter()
.map(|byte| loaded.bytes[*byte as usize])
.collect();
joined.extend(exatok::reference::encode_pretoken(&start, table));
}
assert_eq!(
joined, oracle,
"{} case {index}: the per-pretoken reference loop joined by hand diverged, which \
means a merge crossed a pretoken boundary on {case:?}",
encoding.name
);
compared += 1;
}
}
assert!(
compared == 0 || compared >= 200,
"either no artifacts are present or the whole corpus ran; a partial count means a \
vocabulary was skipped silently, got {compared}"
);
if compared == 0 {
eprintln!("no vocabulary artifacts present, route agreement not exercised");
}
}
fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
!needle.is_empty() && haystack.windows(needle.len()).any(|window| window == needle)
}
#[test]
fn the_specials_wrapper_equals_the_plain_route_when_no_special_is_present() {
let cases = all_cases();
let mut compared = 0_usize;
for encoding in &ENCODINGS {
let Some(loaded) = load(encoding) else {
continue;
};
let named = specials_for(encoding.name)
.unwrap_or_else(|| panic!("{} must declare its specials", encoding.name));
let allowed = named.allowed();
assert!(
!allowed.is_empty(),
"{} declares at least one special, so allowing none would make this test vacuous",
encoding.name
);
for (index, case) in cases.iter().enumerate() {
if allowed
.iter()
.any(|(text, _)| contains_subslice(case, text.as_bytes()))
{
continue;
}
let with_specials = exatok::specials::encode_allowed(
encoding.pattern,
case,
&loaded.bytes,
loaded.vocab.merge_table(),
&allowed,
);
assert_eq!(
with_specials,
loaded.tokenizer.encode(case),
"{} case {index}: the specials wrapper diverged from the plain route on an input \
with no special token: {case:?}",
encoding.name
);
compared += 1;
}
}
if compared == 0 {
eprintln!("no vocabulary artifacts present, the specials wrapper was not exercised");
} else {
let per_vocab = compared / ENCODINGS.len().max(1);
assert!(
per_vocab >= 38,
"only {per_vocab} of the {} corpus cases are special-free, so the skip filter is \
swallowing the corpus rather than excluding a handful of cases",
cases.len()
);
}
}
#[test]
fn a_declared_special_becomes_one_id_and_the_plain_route_disagrees() {
let Some(loaded) = load(&CL100K_BASE) else {
return;
};
let named = specials_for("cl100k_base").expect("cl100k_base declares specials");
let allowed = named.allowed();
let endoftext = allowed
.iter()
.find(|(text, _)| *text == "<|endoftext|>")
.map(|(_, id)| *id)
.expect("cl100k_base declares <|endoftext|>");
assert_eq!(
endoftext,
TokenId(100_257),
"the cl100k_base end-of-text id is 100257"
);
let input = b"a<|endoftext|>b";
let with_specials = exatok::specials::encode_allowed(
CL100K_BASE.pattern,
input,
&loaded.bytes,
loaded.vocab.merge_table(),
&allowed,
);
let plain = loaded.tokenizer.encode(input);
assert_eq!(
with_specials,
vec![
loaded.tokenizer.encode(b"a")[0],
endoftext,
loaded.tokenizer.encode(b"b")[0],
],
"the special must collapse to one id between the ids of the surrounding text"
);
assert_ne!(
with_specials, plain,
"the plain route must encode the special's literal bytes, so the two routes have to \
differ here; if they agree, special handling has been lost"
);
assert!(
!plain.contains(&endoftext),
"the plain route must never emit a special id: {plain:?}"
);
}
#[test]
fn every_case_round_trips_through_decode_byte_for_byte() {
let cases = all_cases();
for encoding in &ENCODINGS {
let Some(loaded) = load(encoding) else {
continue;
};
for (index, case) in cases.iter().enumerate() {
let ids = loaded.tokenizer.encode(case);
let decoded = loaded.tokenizer.decode(&ids).unwrap_or_else(|error| {
panic!(
"{} case {index}: decode rejected ids the encoder produced: {error}",
encoding.name
)
});
assert_eq!(
decoded, *case,
"{} case {index}: round trip changed the bytes",
encoding.name
);
}
}
}
#[test]
fn decoding_an_undefined_id_is_an_error_that_names_the_id() {
let Some(loaded) = load(&CL100K_BASE) else {
return;
};
let bogus = TokenId(1_000_000);
let error = loaded
.tokenizer
.decode(&[TokenId(9906), bogus, TokenId(1917)])
.expect_err("an undefined id must not decode");
let text = error.to_string();
assert!(
text.contains("1000000"),
"the error must name the offending id, got: {text}"
);
assert!(
loaded
.tokenizer
.decode(&[TokenId(9906), TokenId(1917)])
.is_ok(),
"the surrounding ids are valid, so removing the bogus one must decode"
);
}
#[test]
fn the_binary_prints_exactly_the_library_ids() {
let Some(loaded) = load(&CL100K_BASE) else {
return;
};
for (index, case) in parity::corpus().into_iter().take(12).enumerate() {
let output = Command::new(env!("CARGO_BIN_EXE_exatok"))
.args(["encode", "--vocab", "cl100k_base", "--text", &case.text])
.output()
.expect("the binary must be runnable");
assert!(
output.status.success(),
"case {index}: the binary failed: {}",
String::from_utf8_lossy(&output.stderr)
);
let printed: Vec<TokenId> = String::from_utf8(output.stdout)
.expect("id output must be UTF-8")
.split_whitespace()
.map(|field| TokenId(field.parse().expect("each id field must be a u32")))
.collect();
assert_eq!(
printed,
loaded.tokenizer.encode(case.text.as_bytes()),
"case {index}: the binary printed different ids than the library returned for {:?}",
case.text
);
}
}
#[test]
fn the_path_loader_and_the_named_loader_agree() {
let located = vocab::locate(CL100K_BASE);
let path = match &located {
Artifact::Present(path) => path.clone(),
Artifact::Incomplete { .. } => panic!("{}", located.report()),
Artifact::NoCache { .. } => {
assert!(
unprovisioned_is_allowed(),
"{}\nFix: the path loader is the route a consumer with their own rank file \
takes, and it must not be skipped past. Populate {} or point {} at a directory \
holding the rank tables. Set {}=1 only to declare a deliberately unprovisioned \
machine, which makes this comparison skip and report nothing.",
located.report(),
vocab::artifact_dir().display(),
vocab::ARTIFACT_DIR_ENV,
ALLOW_UNPROVISIONED
);
eprintln!("{}", located.report());
return;
}
};
let Some(loaded) = load(&CL100K_BASE) else {
return;
};
let output = Command::new(env!("CARGO_BIN_EXE_exatok"))
.args([
"encode",
"--vocab-path",
&path.display().to_string(),
"--pattern",
"cl100k-base",
"--text",
"the quick brown fox jumps over the lazy dog",
])
.output()
.expect("the binary must be runnable");
assert!(
output.status.success(),
"the path loader failed: {}",
String::from_utf8_lossy(&output.stderr)
);
let printed: Vec<TokenId> = String::from_utf8(output.stdout)
.expect("id output must be UTF-8")
.split_whitespace()
.map(|field| TokenId(field.parse().expect("each id field must be a u32")))
.collect();
assert_eq!(
printed,
loaded
.tokenizer
.encode(b"the quick brown fox jumps over the lazy dog"),
"the path loader produced different ids than the named loader"
);
}
const LIBRARY_ENTRY_POINTS: usize = 6;
const CLI_CASES: usize = 12;
const ENTRY_POINT_INVOCATIONS: usize = LIBRARY_ENTRY_POINTS * ENCODINGS.len() + 1;
const API_SHAPE_IDS_TOTAL: usize = 32696;
const DECODED_BYTES_TOTAL: usize = 15908;
#[test]
fn the_suite_invokes_a_pinned_set_of_entry_points_over_a_pinned_id_total() {
let loaded: Vec<(&NamedVocabulary, Loaded)> = ENCODINGS
.iter()
.filter_map(|encoding| load(encoding).map(|entry| (encoding, entry)))
.collect();
if loaded.is_empty() {
assert!(
unprovisioned_is_allowed(),
"no vocabulary loaded and {ALLOW_UNPROVISIONED} is unset, so this suite invoked \
zero public entry points while reporting success. Fix: populate {} or set {}=1 to \
declare a bare machine.",
vocab::artifact_dir().display(),
ALLOW_UNPROVISIONED
);
return;
}
assert_eq!(
loaded.len(),
ENCODINGS.len(),
"a provisioned box carries all four vocabularies, so a partial set means the routes are \
not being driven against one of them at all"
);
let cases = all_cases();
let mut invocations = 0_usize;
let mut ids_total = 0_usize;
let mut decoded_bytes_total = 0_usize;
for (encoding, entry) in &loaded {
let table = entry.vocab.merge_table();
let allowed = specials_for(encoding.name)
.unwrap_or_else(|| panic!("{} must declare its specials", encoding.name))
.allowed();
for case in &cases {
ids_total += parity::encode_bytes(encoding.pattern, case, &entry.bytes, table).len();
}
invocations += 1;
for case in &cases {
ids_total +=
exatok::tokenizer::encode_bytes(encoding.pattern, case, &entry.bytes, table).len();
}
invocations += 1;
for case in &cases {
ids_total += entry.tokenizer.encode(case).len();
}
invocations += 1;
for case in &cases {
for span in pretokens(encoding.pattern, case) {
let pretoken = span
.bytes(case)
.expect("a span this module produced must lie inside its own input");
let start: Vec<TokenId> = pretoken
.iter()
.map(|byte| entry.bytes[*byte as usize])
.collect();
ids_total += exatok::reference::encode_pretoken(&start, table).len();
}
}
invocations += 1;
for case in &cases {
ids_total += exatok::specials::encode_allowed(
encoding.pattern,
case,
&entry.bytes,
table,
&allowed,
)
.len();
}
invocations += 1;
for case in &cases {
let ids = entry.tokenizer.encode(case);
decoded_bytes_total += entry
.tokenizer
.decode(&ids)
.unwrap_or_else(|error| {
panic!("{}: decode rejected ids its own encoder produced: {error}", encoding.name)
})
.len();
}
invocations += 1;
}
for (index, case) in parity::corpus().into_iter().take(CLI_CASES).enumerate() {
let output = Command::new(env!("CARGO_BIN_EXE_exatok"))
.args(["encode", "--vocab", "cl100k_base", "--text", &case.text])
.output()
.expect("the binary must be runnable");
assert!(
output.status.success(),
"case {index}: the binary failed: {}",
String::from_utf8_lossy(&output.stderr)
);
ids_total += String::from_utf8(output.stdout)
.expect("id output must be UTF-8")
.split_whitespace()
.count();
}
invocations += 1;
assert_eq!(
invocations, ENTRY_POINT_INVOCATIONS,
"the suite drove {invocations} entry-point invocations where \
{ENTRY_POINT_INVOCATIONS} are pinned, so a public route is no longer being exercised \
against every vocabulary"
);
assert!(
ids_total > 0,
"the pinned entry points produced zero ids on a provisioned box, which is the vacuous \
pass this test exists to catch"
);
assert_eq!(
ids_total, API_SHAPE_IDS_TOTAL,
"the entry points produced {ids_total} ids where {API_SHAPE_IDS_TOTAL} are pinned, so \
this suite is driving a different amount of text than it records"
);
assert_eq!(
decoded_bytes_total, DECODED_BYTES_TOTAL,
"decode returned {decoded_bytes_total} bytes where {DECODED_BYTES_TOTAL} are pinned"
);
}