#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use std::collections::BTreeSet;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio};
use std::time::{Duration, Instant};
use clap::CommandFactory;
use exatok::vocab::{self, Artifact, CL100K_BASE, NAMED_VOCABULARIES};
use exatok::{TokenId, ExatokError};
#[path = "../src/main.rs"]
#[allow(dead_code)]
mod binary;
use binary::{
CliError, EXIT_DEVICE, EXIT_INPUT, EXIT_INTERNAL, EXIT_OUTPUT, EXIT_USAGE, EXIT_VOCABULARY,
};
const BIN: &str = env!(concat!("CARGO_BIN_EXE_", env!("CARGO_PKG_NAME")));
fn run(args: &[&str]) -> Output {
run_full(args, &[], &[])
}
fn run_stdin(args: &[&str], stdin: &[u8]) -> Output {
run_full(args, stdin, &[])
}
fn run_env(args: &[&str], env: &[(&str, &str)]) -> Output {
run_full(args, &[], env)
}
fn run_full(args: &[&str], stdin: &[u8], env: &[(&str, &str)]) -> Output {
let mut child = Command::new(BIN)
.args(args)
.envs(env.iter().copied())
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("the binary under test must be runnable");
let _ = child
.stdin
.take()
.expect("stdin was piped")
.write_all(stdin);
child
.wait_with_output()
.expect("the process must be waitable")
}
fn stdout_of(output: &Output) -> String {
String::from_utf8(output.stdout.clone())
.expect("stdout must be UTF-8 for these cases")
.trim_end()
.to_owned()
}
fn stderr_of(output: &Output) -> String {
String::from_utf8(output.stderr.clone()).expect("stderr must be UTF-8")
}
fn exit_code(output: &Output) -> i32 {
output.status.code().expect("the process must exit normally")
}
fn artifacts_available() -> bool {
let mut present = 0_usize;
for vocabulary in NAMED_VOCABULARIES {
match vocab::locate(vocabulary) {
Artifact::Present(_) => present += 1,
Artifact::NoCache { .. } => {}
Artifact::Incomplete { dir, path } => panic!(
"vocabulary cache is incomplete, which is a broken machine rather than an \
unprovisioned one: there is no usable rank file at {}, where a zero-length \
file counts as none, in the cache at {}",
path.display(),
dir.display()
),
}
}
assert!(
present == 0 || present == NAMED_VOCABULARIES.len(),
"expected either no artifacts or all {}, found {present}",
NAMED_VOCABULARIES.len()
);
present == NAMED_VOCABULARIES.len()
}
fn temp_path(label: &str) -> PathBuf {
let path = std::env::temp_dir().join(format!("exatok-cli-{label}"));
if path.is_dir() {
std::fs::remove_dir_all(&path).ok();
} else {
std::fs::remove_file(&path).ok();
}
path
}
const NOT_UTF8: &[u8] = b"hi\xff\xfe\xc3(\x80hi";
#[test]
fn the_exit_code_contract_is_the_documented_numbers() {
assert_eq!(EXIT_USAGE, 2, "usage");
assert_eq!(EXIT_VOCABULARY, 3, "vocabulary");
assert_eq!(EXIT_INPUT, 4, "input");
assert_eq!(EXIT_OUTPUT, 5, "output");
assert_eq!(EXIT_DEVICE, 6, "device");
assert_eq!(EXIT_INTERNAL, 70, "internal");
assert_eq!(CliError::Usage(String::new()).exit_code(), EXIT_USAGE);
assert_eq!(CliError::Vocabulary(String::new()).exit_code(), EXIT_VOCABULARY);
assert_eq!(CliError::Input(String::new()).exit_code(), EXIT_INPUT);
assert_eq!(CliError::Output(String::new()).exit_code(), EXIT_OUTPUT);
assert_eq!(CliError::Device(String::new()).exit_code(), EXIT_DEVICE);
assert_eq!(CliError::Internal(String::new()).exit_code(), EXIT_INTERNAL);
let backend = CliError::from(ExatokError::Backend("dispatch refused".to_owned()));
assert_eq!(backend.exit_code(), EXIT_DEVICE);
assert!(
backend.to_string().contains("Fix:"),
"a device failure must say what to do: {backend}"
);
assert!(
backend.to_string().contains("--device gpu"),
"a device failure must name the flag whose removal encodes on the host: {backend}"
);
let divergence = CliError::from(ExatokError::Divergence {
byte_offset: 7,
host: Some(TokenId(11)),
device: Some(TokenId(12)),
fix: "report this input".to_owned(),
});
assert_eq!(divergence.exit_code(), EXIT_INTERNAL);
assert!(
divergence.to_string().contains("Fix:"),
"even an internal error must say what to do: {divergence}"
);
let artifact = CliError::from(ExatokError::Artifact {
artifact: "cl100k_base".to_owned(),
location: "line 3".to_owned(),
detail: "rank is not a number".to_owned(),
fix: "re-download the rank file".to_owned(),
});
assert_eq!(artifact.exit_code(), EXIT_VOCABULARY);
}
#[test]
fn help_documents_exactly_the_exit_codes_that_exist() {
let output = run(&["--help"]);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
let text = String::from_utf8(output.stdout.clone()).expect("help must be UTF-8");
let mut documented = BTreeSet::new();
let mut in_table = false;
for line in text.lines() {
if line == "Exit codes:" {
in_table = true;
continue;
}
if !in_table {
continue;
}
let indent = line.len() - line.trim_start().len();
if !line.is_empty() && indent == 0 {
break;
}
if indent == 2 {
if let Ok(code) = line
.split_whitespace()
.next()
.unwrap_or_default()
.parse::<i32>()
{
documented.insert(code);
}
}
}
let expected = BTreeSet::from([
0,
EXIT_USAGE,
EXIT_VOCABULARY,
EXIT_INPUT,
EXIT_OUTPUT,
EXIT_DEVICE,
EXIT_INTERNAL,
]);
assert_eq!(
documented, expected,
"the exit-code table in --help must match the codes the binary has"
);
assert!(
text.contains("Status 1 is never returned"),
"the reserved status must stay documented: {text}"
);
}
fn help_flags(args: &[&str]) -> BTreeSet<String> {
let output = run(args);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
let text = String::from_utf8(output.stdout.clone()).expect("help must be UTF-8");
let mut flags = BTreeSet::new();
let mut in_options = false;
for line in text.lines() {
if line == "Options:" {
in_options = true;
continue;
}
if !in_options {
continue;
}
let trimmed = line.trim_start();
let indent = line.len() - trimmed.len();
if !line.is_empty() && indent == 0 {
break;
}
if indent > 6 || !trimmed.starts_with('-') {
continue;
}
if let Some(rest) = trimmed.split("--").nth(1) {
flags.insert(
rest.chars()
.take_while(|c| c.is_ascii_alphanumeric() || *c == '-')
.collect(),
);
}
}
flags
}
fn parser_flags(command: &clap::Command) -> BTreeSet<String> {
command
.get_arguments()
.filter_map(|arg| arg.get_long().map(str::to_owned))
.collect()
}
#[test]
fn help_lists_exactly_the_flags_the_parser_accepts() {
let mut root = binary::Cli::command();
root.build();
assert_eq!(
help_flags(&["--help"]),
parser_flags(&root),
"root command flags"
);
let mut checked = 0_usize;
for sub in root.get_subcommands() {
if sub.is_disable_help_flag_set() {
continue;
}
let name = sub.get_name();
assert_eq!(
help_flags(&[name, "--help"]),
parser_flags(sub),
"flags of subcommand {name}"
);
checked += 1;
}
assert_eq!(
checked, 6,
"every subcommand must be covered; found {checked}"
);
}
#[test]
fn help_names_every_vocabulary_the_binary_accepts() {
let output = run(&["encode", "--help"]);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
let text = stdout_of(&output);
for vocabulary in NAMED_VOCABULARIES {
assert!(
text.contains(vocabulary.name),
"--vocab help must name {}: {text}",
vocabulary.name
);
}
}
#[test]
fn version_reports_the_package_version() {
let output = run(&["--version"]);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
assert_eq!(
stdout_of(&output),
format!("{} {}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")),
"the version output must be exactly the package name and version"
);
}
#[test]
fn encode_produces_the_reference_ids_for_every_pattern() {
if !artifacts_available() {
return;
}
for (name, expected) in [
("cl100k_base", "15339 1917\n"),
("r50k_base", "31373 995\n"),
("o200k_base", "24912 2375\n"),
] {
let output = run(&["encode", "--vocab", name, "--text", "hello world"]);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
assert_eq!(output.stdout, expected.as_bytes(), "vocabulary {name}");
}
}
#[test]
fn json_format_is_exactly_a_json_array() {
if !artifacts_available() {
return;
}
let output = run(&[
"encode",
"--vocab",
"cl100k_base",
"--text",
"hello world",
"--format",
"json",
]);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
assert_eq!(output.stdout, b"[15339,1917]\n");
}
#[test]
fn count_equals_the_number_of_ids_encode_prints() {
if !artifacts_available() {
return;
}
let text = "The quick brown fox jumps over the lazy dog, 1234567890 times.";
let encoded = run(&["encode", "--vocab", "cl100k_base", "--text", text]);
let counted = run(&["count", "--vocab", "cl100k_base", "--text", text]);
let ids = stdout_of(&encoded);
let id_count = ids.split_whitespace().count();
assert!(id_count > 1, "fixture must not be a single token: {ids}");
assert_eq!(counted.stdout, format!("{id_count}\n").into_bytes());
}
#[test]
fn the_three_input_sources_produce_identical_output() {
if !artifacts_available() {
return;
}
let text = "hello world, and a tail long enough to fill more than one token";
let path = temp_path("three-sources.txt");
std::fs::write(&path, text).expect("temp file must be writable");
let by_text = run(&["encode", "--vocab", "cl100k_base", "--text", text]);
let by_file = run(&[
"encode",
"--vocab",
"cl100k_base",
"--file",
path.to_str().unwrap(),
]);
let by_stdin = run_stdin(&["encode", "--vocab", "cl100k_base"], text.as_bytes());
assert_eq!(exit_code(&by_text), 0, "{}", stderr_of(&by_text));
assert_eq!(exit_code(&by_file), 0, "{}", stderr_of(&by_file));
assert_eq!(exit_code(&by_stdin), 0, "{}", stderr_of(&by_stdin));
assert_eq!(by_file.stdout, by_text.stdout, "--file against --text");
assert_eq!(by_stdin.stdout, by_text.stdout, "stdin against --text");
std::fs::remove_file(&path).ok();
}
#[test]
fn non_utf8_input_on_stdin_encodes_without_a_panic() {
if !artifacts_available() {
return;
}
let output = run_stdin(&["encode", "--vocab", "cl100k_base"], NOT_UTF8);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
assert_eq!(output.stdout, b"6151 187 186 127 7 222 6151\n");
assert_eq!(
stderr_of(&output),
"",
"a byte-oriented encoder must not complain about bytes"
);
}
#[test]
fn empty_input_produces_empty_output_and_exit_zero() {
if !artifacts_available() {
return;
}
for args in [
vec!["encode", "--vocab", "cl100k_base"],
vec!["split", "--vocab", "cl100k_base"],
vec!["decode", "--vocab", "cl100k_base"],
] {
let output = run_stdin(&args, b"");
assert_eq!(exit_code(&output), 0, "{args:?}: {}", stderr_of(&output));
assert_eq!(output.stdout, b"", "{args:?} must write nothing at all");
}
let by_flag = run(&["encode", "--vocab", "cl100k_base", "--text", ""]);
assert_eq!(exit_code(&by_flag), 0, "{}", stderr_of(&by_flag));
assert_eq!(by_flag.stdout, b"");
let json = run(&[
"encode",
"--vocab",
"cl100k_base",
"--text",
"",
"--format",
"json",
]);
assert_eq!(json.stdout, b"[]\n");
let counted = run(&["count", "--vocab", "cl100k_base", "--text", ""]);
assert_eq!(exit_code(&counted), 0, "{}", stderr_of(&counted));
assert_eq!(counted.stdout, b"0\n");
}
#[test]
fn a_large_input_encodes_whole_through_stdin() {
if !artifacts_available() {
return;
}
let line = "The quick brown fox jumps over the lazy dog 1234567890, don't stop.\n";
let corpus = line.repeat(4096);
assert!(corpus.len() > 256 * 1024, "fixture must exceed 256 KiB");
let path = temp_path("large-input.txt");
std::fs::write(&path, &corpus).expect("temp file must be writable");
let by_stdin = run_stdin(&["count", "--vocab", "cl100k_base"], corpus.as_bytes());
let by_file = run(&[
"count",
"--vocab",
"cl100k_base",
"--file",
path.to_str().unwrap(),
]);
assert_eq!(exit_code(&by_stdin), 0, "{}", stderr_of(&by_stdin));
assert_eq!(by_stdin.stdout, by_file.stdout, "stdin must not truncate");
let counted: usize = stdout_of(&by_stdin).parse().expect("count must be a number");
assert!(
counted > 4096,
"4096 lines must exceed 4096 tokens, got {counted}"
);
std::fs::remove_file(&path).ok();
}
#[test]
fn decode_returns_exact_bytes_for_invalid_utf8() {
if !artifacts_available() {
return;
}
let path = temp_path("invalid-utf8.bin");
std::fs::write(&path, NOT_UTF8).expect("temp file must be writable");
let encoded = run(&[
"encode",
"--vocab",
"cl100k_base",
"--file",
path.to_str().unwrap(),
]);
assert_eq!(exit_code(&encoded), 0, "{}", stderr_of(&encoded));
let ids = stdout_of(&encoded);
let decoded = run(&["decode", "--vocab", "cl100k_base", "--ids", &ids]);
assert_eq!(exit_code(&decoded), 0, "{}", stderr_of(&decoded));
assert_eq!(
decoded.stdout, NOT_UTF8,
"decode must reproduce the original bytes exactly"
);
std::fs::remove_file(&path).ok();
}
#[test]
fn encode_then_decode_round_trips_non_utf8_for_every_vocabulary() {
if !artifacts_available() {
return;
}
let path = temp_path("round-trip.bin");
std::fs::write(&path, NOT_UTF8).expect("temp file must be writable");
for vocabulary in NAMED_VOCABULARIES {
let name = vocabulary.name;
let encoded = run(&[
"encode",
"--vocab",
name,
"--file",
path.to_str().unwrap(),
]);
assert_eq!(exit_code(&encoded), 0, "{name}: {}", stderr_of(&encoded));
let ids = stdout_of(&encoded);
assert!(!ids.is_empty(), "{name} must encode to something");
let decoded = run(&["decode", "--vocab", name, "--ids", &ids]);
assert_eq!(exit_code(&decoded), 0, "{name}: {}", stderr_of(&decoded));
assert_eq!(
decoded.stdout, NOT_UTF8,
"{name} must round-trip the exact bytes, ids were {ids}"
);
}
std::fs::remove_file(&path).ok();
}
#[test]
fn decode_names_a_registered_special_token() {
if !artifacts_available() {
return;
}
let output = run(&["decode", "--vocab", "cl100k_base", "--ids", "100257"]);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
assert_eq!(output.stdout, b"<|endoftext|>");
}
#[test]
fn id_lists_accept_commas_and_whitespace_identically() {
if !artifacts_available() {
return;
}
for spelling in ["15339,1917", "15339 1917", "15339, 1917", "15339\t1917"] {
let output = run(&["decode", "--vocab", "cl100k_base", "--ids", spelling]);
assert_eq!(exit_code(&output), 0, "{spelling}: {}", stderr_of(&output));
assert_eq!(output.stdout, b"hello world", "spelling {spelling}");
}
}
#[test]
fn decode_reads_ids_from_stdin_when_the_flag_is_absent() {
if !artifacts_available() {
return;
}
let output = run_stdin(&["decode", "--vocab", "cl100k_base"], b"15339 1917\n");
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
assert_eq!(output.stdout, b"hello world");
}
#[test]
fn info_reproduces_the_published_n_vocab_for_every_vocabulary() {
if !artifacts_available() {
return;
}
for (name, n_vocab, specials) in [
("cl100k_base", "100277", "5"),
("r50k_base", "50257", "1"),
("p50k_base", "50281", "1"),
("o200k_base", "200019", "2"),
] {
let output = run(&["info", "--vocab", name]);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
let text = stdout_of(&output);
let field = |key: &str| -> String {
text.lines()
.find_map(|line| line.strip_prefix(key))
.map(|rest| rest.trim().to_owned())
.unwrap_or_else(|| panic!("{name} info must report {key}: {text}"))
};
assert_eq!(field("n_vocab"), n_vocab, "vocabulary {name}");
assert_eq!(field("specials"), specials, "vocabulary {name}");
}
}
#[test]
fn split_output_reconstructs_the_input() {
if !artifacts_available() {
return;
}
let text = "don't stop 1234567 now";
let output = run(&["split", "--vocab", "cl100k_base", "--text", text]);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
let body = String::from_utf8(output.stdout.clone()).unwrap();
let joined: String = body.lines().collect();
assert_eq!(joined, text, "pretokens must partition the input");
}
#[test]
fn digit_runs_are_capped_for_cl100k_and_uncapped_for_gpt2() {
if !artifacts_available() {
return;
}
let capped = run(&["split", "--vocab", "cl100k_base", "--text", "1234567"]);
let uncapped = run(&["split", "--vocab", "r50k_base", "--text", "1234567"]);
assert_eq!(capped.stdout, b"123\n456\n7\n");
assert_eq!(uncapped.stdout, b"1234567\n");
}
#[test]
fn a_pretoken_containing_newlines_is_written_verbatim() {
if !artifacts_available() {
return;
}
let output = run(&["split", "--vocab", "cl100k_base", "--text", "a\n\nb"]);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
assert_eq!(output.stdout, b"a\n\n\n\nb\n");
let body = String::from_utf8(output.stdout.clone()).unwrap();
assert_eq!(
body.lines().count(),
5,
"the ambiguity is the point: more lines than pretokens"
);
}
#[test]
fn a_rejected_id_leaves_no_partial_output() {
if !artifacts_available() {
return;
}
let output = run(&[
"decode",
"--vocab",
"cl100k_base",
"--ids",
"15339,1917,999999",
]);
assert_eq!(exit_code(&output), EXIT_USAGE);
assert_eq!(
output.stdout, b"",
"the two valid ids must not reach stdout before the third is refused"
);
assert!(stderr_of(&output).contains("999999"));
}
#[test]
fn parity_status_names_every_vocabulary_and_the_search_directory() {
let output = run(&["parity-status"]);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
let text = stdout_of(&output);
assert!(
text.contains(&vocab::artifact_dir().display().to_string()),
"must name the directory it searched: {text}"
);
for vocabulary in NAMED_VOCABULARIES {
assert!(
text.contains(vocabulary.name),
"must list {}: {text}",
vocabulary.name
);
}
}
#[test]
fn parity_status_reports_the_overridden_directory() {
let dir = temp_path("status-dir");
let output = run_env(
&["parity-status"],
&[(vocab::ARTIFACT_DIR_ENV, dir.to_str().unwrap())],
);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
let mut expected = format!("vocabulary directory: {}\n", dir.display());
for vocabulary in NAMED_VOCABULARIES {
expected.push_str(&format!(
"{:<12} {:<14} absent no rank files\n",
vocabulary.name,
vocabulary.pattern.name()
));
}
assert_eq!(
stdout_of(&output),
expected.trim_end(),
"every row must report the overridden directory's absence, exactly"
);
}
#[test]
fn a_vocabulary_path_without_a_pattern_is_refused() {
let output = run(&[
"encode",
"--vocab-path",
"/nonexistent/ranks.tiktoken",
"--text",
"hi",
]);
assert_eq!(exit_code(&output), EXIT_USAGE);
let message = stderr_of(&output);
assert!(message.contains("--pattern"), "{message}");
assert!(
message.contains("Fix:"),
"every usage error must carry an actionable Fix clause: {message}"
);
assert!(
!message.contains("No such file"),
"the flag is missing regardless of the path: {message}"
);
}
#[test]
fn a_tiktoken_path_with_a_pattern_matches_the_named_vocabulary() {
if !artifacts_available() {
return;
}
let Artifact::Present(path) = vocab::locate(CL100K_BASE) else {
return;
};
let by_path = run(&[
"encode",
"--vocab-path",
path.to_str().unwrap(),
"--pattern",
"cl100k-base",
"--text",
"hello world",
]);
assert_eq!(exit_code(&by_path), 0, "{}", stderr_of(&by_path));
assert_eq!(by_path.stdout, b"15339 1917\n");
}
#[test]
fn a_pattern_with_a_named_vocabulary_is_refused() {
let output = run(&[
"encode",
"--vocab",
"cl100k_base",
"--pattern",
"gpt2",
"--text",
"hi",
]);
assert_eq!(exit_code(&output), EXIT_USAGE);
assert!(stderr_of(&output).contains("--pattern"));
}
#[test]
fn an_unknown_vocabulary_name_lists_the_accepted_names() {
let output = run(&["encode", "--vocab", "gpt5_base", "--text", "hi"]);
assert_eq!(exit_code(&output), EXIT_USAGE);
let message = stderr_of(&output);
assert!(message.contains("gpt5_base"), "{message}");
for vocabulary in NAMED_VOCABULARIES {
assert!(
message.contains(vocabulary.name),
"must list {}: {message}",
vocabulary.name
);
}
}
#[test]
fn text_and_file_together_are_refused() {
let output = run(&[
"encode",
"--vocab",
"cl100k_base",
"--text",
"hi",
"--file",
"/etc/hostname",
]);
assert_eq!(exit_code(&output), EXIT_USAGE);
let message = stderr_of(&output);
assert!(message.contains("mutually exclusive"), "{message}");
assert!(message.contains("/etc/hostname"), "{message}");
}
#[test]
fn vocab_and_vocab_path_together_are_refused() {
let output = run(&[
"encode",
"--vocab",
"cl100k_base",
"--vocab-path",
"/nonexistent/ranks.tiktoken",
"--pattern",
"gpt2",
"--text",
"hi",
]);
assert_eq!(exit_code(&output), EXIT_USAGE);
assert!(stderr_of(&output).contains("mutually exclusive"));
}
#[test]
fn an_unrecognized_vocabulary_extension_is_refused() {
let output = run(&[
"encode",
"--vocab-path",
"/tmp/vocab.txt",
"--pattern",
"gpt2",
"--text",
"hi",
]);
assert_eq!(exit_code(&output), EXIT_USAGE);
assert!(stderr_of(&output).contains(".tiktoken"));
}
#[test]
fn a_missing_vocabulary_argument_is_refused() {
let output = run(&["encode", "--text", "hi"]);
assert_eq!(exit_code(&output), EXIT_USAGE);
let message = stderr_of(&output);
assert!(message.contains("--vocab"), "{message}");
assert!(message.contains("--vocab-path"), "{message}");
}
#[test]
fn a_non_numeric_id_argument_is_refused_and_quoted() {
if !artifacts_available() {
return;
}
let output = run(&["decode", "--vocab", "cl100k_base", "--ids", "15339,abc"]);
assert_eq!(exit_code(&output), EXIT_USAGE);
let message = stderr_of(&output);
assert!(message.contains("abc"), "{message}");
assert!(message.contains("Fix:"), "{message}");
}
#[test]
fn decode_rejects_an_out_of_range_id_and_reports_the_decodable_ceiling() {
if !artifacts_available() {
return;
}
let output = run(&["decode", "--vocab", "cl100k_base", "--ids", "999999"]);
assert_eq!(exit_code(&output), EXIT_USAGE);
let message = stderr_of(&output);
assert!(message.contains("999999"), "{message}");
assert!(
message.contains("100276"),
"must report the highest DECODABLE id, specials included: {message}"
);
}
#[test]
fn an_unknown_flag_is_refused_with_a_fix_clause() {
let output = run(&["encode", "--vocab", "cl100k_base", "--txt", "hi"]);
assert_eq!(exit_code(&output), EXIT_USAGE);
let message = stderr_of(&output);
assert!(message.contains("--txt"), "{message}");
assert!(message.contains("Fix:"), "{message}");
}
#[test]
fn an_invalid_format_value_names_the_alternatives() {
let output = run(&[
"encode",
"--vocab",
"cl100k_base",
"--format",
"yaml",
"--text",
"hi",
]);
assert_eq!(exit_code(&output), EXIT_USAGE);
let message = stderr_of(&output);
assert!(message.contains("yaml"), "{message}");
assert!(message.contains("ids"), "{message}");
assert!(message.contains("json"), "{message}");
}
#[test]
fn a_bare_invocation_is_refused() {
let output = run(&[]);
assert_eq!(exit_code(&output), EXIT_USAGE);
assert!(stderr_of(&output).contains("Fix:"));
}
#[test]
fn an_unprovisioned_vocabulary_exits_three() {
let missing = temp_path("no-such-cache");
let absent = run_env(
&["encode", "--vocab", "cl100k_base", "--text", "hi"],
&[(vocab::ARTIFACT_DIR_ENV, missing.to_str().unwrap())],
);
assert_eq!(exit_code(&absent), EXIT_VOCABULARY, "{}", stderr_of(&absent));
let message = stderr_of(&absent);
assert!(message.contains("cl100k_base"), "{message}");
assert!(message.contains(missing.to_str().unwrap()), "{message}");
assert!(message.contains(vocab::ARTIFACT_DIR_ENV), "{message}");
assert!(message.contains("Fix:"), "{message}");
assert!(
!message.contains("SKIPPED"),
"a failed command did not skip anything: {message}"
);
let empty = temp_path("empty-cache");
std::fs::create_dir_all(&empty).expect("temp dir must be creatable");
let nothing_in_it = run_env(
&["encode", "--vocab", "cl100k_base", "--text", "hi"],
&[(vocab::ARTIFACT_DIR_ENV, empty.to_str().unwrap())],
);
assert_eq!(exit_code(¬hing_in_it), EXIT_VOCABULARY);
let message = stderr_of(¬hing_in_it);
assert!(message.contains("is not provisioned"), "{message}");
assert!(
!message.contains("incomplete"),
"a directory holding no rank table was never provisioned, so there is \
nothing to repair: {message}"
);
assert!(message.contains("Fix:"), "{message}");
std::fs::remove_dir_all(&empty).ok();
let partial = temp_path("partial-cache");
std::fs::create_dir_all(&partial).expect("temp dir must be creatable");
std::fs::write(partial.join("r50k_base.tiktoken"), b"YQ== 0\n")
.expect("a sibling artifact must be writable");
let half = run_env(
&["encode", "--vocab", "cl100k_base", "--text", "hi"],
&[(vocab::ARTIFACT_DIR_ENV, partial.to_str().unwrap())],
);
assert_eq!(exit_code(&half), EXIT_VOCABULARY);
let message = stderr_of(&half);
assert!(message.contains("incomplete"), "{message}");
assert!(
!message.contains("is not provisioned"),
"a populated cache is repaired, not provisioned: {message}"
);
assert!(
message.contains(
partial
.join("cl100k_base.tiktoken")
.to_str()
.expect("the temporary directory must be UTF-8")
),
"the one file to re-fetch must be named: {message}"
);
assert!(message.contains("Fix:"), "{message}");
std::fs::remove_dir_all(&partial).ok();
}
#[test]
fn an_unreadable_vocabulary_artifact_exits_three() {
let missing = temp_path("absent-ranks.tiktoken");
let output = run(&[
"encode",
"--vocab-path",
missing.to_str().unwrap(),
"--pattern",
"gpt2",
"--text",
"hi",
]);
assert_eq!(exit_code(&output), EXIT_VOCABULARY, "{}", stderr_of(&output));
let message = stderr_of(&output);
assert!(message.contains(missing.to_str().unwrap()), "{message}");
assert!(message.contains("Fix:"), "{message}");
assert!(
!message.contains("malformed"),
"a file that is not there is not malformed: {message}"
);
let directory = temp_path("directory-ranks.tiktoken");
std::fs::create_dir_all(&directory).expect("temp dir must be creatable");
let output = run(&[
"encode",
"--vocab-path",
directory.to_str().unwrap(),
"--pattern",
"gpt2",
"--text",
"hi",
]);
assert_eq!(exit_code(&output), EXIT_VOCABULARY);
assert!(stderr_of(&output).contains("directory"), "{}", stderr_of(&output));
std::fs::remove_dir_all(&directory).ok();
let forbidden = temp_path("forbidden-ranks.tiktoken");
std::fs::write(&forbidden, "IQ== 0\n").expect("temp file must be writable");
if set_unreadable(&forbidden) {
let output = run(&[
"encode",
"--vocab-path",
forbidden.to_str().unwrap(),
"--pattern",
"gpt2",
"--text",
"hi",
]);
assert_eq!(exit_code(&output), EXIT_VOCABULARY);
let message = stderr_of(&output);
assert!(message.contains(forbidden.to_str().unwrap()), "{message}");
assert!(message.contains("Fix:"), "{message}");
}
std::fs::remove_file(&forbidden).ok();
}
fn set_unreadable(path: &Path) -> bool {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o000)).ok();
return std::fs::read(path).is_err();
}
#[cfg(not(unix))]
{
let _ = path;
false
}
}
#[test]
fn an_unreadable_input_file_exits_four_and_names_the_path() {
if !artifacts_available() {
return;
}
let missing = temp_path("absent-input.txt");
let output = run(&[
"encode",
"--vocab",
"cl100k_base",
"--file",
missing.to_str().unwrap(),
]);
assert_eq!(exit_code(&output), EXIT_INPUT, "{}", stderr_of(&output));
let message = stderr_of(&output);
assert!(message.contains(missing.to_str().unwrap()), "{message}");
assert!(message.contains("Fix:"), "{message}");
}
#[test]
fn a_bad_id_on_stdin_is_an_input_error() {
if !artifacts_available() {
return;
}
let piped = run_stdin(&["decode", "--vocab", "cl100k_base"], b"15339 abc\n");
assert_eq!(exit_code(&piped), EXIT_INPUT, "{}", stderr_of(&piped));
assert!(stderr_of(&piped).contains("abc"), "{}", stderr_of(&piped));
let argument = run(&["decode", "--vocab", "cl100k_base", "--ids", "15339 abc"]);
assert_eq!(exit_code(&argument), EXIT_USAGE);
assert_eq!(
stderr_of(&piped),
stderr_of(&argument),
"the same fault must read the same either way; only the code differs"
);
}
#[test]
fn a_non_text_id_stream_exits_four_and_names_the_byte() {
if !artifacts_available() {
return;
}
let output = run_stdin(&["decode", "--vocab", "cl100k_base"], NOT_UTF8);
assert_eq!(exit_code(&output), EXIT_INPUT, "{}", stderr_of(&output));
let message = stderr_of(&output);
assert!(message.contains("0xff"), "must name the byte: {message}");
assert!(message.contains("byte 2"), "must name the offset: {message}");
assert!(message.contains("Fix:"), "{message}");
}
#[test]
fn output_that_cannot_be_written_exits_five() {
if !artifacts_available() {
return;
}
if std::fs::OpenOptions::new()
.write(true)
.open("/dev/full")
.is_err()
{
return;
}
for args in [
vec!["encode", "--vocab", "cl100k_base", "--text", "hi"],
vec!["count", "--vocab", "cl100k_base", "--text", "hi"],
vec!["decode", "--vocab", "cl100k_base", "--ids", "15339"],
vec!["split", "--vocab", "cl100k_base", "--text", "hi"],
vec!["info", "--vocab", "cl100k_base"],
vec!["parity-status"],
vec!["--help"],
] {
let full = std::fs::OpenOptions::new()
.write(true)
.open("/dev/full")
.expect("checked above");
let status = Command::new(BIN)
.args(&args)
.stdin(Stdio::null())
.stdout(Stdio::from(full))
.stderr(Stdio::piped())
.spawn()
.expect("the binary under test must be runnable")
.wait_with_output()
.expect("the process must be waitable");
assert_eq!(
status.status.code(),
Some(EXIT_OUTPUT),
"{args:?} must report a failed write: {}",
stderr_of(&status)
);
}
}
#[test]
fn every_failure_names_the_offender_and_a_fix() {
let missing_cache = temp_path("fix-clause-cache");
let cases: [(&[&str], &[(&str, &str)], &str); 8] = [
(&["encode", "--text", "hi"], &[], "--vocab"),
(
&["encode", "--vocab", "gpt5_base", "--text", "hi"],
&[],
"gpt5_base",
),
(
&["encode", "--vocab", "cl100k_base", "--txt", "hi"],
&[],
"--txt",
),
(
&["encode", "--vocab-path", "/tmp/v.txt", "--pattern", "gpt2", "--text", "hi"],
&[],
"/tmp/v.txt",
),
(
&["encode", "--vocab-path", "/tmp/v.tiktoken", "--text", "hi"],
&[],
"--pattern",
),
(
&["encode", "--vocab", "cl100k_base", "--text", "hi"],
&[(vocab::ARTIFACT_DIR_ENV, "")],
"cl100k_base",
),
(
&["decode", "--vocab", "cl100k_base", "--ids", "zz"],
&[],
"zz",
),
(
&["encode", "--vocab", "cl100k_base", "--format", "yaml", "--text", "hi"],
&[],
"yaml",
),
];
for (args, env, offender) in cases {
let env: Vec<(&str, &str)> = env
.iter()
.map(|(key, value)| {
if value.is_empty() {
(*key, missing_cache.to_str().unwrap())
} else {
(*key, *value)
}
})
.collect();
let output = run_env(args, &env);
let code = exit_code(&output);
let message = stderr_of(&output);
assert_ne!(code, 0, "{args:?} must fail");
assert_ne!(code, 1, "{args:?} must not return the reserved status 1");
assert!(
message.contains(offender),
"{args:?} must quote {offender}: {message}"
);
assert!(
message.contains("Fix:"),
"{args:?} must carry a Fix clause: {message}"
);
}
}
#[test]
fn usage_is_reported_before_vocabulary_and_vocabulary_before_input() {
let missing_cache = temp_path("precedence-cache");
let cache = [(vocab::ARTIFACT_DIR_ENV, missing_cache.to_str().unwrap())];
let usage_first = run_env(
&["encode", "--vocab", "gpt5_base", "--file", "/nonexistent"],
&cache,
);
assert_eq!(
exit_code(&usage_first),
EXIT_USAGE,
"{}",
stderr_of(&usage_first)
);
let vocabulary_next = run_env(
&["encode", "--vocab", "cl100k_base", "--file", "/nonexistent"],
&cache,
);
assert_eq!(
exit_code(&vocabulary_next),
EXIT_VOCABULARY,
"{}",
stderr_of(&vocabulary_next)
);
if artifacts_available() {
let input_last = run(&[
"encode",
"--vocab",
"cl100k_base",
"--file",
"/nonexistent",
]);
assert_eq!(
exit_code(&input_last),
EXIT_INPUT,
"{}",
stderr_of(&input_last)
);
}
}
#[test]
fn a_failure_that_needs_no_input_does_not_wait_on_stdin() {
let missing_cache = temp_path("no-wait-cache");
let cases: [(&[&str], &[(&str, &str)], i32); 3] = [
(&["encode", "--vocab", "gpt5_base"], &[], EXIT_USAGE),
(&["count", "--text", "hi", "--file", "/etc/hostname"], &[], EXIT_USAGE),
(
&["encode", "--vocab", "cl100k_base"],
&[(vocab::ARTIFACT_DIR_ENV, "")],
EXIT_VOCABULARY,
),
];
for (args, env, expected) in cases {
let env: Vec<(&str, &str)> = env
.iter()
.map(|(key, _)| (*key, missing_cache.to_str().unwrap()))
.collect();
let mut child = Command::new(BIN)
.args(args)
.envs(env)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("the binary under test must be runnable");
let deadline = Instant::now() + Duration::from_secs(10);
loop {
match child.try_wait().expect("the child must be waitable") {
Some(status) => {
assert_eq!(status.code(), Some(expected), "{args:?}");
break;
}
None if Instant::now() >= deadline => {
child.kill().ok();
child.wait().ok();
panic!("{args:?} waited on stdin for a failure it already knew about");
}
None => std::thread::sleep(Duration::from_millis(10)),
}
}
}
}
#[test]
fn no_failure_returns_the_reserved_status_one() {
let known = BTreeSet::from([
EXIT_USAGE,
EXIT_VOCABULARY,
EXIT_INPUT,
EXIT_OUTPUT,
EXIT_INTERNAL,
]);
let failures: [&[&str]; 6] = [
&[],
&["encode", "--text", "hi"],
&["encode", "--vocab", "nope", "--text", "hi"],
&["decode", "--vocab", "cl100k_base", "--ids", "xyz"],
&["encode", "--vocab-path", "/tmp/v.txt", "--pattern", "gpt2", "--text", "hi"],
&["frobnicate"],
];
for args in failures {
let output = run(args);
let code = exit_code(&output);
assert_ne!(code, 1, "{args:?} must not return the reserved status 1");
assert!(
known.contains(&code),
"{args:?} returned undocumented status {code}: {}",
stderr_of(&output)
);
}
}
#[test]
fn a_successful_run_is_silent_on_stderr() {
if !artifacts_available() {
return;
}
for args in [
vec!["encode", "--vocab", "cl100k_base", "--text", "hi"],
vec!["count", "--vocab", "cl100k_base", "--text", "hi"],
vec!["decode", "--vocab", "cl100k_base", "--ids", "15339"],
vec!["split", "--vocab", "cl100k_base", "--text", "hi"],
vec!["info", "--vocab", "cl100k_base"],
vec!["parity-status"],
] {
let output = run(&args);
assert_eq!(exit_code(&output), 0, "{args:?}");
assert_eq!(stderr_of(&output), "", "{args:?} must be silent on success");
}
}
#[test]
fn printed_ids_are_bare_decimal_numbers() {
if !artifacts_available() {
return;
}
let output = run(&["encode", "--vocab", "cl100k_base", "--text", "hello"]);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
let printed = stdout_of(&output);
let parsed: Vec<TokenId> = printed
.split_whitespace()
.map(|field| TokenId(field.parse().expect("every field must be a bare number")))
.collect();
assert_eq!(parsed, vec![TokenId(15339)]);
}
#[test]
fn the_default_device_is_the_host_and_matches_an_explicit_cpu() {
if !artifacts_available() {
return;
}
let text = "don't stop, 1234567890 times, and a tail to fill more than one token";
let implicit = run(&["encode", "--vocab", "cl100k_base", "--text", text]);
let explicit = run(&[
"encode",
"--vocab",
"cl100k_base",
"--text",
text,
"--device",
"cpu",
]);
assert_eq!(exit_code(&implicit), 0, "{}", stderr_of(&implicit));
assert_eq!(exit_code(&explicit), 0, "{}", stderr_of(&explicit));
assert_eq!(
implicit.stdout, explicit.stdout,
"omitting --device must be exactly --device cpu"
);
assert!(
!implicit.stdout.is_empty(),
"the comparison is only meaningful if ids were actually printed"
);
assert_eq!(
stderr_of(&explicit),
"",
"a host run must not print a routing report, which belongs to device runs only"
);
}
#[test]
fn help_documents_the_device_flag_and_its_refusal_to_fall_back() {
let output = run(&["--help"]);
assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
let text = String::from_utf8(output.stdout.clone()).expect("help must be UTF-8");
for needle in [
"--device",
"Device:",
"THE SAME IDS",
"FAILS with status 6",
] {
assert!(
text.contains(needle),
"--help must document {needle:?} so the device contract is readable without the \
source"
);
}
}
#[test]
fn device_gpu_refuses_and_produces_no_ids_in_every_build() {
if !artifacts_available() {
return;
}
let output = run_env(
&[
"encode",
"--vocab",
"cl100k_base",
"--text",
"hello world",
"--device",
"gpu",
],
&[("CUDA_VISIBLE_DEVICES", "")],
);
assert_eq!(
exit_code(&output),
EXIT_DEVICE,
"--device gpu with no usable device must exit EXIT_DEVICE: {}",
stderr_of(&output)
);
assert!(
output.stdout.is_empty(),
"a refused device run must print no ids, or the caller cannot tell it was refused: {:?}",
String::from_utf8_lossy(&output.stdout)
);
let message = stderr_of(&output);
assert!(
message.contains("Fix:"),
"every failure in this binary carries a Fix clause: {message}"
);
let unsupported = message.contains("no CUDA support");
let no_device = message.contains("no usable CUDA device");
assert!(
unsupported ^ no_device,
"the refusal must be exactly one of the two known arms and not a third shape, \
or an unrecognized message would be filed under whichever arm happened to run: \
{message}"
);
if unsupported {
assert!(
message.contains("--features cuda"),
"a build without cuda support must name the rebuild that would honor the flag: \
{message}"
);
}
}
#[cfg(feature = "cuda")]
#[test]
fn device_gpu_produces_the_host_ids_and_reports_its_routing() {
if !artifacts_available() {
return;
}
let text = "don't stop, 1234567890 times, and a tail to fill more than one token";
let host = run(&[
"encode",
"--vocab",
"cl100k_base",
"--text",
text,
"--device",
"cpu",
]);
let device = run(&[
"encode",
"--vocab",
"cl100k_base",
"--text",
text,
"--device",
"gpu",
]);
assert_eq!(exit_code(&host), 0, "{}", stderr_of(&host));
assert_eq!(
exit_code(&device),
0,
"the device run must succeed on a machine with a device rather than skip: {}",
stderr_of(&device)
);
assert_eq!(
device.stdout, host.stdout,
"the device path is exact, so its ids must equal the host's byte for byte"
);
assert!(
!host.stdout.is_empty(),
"the comparison is only meaningful if ids were actually printed"
);
let report = stderr_of(&device);
assert!(
report.contains("routed") && report.contains("to the device"),
"a device run must report where the work happened: {report}"
);
assert!(
report.contains("0 to the host"),
"this input has no pretoken above the device cap, so nothing should have been routed to \
the host: {report}"
);
}
#[cfg(feature = "cuda")]
#[test]
fn count_agrees_between_the_host_and_the_device() {
if !artifacts_available() {
return;
}
let text = "The quick brown fox jumps over the lazy dog, 1234567890 times.";
let host = run(&[
"count",
"--vocab",
"cl100k_base",
"--text",
text,
"--device",
"cpu",
]);
let device = run(&[
"count",
"--vocab",
"cl100k_base",
"--text",
text,
"--device",
"gpu",
]);
assert_eq!(exit_code(&host), 0, "{}", stderr_of(&host));
assert_eq!(exit_code(&device), 0, "{}", stderr_of(&device));
assert_eq!(
device.stdout, host.stdout,
"the token count cannot depend on where the encode ran"
);
assert!(
stderr_of(&device).contains("routed"),
"count must report its routing too, so a device count is as auditable as a device encode"
);
}