use std::io::{BufWriter, Read, Write};
use std::path::{Path, PathBuf};
use clap::{Parser, Subcommand, ValueEnum};
use exatok::pretokenize::{SplitPattern, unicode_version};
use exatok::specials;
use exatok::vocab::{self, Artifact, NAMED_VOCABULARIES, NamedVocabulary, Vocabulary};
use exatok::{Encoding, TokenId, Tokenizer, UnknownTokenId, ExatokError};
const NAME: &str = env!("CARGO_PKG_NAME");
pub const EXIT_USAGE: i32 = 2;
pub const EXIT_VOCABULARY: i32 = 3;
pub const EXIT_INPUT: i32 = 4;
pub const EXIT_OUTPUT: i32 = 5;
pub const EXIT_DEVICE: i32 = 6;
pub const EXIT_INTERNAL: i32 = 70;
pub const LONG_ABOUT: &str = concat!(
"Exact byte-pair encoding. Token ids match the reference tokenizer byte for byte.\n",
"Run `",
env!("CARGO_PKG_NAME"),
" parity-status` to see which vocabularies are provisioned on this machine.\n",
"\n",
"Input:\n",
" --text takes the input as an argument, --file reads it from a path, and with neither the\n",
" input is read from stdin. Input is bytes and not text: a file or a stdin stream that is\n",
" not valid UTF-8 encodes normally, because the encoder is defined on bytes. An input that\n",
" encodes to no tokens produces no output and exit 0.\n",
"\n",
" The input is read whole into memory rather than streamed, because a pretoken can span any\n",
" byte position and cutting inside one would change the ids. Peak memory is a few times the\n",
" input size.\n",
"\n",
"Output:\n",
" Everything goes to stdout and nothing else is written. encode streams its ids as it\n",
" formats them; decode and split assemble their bytes first, so a rejected id leaves no\n",
" partial output behind.\n",
"\n",
"Device:\n",
" --device cpu, the default, encodes on the host. --device gpu encodes on a CUDA device and\n",
" produces THE SAME IDS: the device path is exact, not an approximation of the host one.\n",
"\n",
" A device run reports on stderr how the work was routed. A pretoken longer than the\n",
" device's per-pretoken cap is encoded on the host and spliced back in order, which is\n",
" scheduling rather than a fallback, and the line is printed on every device run so that\n",
" \"no line\" and \"the line said zero\" are not the same observation. Ids go to stdout, so\n",
" the report cannot corrupt what a program parses.\n",
"\n",
" --device gpu FAILS with status 6 rather than encoding on the host when no CUDA device is\n",
" usable, or when this binary was built without its cuda feature. Encoding on the host\n",
" instead would report success for work you asked to run on the device.\n",
"\n",
"Vocabulary:\n",
" --vocab resolves a name to one .tiktoken rank table in the artifact directory:\n",
" ~/.cache/exatok-vocab by default, or the directory EXATOK_VOCAB_DIR names when\n",
" it is set. The override relocates the whole cache for the library, this binary,\n",
" and every artifact-dependent test at once, so one export points all of them at\n",
" the same directory.\n",
"\n",
" A missing or unreadable artifact is a vocabulary error, status 3, and a\n",
" zero-length one is a broken cache rather than nothing provisioned. `",
env!("CARGO_PKG_NAME"),
" parity-status` reports what is provisioned on this machine, so it is the check\n",
" to run on a fresh machine or after changing the override.\n",
"\n",
"Exit codes:\n",
" 0 success.\n",
" 2 usage error. An unknown flag, a contradictory pair, a missing one, or a value given\n",
" on the command line that this binary will not accept.\n",
" 3 vocabulary error. An artifact was missing, unreadable, or not the table it claims.\n",
" 4 input error. An input file or stream could not be read, or its contents could not be\n",
" decoded. A bad id on the command line is a usage error; a bad id arriving on stdin is\n",
" this, because one is fixed by editing the command and the other by fixing the\n",
" producer.\n",
" 5 output error. Standard output could not be written, a closed pipe for instance.\n",
" 6 device error. --device gpu was asked for, but no CUDA device is usable or this build\n",
" has no cuda feature. No ids were produced and nothing was encoded on the host\n",
" instead.\n",
" 70 internal error. A defect in this binary, so the run proves nothing.\n",
"\n",
" Status 1 is never returned, so a caller can tell \"ran and said no\" from \"could not run\".\n",
" Any other status, 101 for instance, means this binary crashed instead of reporting, which\n",
" is a defect and not a refusal.",
);
#[derive(Debug, thiserror::Error)]
pub enum CliError {
#[error("{0}")]
Usage(String),
#[error("{0}")]
Vocabulary(String),
#[error("{0}")]
Input(String),
#[error("{0}")]
Output(String),
#[error("{0}")]
Device(String),
#[error("{0}")]
Internal(String),
}
impl CliError {
#[must_use]
pub const fn exit_code(&self) -> i32 {
match self {
Self::Usage(_) => EXIT_USAGE,
Self::Vocabulary(_) => EXIT_VOCABULARY,
Self::Input(_) => EXIT_INPUT,
Self::Output(_) => EXIT_OUTPUT,
Self::Device(_) => EXIT_DEVICE,
Self::Internal(_) => EXIT_INTERNAL,
}
}
}
impl From<ExatokError> for CliError {
fn from(error: ExatokError) -> Self {
match &error {
ExatokError::Artifact { .. } => Self::Vocabulary(error.to_string()),
ExatokError::Backend(detail) => Self::Device(format!(
"the CUDA device failed to run the encode: {detail}\nFix: re-run without \
--device gpu to encode on the host, and report this with the input if the host \
path succeeds."
)),
other => Self::Internal(format!(
"{other}\nFix: this is a defect in {NAME}; report it with the command you ran."
)),
}
}
}
fn output_error(source: &std::io::Error) -> CliError {
CliError::Output(format!(
"cannot write to standard output: {source}\nFix: check the pipe or file stdout is \
redirected to; the reader may have exited or the destination may be full."
))
}
fn emit(bytes: &[u8]) -> Result<(), CliError> {
let stdout = std::io::stdout();
let mut out = stdout.lock();
out.write_all(bytes).map_err(|source| output_error(&source))?;
out.flush().map_err(|source| output_error(&source))
}
#[derive(Debug, Clone, Copy, ValueEnum)]
pub enum PatternArg {
Gpt2,
Cl100kBase,
O200kBase,
}
impl From<PatternArg> for SplitPattern {
fn from(value: PatternArg) -> Self {
match value {
PatternArg::Gpt2 => Self::Gpt2,
PatternArg::Cl100kBase => Self::Cl100kBase,
PatternArg::O200kBase => Self::O200kBase,
}
}
}
#[derive(Debug, Clone, Copy, ValueEnum)]
pub enum IdFormat {
Ids,
Json,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum, Default)]
pub enum DeviceArg {
#[default]
Cpu,
Gpu,
}
#[derive(Debug, Parser)]
#[command(about = "Exact BPE tokenization.", long_about = LONG_ABOUT, version)]
pub struct Cli {
#[command(subcommand)]
pub command: Command,
}
#[derive(Debug, Subcommand)]
pub enum Command {
Encode {
#[command(flatten)]
vocab: VocabArgs,
#[command(flatten)]
input: InputArgs,
#[arg(long, value_enum, default_value_t = DeviceArg::Cpu)]
device: DeviceArg,
#[arg(long, value_enum, default_value_t = IdFormat::Ids)]
format: IdFormat,
},
Count {
#[command(flatten)]
vocab: VocabArgs,
#[command(flatten)]
input: InputArgs,
#[arg(long, value_enum, default_value_t = DeviceArg::Cpu)]
device: DeviceArg,
},
Decode {
#[command(flatten)]
vocab: VocabArgs,
#[arg(long)]
ids: Option<String>,
},
#[command(long_about = "Show the pretokenizer split, one pretoken per line.\n\n\
A pretoken may itself contain a newline, a run of blank lines being \
one pretoken, so a pretoken can span output lines and the line \
breakdown is not a parseable framing. Read it, and take ids from \
`encode` when a program needs the partition.")]
Split {
#[command(flatten)]
vocab: VocabArgs,
#[command(flatten)]
input: InputArgs,
},
Info {
#[command(flatten)]
vocab: VocabArgs,
},
ParityStatus,
}
#[derive(Debug, clap::Args)]
pub struct VocabArgs {
#[arg(long, value_name = "NAME", help = vocab_help())]
pub vocab: Option<String>,
#[arg(long)]
pub vocab_path: Option<PathBuf>,
#[arg(long, value_enum)]
pub pattern: Option<PatternArg>,
}
#[derive(Debug, clap::Args)]
pub struct InputArgs {
#[arg(long)]
pub text: Option<String>,
#[arg(long)]
pub file: Option<PathBuf>,
}
#[derive(Debug)]
pub enum InputSource<'args> {
Text(&'args str),
File(&'args Path),
Stdin,
}
impl InputArgs {
fn source(&self) -> Result<InputSource<'_>, CliError> {
match (self.text.as_deref(), self.file.as_deref()) {
(Some(text), None) => Ok(InputSource::Text(text)),
(None, Some(path)) => Ok(InputSource::File(path)),
(None, None) => Ok(InputSource::Stdin),
(Some(text), Some(path)) => Err(CliError::Usage(format!(
"--text {text:?} and --file {} are mutually exclusive, and neither silently \
wins.\nFix: pass one of them, or neither to read stdin.",
path.display()
))),
}
}
}
impl InputSource<'_> {
fn read(&self) -> Result<Vec<u8>, CliError> {
match *self {
Self::Text(text) => Ok(text.as_bytes().to_vec()),
Self::File(path) => std::fs::read(path).map_err(|source| {
CliError::Input(format!(
"cannot read the input file {}: {source}\nFix: check the path exists and is \
readable, or pass --text to give the input on the command line.",
path.display()
))
}),
Self::Stdin => {
let mut buffer = Vec::new();
std::io::stdin().read_to_end(&mut buffer).map_err(|source| {
CliError::Input(format!(
"cannot read the input from stdin: {source}\nFix: redirect a file or a \
pipe into this command, or pass --text or --file instead."
))
})?;
Ok(buffer)
}
}
}
}
fn named(name: &str) -> Option<NamedVocabulary> {
NAMED_VOCABULARIES.into_iter().find(|v| v.name == name)
}
fn known_names() -> String {
NAMED_VOCABULARIES
.iter()
.map(|v| v.name)
.collect::<Vec<_>>()
.join(", ")
}
fn vocab_help() -> String {
format!("Vocabulary name, one of {}", known_names())
}
impl VocabArgs {
fn load(&self) -> Result<Tokenizer, CliError> {
match (self.vocab.as_deref(), self.vocab_path.as_deref()) {
(None, None) => Err(CliError::Usage(format!(
"no vocabulary given.\nFix: pass --vocab <{}> or --vocab-path <file>.",
known_names()
))),
(Some(name), Some(path)) => Err(CliError::Usage(format!(
"--vocab {name:?} and --vocab-path {} are mutually exclusive, because a name \
carries its own pattern and count checks while a path carries neither.\nFix: \
pass one of them.",
path.display()
))),
(Some(name), None) => self.load_named(name),
(None, Some(path)) => self.load_path(path),
}
}
fn load_named(&self, name: &str) -> Result<Tokenizer, CliError> {
if self.pattern.is_some() {
return Err(CliError::Usage(format!(
"--pattern is not accepted with --vocab, because {name} already states its \
pattern.\nFix: drop --pattern, or use --vocab-path to override deliberately."
)));
}
let Some(vocabulary) = named(name) else {
return Err(CliError::Usage(format!(
"unknown vocabulary {name:?}.\nFix: use one of {}, or --vocab-path for an \
artifact elsewhere.",
known_names()
)));
};
match vocab::locate(vocabulary) {
Artifact::Present(path) => {
let mut loaded = vocab::load(vocabulary, &path)?;
specials::register(&mut loaded, vocabulary.name)?;
Ok(Tokenizer::new(
loaded,
Encoding::new(vocabulary.name, vocabulary.pattern),
)?)
}
Artifact::NoCache { dir, path } => Err(CliError::Vocabulary(format!(
"{name} is not provisioned: the cache holds no tiktoken rank table at all. \
Looked for {} in {}.\nFix: put the tiktoken rank files in that directory, \
creating it if it does not exist, or set {} to a directory that already holds \
them.",
path.display(),
dir.display(),
vocab::ARTIFACT_DIR_ENV
))),
Artifact::Incomplete { dir, path } => Err(CliError::Vocabulary(format!(
"{name} has no usable rank file, so this cache is incomplete rather than \
unprovisioned: nothing loadable at {}, where an empty file counts as nothing.\n\
Fix: fetch that one rank file into {} again, replacing a zero-length one if \
that is what is there, or set {} to a complete directory.",
path.display(),
dir.display(),
vocab::ARTIFACT_DIR_ENV
))),
}
}
fn load_path(&self, path: &Path) -> Result<Tokenizer, CliError> {
let Some(pattern) = self.pattern else {
return Err(CliError::Usage(format!(
"--pattern is required with --vocab-path {}, because a rank file does not record \
its pretokenizer and this binary does not yet read the one a tokenizer.json \
bundle records, and the three patterns give different ids for the same \
bytes.\nFix: pass --pattern gpt2, cl100k-base, or o200k-base.",
path.display()
)));
};
let extension = path
.extension()
.and_then(|e| e.to_str())
.unwrap_or_default()
.to_ascii_lowercase();
if extension != "tiktoken" && extension != "json" {
return Err(CliError::Usage(format!(
"cannot tell the format of {} from its extension.\nFix: use a .tiktoken rank \
file or a .json tokenizer bundle.",
path.display()
)));
}
match std::fs::File::open(path) {
Ok(file) => {
if file.metadata().is_ok_and(|found| found.is_dir()) {
return Err(CliError::Vocabulary(format!(
"the vocabulary artifact {} is a directory, not a rank file or a \
bundle.\nFix: name the file inside it.",
path.display()
)));
}
}
Err(source) => {
return Err(CliError::Vocabulary(format!(
"cannot read the vocabulary artifact {}: {source}\nFix: check the path exists \
and is readable, or pass --vocab <{}> for a provisioned vocabulary.",
path.display(),
known_names()
)));
}
}
let loaded = if extension == "tiktoken" {
Vocabulary::load_tiktoken_path(path)?
} else {
Vocabulary::load_tokenizer_json_path(path)?
};
Ok(Tokenizer::new(
loaded,
Encoding::new(path.display().to_string(), pattern.into()),
)?)
}
}
fn write_ids(out: &mut impl Write, ids: &[TokenId], format: IdFormat) -> std::io::Result<()> {
match format {
IdFormat::Ids => {
for (index, id) in ids.iter().enumerate() {
if index > 0 {
out.write_all(b" ")?;
}
write!(out, "{}", id.0)?;
}
if !ids.is_empty() {
out.write_all(b"\n")?;
}
}
IdFormat::Json => {
out.write_all(b"[")?;
for (index, id) in ids.iter().enumerate() {
if index > 0 {
out.write_all(b",")?;
}
write!(out, "{}", id.0)?;
}
out.write_all(b"]\n")?;
}
}
out.flush()
}
fn print_ids(ids: &[TokenId], format: IdFormat) -> Result<(), CliError> {
let stdout = std::io::stdout();
let mut out = BufWriter::new(stdout.lock());
write_ids(&mut out, ids, format).map_err(|source| output_error(&source))
}
#[derive(Debug, Clone, Copy)]
pub enum IdOrigin {
Argument,
Stdin,
}
impl IdOrigin {
fn fault(self, message: String) -> CliError {
match self {
Self::Argument => CliError::Usage(message),
Self::Stdin => CliError::Input(message),
}
}
}
fn read_ids_from_stdin() -> Result<String, CliError> {
let mut buffer = Vec::new();
std::io::stdin().read_to_end(&mut buffer).map_err(|source| {
CliError::Input(format!(
"cannot read the id list from stdin: {source}\nFix: pipe the ids in, or pass --ids."
))
})?;
String::from_utf8(buffer).map_err(|error| {
let offset = error.utf8_error().valid_up_to();
let byte = error.as_bytes().get(offset).copied().unwrap_or_default();
CliError::Input(format!(
"the id list on stdin is not text: byte {offset} is {byte:#04x}, which is not valid \
UTF-8.\nFix: pipe decimal ids separated by commas or spaces; raw bytes are what \
decode WRITES, not what it reads."
))
})
}
fn parse_ids(text: &str, origin: IdOrigin) -> Result<Vec<TokenId>, CliError> {
let mut ids = Vec::new();
for field in text.split([',', ' ', '\t', '\n', '\r']) {
if field.is_empty() {
continue;
}
let id: u32 = field.parse().map_err(|_| {
origin.fault(format!(
"{field:?} is not a token id.\nFix: pass decimal ids separated by commas or \
spaces, as encode prints them."
))
})?;
ids.push(TokenId(id));
}
Ok(ids)
}
fn undecodable(error: &UnknownTokenId, artifact: &str, origin: IdOrigin) -> CliError {
origin.fault(format!(
"cannot decode with {artifact}: {error}.\nFix: decode with the vocabulary the ids were \
produced by."
))
}
fn info_report(tokenizer: &Tokenizer) -> String {
let vocabulary = tokenizer.vocabulary();
let mut text = String::new();
text.push_str(&format!(
"pattern {}\n",
tokenizer.encoding().pattern().name()
));
text.push_str(&format!("unicode {}\n", unicode_version()));
text.push_str(&format!("tokens {}\n", vocabulary.len()));
text.push_str(&format!("pairs {}\n", vocabulary.merge_count()));
text.push_str(&format!("n_vocab {}\n", vocabulary.n_vocab()));
text.push_str(&format!("specials {}\n", vocabulary.special_len()));
for (special, id) in vocabulary.specials() {
text.push_str(&format!(" {id:>7} {special}\n", id = id.0));
}
text
}
fn parity_status_report() -> String {
let mut text = format!("vocabulary directory: {}\n", vocab::artifact_dir().display());
for vocabulary in NAMED_VOCABULARIES {
let status = match vocab::locate(vocabulary) {
Artifact::Present(path) => format!("present {}", path.display()),
Artifact::NoCache { .. } => "absent no rank files".to_owned(),
Artifact::Incomplete { .. } => "MISSING cache is incomplete".to_owned(),
};
text.push_str(&format!(
"{:<12} {:<14} {status}\n",
vocabulary.name,
vocabulary.pattern.name()
));
}
text
}
fn encode_on(
tokenizer: &Tokenizer,
bytes: &[u8],
device: DeviceArg,
) -> Result<Vec<TokenId>, CliError> {
match device {
DeviceArg::Cpu => Ok(tokenizer.encode(bytes)),
DeviceArg::Gpu => encode_on_cuda(tokenizer, bytes),
}
}
#[cfg(feature = "cuda")]
fn report_routing(routing: exatok::device::Routing) {
eprintln!(
"{NAME}: routed {} pretoken(s) to the device, {} to the host (longest host pretoken {} \
tokens)",
routing.on_device(),
routing.on_host(),
routing.largest_host_tokens()
);
}
#[cfg(feature = "cuda")]
fn encode_on_cuda(tokenizer: &Tokenizer, bytes: &[u8]) -> Result<Vec<TokenId>, CliError> {
let backend = vyre_driver_cuda::cuda_factory().map_err(|error| {
CliError::Device(format!(
"no usable CUDA device: {error}\nFix: run without --device gpu to encode on the \
host, or check that a CUDA device and its driver are present."
))
})?;
let device =
exatok::device::DeviceTokenizer::new(tokenizer.vocabulary(), tokenizer.encoding())?;
let encoded = device.encode_batch(backend.as_ref(), &[bytes])?;
report_routing(encoded.routing());
Ok(encoded.into_ids().pop().unwrap_or_default())
}
#[cfg(not(feature = "cuda"))]
fn encode_on_cuda(_tokenizer: &Tokenizer, _bytes: &[u8]) -> Result<Vec<TokenId>, CliError> {
Err(CliError::Device(format!(
"this build of {NAME} has no CUDA support, so --device gpu cannot be honored.\nFix: \
rebuild with `--features cuda`, or run without --device gpu to encode on the host."
)))
}
pub fn run(cli: Cli) -> Result<(), CliError> {
match cli.command {
Command::Encode {
vocab,
input,
format,
device,
} => {
let source = input.source()?;
let tokenizer = vocab.load()?;
let ids = encode_on(&tokenizer, &source.read()?, device)?;
print_ids(&ids, format)
}
Command::Count {
vocab,
input,
device,
} => {
let source = input.source()?;
let tokenizer = vocab.load()?;
let ids = encode_on(&tokenizer, &source.read()?, device)?;
emit(format!("{}\n", ids.len()).as_bytes())
}
Command::Decode { vocab, ids } => {
let tokenizer = vocab.load()?;
let (text, origin) = match ids {
Some(text) => (text, IdOrigin::Argument),
None => (read_ids_from_stdin()?, IdOrigin::Stdin),
};
let ids = parse_ids(&text, origin)?;
let decoded = tokenizer
.decode(&ids)
.map_err(|error| undecodable(&error, tokenizer.encoding().artifact(), origin))?;
emit(&decoded)
}
Command::Split { vocab, input } => {
let source = input.source()?;
let tokenizer = vocab.load()?;
let bytes = source.read()?;
let mut out = Vec::new();
for span in tokenizer.split(&bytes) {
let Some(pretoken) = span.bytes(&bytes) else {
return Err(CliError::Internal(format!(
"the pretokenizer returned {span:?}, which is outside the {} bytes it \
was given.\nFix: this is a defect in {NAME}; report it with the input \
that produced it.",
bytes.len()
)));
};
out.extend_from_slice(pretoken);
out.push(b'\n');
}
emit(&out)
}
Command::Info { vocab } => {
let tokenizer = vocab.load()?;
emit(info_report(&tokenizer).as_bytes())
}
Command::ParityStatus => emit(parity_status_report().as_bytes()),
}
}
fn main() {
let cli = match Cli::try_parse() {
Ok(cli) => cli,
Err(error) => {
let usage = error.use_stderr();
if error.print().is_err() {
std::process::exit(EXIT_OUTPUT);
}
if usage {
eprintln!("Fix: run `{NAME} --help` for the flag list and the exit-code table.");
std::process::exit(EXIT_USAGE);
}
std::process::exit(0);
}
};
if let Err(error) = run(cli) {
eprintln!("{NAME}: {error}");
std::process::exit(error.exit_code());
}
}