#[cfg(not(feature = "cuda"))]
fn main() {
println!(
"encode_profile measured nothing: it needs --features cuda and a CUDA device. \
Rerun with `cargo run --release --features cuda --example encode_profile`."
);
}
#[cfg(feature = "cuda")]
fn main() -> Result<(), String> {
profile::run()
}
#[cfg(feature = "cuda")]
mod profile {
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::time::{Duration, Instant};
use vyre_driver::VyreBackend;
use exatok::device::{DeviceTokenizer, EncodePhase, EncodeProfile};
use exatok::vocab::{self, Artifact, CL100K_BASE, Vocabulary};
use exatok::{Encoding, TokenId};
const SHAPES: [&str; 4] = ["short_pretokens", "prose", "cjk", "code"];
const WARMUP: usize = 3;
const REPEATS: usize = 15;
const STAGE_SIZES: [usize; 6] = [4, 4 << 10, 64 << 10, 1 << 20, 4 << 20, 16 << 20];
const FIT_MAX_BYTES: usize = 1 << 20;
const STAGE_REPEATS: usize = 12;
pub fn run() -> Result<(), String> {
let corpora = corpus_dir()?;
let vocabulary = load_cl100k()?;
let backend = vyre_driver_cuda::cuda_factory()
.map_err(|error| format!("CUDA backend acquisition failed: {error}"))?;
let backend: &dyn VyreBackend = &*backend;
let tokenizer = DeviceTokenizer::new(&vocabulary, &Encoding::cl100k_base())
.map_err(|error| format!("device merge table build failed: {error}"))?;
println!("exatok encode profile");
println!(" vocabulary cl100k_base");
println!(" corpora {}", corpora.display());
println!(" loadavg {}", loadavg());
println!(" gpu {}", gpu_state());
println!(
" table {} wire bytes",
tokenizer.table().slot_words().len() * 4
);
println!();
println!("staging model, COLD: no encode has run in this process yet");
let cold = stage_cost(backend)?;
let mut splits = Vec::new();
for shape in SHAPES {
let path = corpora.join(format!("{shape}.txt"));
let bytes = std::fs::read(&path).map_err(|error| {
format!(
"cannot read {}: {error}. Fix: point EXATOK_BENCH_CORPUS at a directory a \
throughput bench printed as `corpora:`.",
path.display()
)
})?;
splits.push(measure_shape(&tokenizer, backend, shape, &bytes)?);
}
println!();
println!(
"cold round trip {:.2}us, warm round trips per shape above",
cold.fixed.as_nanos() as f64 / 1000.0
);
for split in &splits {
println!(
" {:<17} warm {:.2}us per round trip, cold is {:.1}x that",
split.shape,
split.staging.fixed.as_nanos() as f64 / 1000.0,
cold.fixed.as_nanos() as f64 / split.staging.fixed.as_nanos().max(1) as f64
);
}
report_ownership(&splits);
Ok(())
}
#[derive(Clone, Copy)]
struct StageModel {
fixed: Duration,
per_byte: f64,
}
impl StageModel {
fn predict(self, operations: u64, bytes: u64) -> Duration {
let nanos =
self.fixed.as_nanos() as f64 * operations as f64 + self.per_byte * bytes as f64;
Duration::from_nanos(nanos.max(0.0) as u64)
}
}
fn measure_shape(
tokenizer: &DeviceTokenizer,
backend: &dyn VyreBackend,
shape: &str,
bytes: &[u8],
) -> Result<ShapeSplit, String> {
let inputs = [bytes];
for _ in 0..WARMUP {
encode(tokenizer, backend, &inputs)?;
}
let reference = encode(tokenizer, backend, &inputs)?;
let mut plain: Vec<Duration> = Vec::with_capacity(REPEATS);
for _ in 0..REPEATS {
let started = Instant::now();
let ids = encode(tokenizer, backend, &inputs)?;
plain.push(started.elapsed());
same_ids(&reference, &ids, shape)?;
}
let mut walls: Vec<Duration> = Vec::with_capacity(REPEATS);
let mut runs: Vec<EncodeProfile> = Vec::with_capacity(REPEATS);
for _ in 0..REPEATS {
let started = Instant::now();
let (encoded, profile) = tokenizer
.encode_batch_profiled(backend, &inputs)
.map_err(|error| format!("{shape}: profiled encode failed: {error}"))?;
walls.push(started.elapsed());
same_ids(
&reference,
encoded.ids().first().unwrap_or(&Vec::new()),
shape,
)?;
runs.push(profile);
}
let last = runs.last().ok_or("no profiled run was taken")?;
println!();
println!(
"shape {shape}: {} bytes, {} pretokens on device, {} on host, {} dispatches, \
{} state words, {} ids",
last.input_bytes(),
last.device_pretokens(),
last.host_pretokens(),
last.dispatches(),
last.state_words(),
last.output_ids()
);
println!(" loadavg {} gpu {}", loadavg(), gpu_state());
println!(
" {:<17}{:>11}{:>11}{:>11}{:>9}",
"component", "median", "min", "max", "share"
);
let totals: Vec<Duration> = runs.iter().map(EncodeProfile::total).collect();
let total_median = median(&totals);
for phase in EncodePhase::ALL {
let samples: Vec<Duration> = runs.iter().map(|run| run.phase(phase)).collect();
row(phase.label(), &samples, total_median);
}
let named: Vec<Duration> = runs.iter().map(EncodeProfile::named).collect();
let remainder: Vec<Duration> = runs.iter().map(EncodeProfile::unattributed).collect();
println!(" {:-<17}{:->42}", "", "");
row("NAMED TOTAL", &named, total_median);
row("UNATTRIBUTED", &remainder, total_median);
row("observed wall", &totals, total_median);
println!(" {:-<17}{:->42}", "", "");
row("caller wall", &walls, total_median);
row("caller wall plain", &plain, total_median);
let mut split = vyre_split(tokenizer, backend, shape, &inputs, &runs)?;
println!(" staging model, WARM and in-run for this shape");
split.staging = stage_cost(backend)?;
Ok(split)
}
fn vyre_split(
tokenizer: &DeviceTokenizer,
backend: &dyn VyreBackend,
shape: &str,
inputs: &[&[u8]],
runs: &[EncodeProfile],
) -> Result<ShapeSplit, String> {
let keys = [
"cuda_timed_wall_ns_total",
"cuda_timed_enqueue_ns_total",
"cuda_timed_device_ns_total",
"cuda_timed_wait_ns_total",
"cuda_timed_dispatches",
"cuda_timed_dispatches_missing_device_time",
"cuda_host_upload_operations",
"cuda_host_to_device_bytes",
"cuda_device_readback_operations",
"cuda_device_to_host_bytes",
"cuda_kernel_launches",
"cuda_sync_points",
];
let mut samples: BTreeMap<&str, Vec<u64>> = BTreeMap::new();
let mut dispatches: Vec<Duration> = Vec::with_capacity(REPEATS);
let mut table_bytes = 0;
for _ in 0..REPEATS {
let before = metrics(backend);
let (_, profile) = tokenizer
.encode_batch_profiled(backend, inputs)
.map_err(|error| format!("{shape}: counted encode failed: {error}"))?;
let after = metrics(backend);
for key in keys {
let step =
after.get(key).copied().unwrap_or(0) - before.get(key).copied().unwrap_or(0);
samples.entry(key).or_default().push(step);
}
dispatches.push(profile.phase(EncodePhase::Dispatch));
table_bytes = profile.table_bytes() as u64;
}
let delta = |key: &str| {
let mut values = samples.get(key).cloned().unwrap_or_default();
values.sort_unstable();
values.get(values.len() / 2).copied().unwrap_or(0)
};
let dispatch = median(&dispatches);
let timed_wall = Duration::from_nanos(delta("cuda_timed_wall_ns_total"));
let enqueue = Duration::from_nanos(delta("cuda_timed_enqueue_ns_total"));
let device = Duration::from_nanos(delta("cuda_timed_device_ns_total"));
let wait = Duration::from_nanos(delta("cuda_timed_wait_ns_total"));
let window_span = {
let mut values = samples
.get("cuda_timed_wall_ns_total")
.cloned()
.unwrap_or_default();
values.sort_unstable();
(
Duration::from_nanos(values.first().copied().unwrap_or(0)),
Duration::from_nanos(values.last().copied().unwrap_or(0)),
)
};
println!(" vyre split of the dispatch phase, from the CUDA backend's own counters");
println!(
" {:<28}{:>11} median of {REPEATS} counted encodes",
"dispatch phase",
millis(dispatch)
);
println!(
" {:<28}{:>11} spread {} to {}",
"vyre timed window",
millis(timed_wall),
millis(window_span.0),
millis(window_span.1)
);
println!(
" {:<28}{:>11} vyre host: prepare, PTX lookup, launch",
" enqueue",
millis(enqueue)
);
println!(
" {:<28}{:>11} host blocked plus output readback",
" wait",
millis(wait)
);
println!(
" {:<28}{:>11} CUDA events: kernel only",
" of which device",
millis(device)
);
let outside = dispatch.saturating_sub(timed_wall);
let table_operations = 1;
let staged_operations =
delta("cuda_host_upload_operations").saturating_sub(table_operations);
let staged_bytes = delta("cuda_host_to_device_bytes").saturating_sub(table_bytes);
println!(
" {:<28}{:>11} gpu_loop marshalling plus resident staging",
"outside the timed window",
millis(outside)
);
println!(
" {:<28}{:>11}",
"timed dispatches",
delta("cuda_timed_dispatches")
);
println!(
" {:<28}{:>11}",
"missing device time",
delta("cuda_timed_dispatches_missing_device_time")
);
println!(
" {:<28}{:>11}",
"host upload operations",
delta("cuda_host_upload_operations")
);
println!(
" {:<28}{:>11}",
"host to device bytes",
delta("cuda_host_to_device_bytes")
);
println!(
" {:<28}{:>11}",
"readback operations",
delta("cuda_device_readback_operations")
);
println!(
" {:<28}{:>11}",
"device to host bytes",
delta("cuda_device_to_host_bytes")
);
println!(
" {:<28}{:>11}",
"kernel launches",
delta("cuda_kernel_launches")
);
println!(" {:<28}{:>11}", "sync points", delta("cuda_sync_points"));
let median_phase = |phase: EncodePhase| {
let samples: Vec<Duration> = runs.iter().map(|run| run.phase(phase)).collect();
median(&samples)
};
let observed: Vec<Duration> = runs.iter().map(EncodeProfile::total).collect();
Ok(ShapeSplit {
shape: shape.to_owned(),
observed: median(&observed),
timed_wall,
device,
outside,
staged_operations,
staged_bytes,
table: median_phase(EncodePhase::TableUpload) + median_phase(EncodePhase::TableFree),
staging: StageModel {
fixed: Duration::ZERO,
per_byte: 0.0,
},
})
}
struct ShapeSplit {
shape: String,
observed: Duration,
timed_wall: Duration,
device: Duration,
outside: Duration,
staged_operations: u64,
staged_bytes: u64,
table: Duration,
staging: StageModel,
}
fn report_ownership(splits: &[ShapeSplit]) {
println!();
println!("ownership of the whole encode, in-run staging model per shape");
for split in splits {
let staged = split
.staging
.predict(split.staged_operations, split.staged_bytes);
if staged > split.outside {
println!(" {}", split.shape);
println!(
" staging model REFUSED: it prices {} staging round trips over {} bytes \
at {}, which exceeds the {} the dispatch phase spent outside vyre's timed \
window. The fit does not describe this run, so no ownership split is \
reported for this shape. Rerun on a quieter box.",
split.staged_operations,
split.staged_bytes,
millis(staged),
millis(split.outside)
);
continue;
}
let gpu_loop = split.outside - staged;
let vyre = split.timed_wall.saturating_sub(split.device) + staged + split.table;
let exatok = split
.observed
.saturating_sub(vyre)
.saturating_sub(split.device);
println!(" {}", split.shape);
println!(
" {:<28}{:>11}{:>8} table upload and free, dispatch window less device, staging",
"vyre",
millis(vyre),
share(vyre, split.observed)
);
println!(
" {:<28}{:>11}{:>8} CUDA event time on the kernel",
"device",
millis(split.device),
share(split.device, split.observed)
);
println!(
" {:<28}{:>11}{:>8} this crate, host",
"exatok",
millis(exatok),
share(exatok, split.observed)
);
println!(
" {:<28}{:>11}{:>8}",
"observed wall (median)",
millis(split.observed),
share(split.observed, split.observed)
);
println!(
" staging {} round trips over {} bytes priced at {}, leaving {} for gpu_loop",
split.staged_operations,
split.staged_bytes,
millis(staged),
millis(gpu_loop)
);
}
}
fn stage_cost(backend: &dyn VyreBackend) -> Result<StageModel, String> {
println!();
println!("resident staging: allocate + upload + free, measured at each size");
println!(" loadavg {} gpu {}", loadavg(), gpu_state());
println!(
" {:<14}{:>11}{:>11}{:>11}",
"bytes", "median", "min", "max"
);
let mut fit: Vec<(f64, f64)> = Vec::with_capacity(STAGE_SIZES.len());
for size in STAGE_SIZES {
let payload = vec![0_u8; size];
stage_once(backend, &payload)?;
let mut samples = Vec::with_capacity(STAGE_REPEATS);
for _ in 0..STAGE_REPEATS {
samples.push(stage_once(backend, &payload)?);
}
samples.sort_unstable();
let center = median(&samples);
if size <= FIT_MAX_BYTES {
fit.push((size as f64, center.as_nanos() as f64));
}
println!(
" {size:<14}{:>11}{:>11}{:>11}",
micros(center),
micros(samples.first().copied().unwrap_or_default()),
micros(samples.last().copied().unwrap_or_default())
);
}
let model = least_squares(&fit);
println!(
" fitted over {} of {} points, up to {FIT_MAX_BYTES} bytes: {:.2}us per round trip \
+ {:.4}us per KiB",
fit.len(),
STAGE_SIZES.len(),
model.fixed.as_nanos() as f64 / 1000.0,
model.per_byte * 1024.0 / 1000.0
);
for (bytes, measured) in &fit {
let predicted = model.predict(1, *bytes as u64).as_nanos() as f64;
println!(
" {:<12} measured {:>9.2}us predicted {:>9.2}us residual {:>+8.2}us",
*bytes as u64,
measured / 1000.0,
predicted / 1000.0,
(predicted - measured) / 1000.0
);
}
Ok(model)
}
fn least_squares(points: &[(f64, f64)]) -> StageModel {
let count = points.len() as f64;
if count == 0.0 {
return StageModel {
fixed: Duration::ZERO,
per_byte: 0.0,
};
}
let mean_x = points.iter().map(|point| point.0).sum::<f64>() / count;
let mean_y = points.iter().map(|point| point.1).sum::<f64>() / count;
let covariance: f64 = points
.iter()
.map(|point| (point.0 - mean_x) * (point.1 - mean_y))
.sum();
let variance: f64 = points
.iter()
.map(|point| (point.0 - mean_x) * (point.0 - mean_x))
.sum();
let per_byte = if variance > 0.0 {
covariance / variance
} else {
0.0
};
let fixed = (mean_y - per_byte * mean_x).max(0.0);
StageModel {
fixed: Duration::from_nanos(fixed as u64),
per_byte,
}
}
fn share(part: Duration, whole: Duration) -> String {
if whole.is_zero() {
return " n/a".to_owned();
}
format!("{:.1}%", 100.0 * part.as_secs_f64() / whole.as_secs_f64())
}
fn stage_once(backend: &dyn VyreBackend, payload: &[u8]) -> Result<Duration, String> {
let started = Instant::now();
let resource = backend
.allocate_resident(payload.len())
.map_err(|error| format!("resident allocation of {} failed: {error}", payload.len()))?;
let uploaded = backend.upload_resident(&resource, payload);
let freed = backend.free_resident(resource);
let elapsed = started.elapsed();
uploaded.map_err(|error| format!("resident upload failed: {error}"))?;
freed.map_err(|error| format!("resident free failed: {error}"))?;
Ok(elapsed)
}
fn encode(
tokenizer: &DeviceTokenizer,
backend: &dyn VyreBackend,
inputs: &[&[u8]],
) -> Result<Vec<TokenId>, String> {
let encoded = tokenizer
.encode_batch(backend, inputs)
.map_err(|error| format!("encode failed: {error}"))?;
Ok(encoded.into_ids().pop().unwrap_or_default())
}
fn same_ids(reference: &[TokenId], ids: &[TokenId], shape: &str) -> Result<(), String> {
if reference == ids {
return Ok(());
}
Err(format!(
"{shape}: two encodes of one input returned different ids, {} against {}",
reference.len(),
ids.len()
))
}
fn row(label: &str, samples: &[Duration], total: Duration) {
let mut sorted = samples.to_vec();
sorted.sort_unstable();
let middle = median(&sorted);
let share = if total.is_zero() {
0.0
} else {
100.0 * middle.as_secs_f64() / total.as_secs_f64()
};
println!(
" {label:<17}{:>11}{:>11}{:>11}{:>8.1}%",
millis(middle),
millis(sorted.first().copied().unwrap_or_default()),
millis(sorted.last().copied().unwrap_or_default()),
share
);
}
fn median(samples: &[Duration]) -> Duration {
let mut sorted = samples.to_vec();
sorted.sort_unstable();
sorted.get(sorted.len() / 2).copied().unwrap_or_default()
}
fn millis(value: Duration) -> String {
format!("{:.4}ms", value.as_secs_f64() * 1e3)
}
fn micros(value: Duration) -> String {
format!("{:.2}us", value.as_secs_f64() * 1e6)
}
fn metrics(backend: &dyn VyreBackend) -> BTreeMap<&'static str, u64> {
backend.backend_metric_snapshot().into_iter().collect()
}
fn gpu_state() -> String {
let output = std::process::Command::new("nvidia-smi")
.args([
"--query-gpu=utilization.gpu,memory.used,memory.total",
"--format=csv,noheader,nounits",
])
.output();
match output {
Ok(done) if done.status.success() => {
let text = String::from_utf8_lossy(&done.stdout);
let fields: Vec<&str> = text.trim().split(',').map(str::trim).collect();
match fields.as_slice() {
[used_pct, used_mib, total_mib] => {
format!("{used_pct}% busy, {used_mib} of {total_mib} MiB in use")
}
_ => text.trim().to_owned(),
}
}
_ => "unavailable (nvidia-smi did not answer)".to_owned(),
}
}
fn loadavg() -> String {
std::fs::read_to_string("/proc/loadavg").map_or_else(
|_| "unavailable".to_owned(),
|text| {
text.split_whitespace()
.take(3)
.collect::<Vec<_>>()
.join(" ")
},
)
}
fn corpus_dir() -> Result<PathBuf, String> {
std::env::var_os("EXATOK_BENCH_CORPUS")
.map(PathBuf::from)
.ok_or_else(|| {
"EXATOK_BENCH_CORPUS is unset. Fix: set it to a directory a throughput bench \
printed as `corpora:`, which holds prose.txt and short_pretokens.txt."
.to_owned()
})
}
fn load_cl100k() -> Result<Vocabulary, String> {
match vocab::locate(CL100K_BASE) {
Artifact::Present(path) => Vocabulary::load_tiktoken_path(&path)
.map_err(|error| format!("cannot load {}: {error}", path.display())),
other => Err(format!(
"cl100k_base rank file is not provisioned: {other:?}. Fix: place it under \
$EXATOK_VOCAB_DIR (default ~/.cache/exatok-vocab)."
)),
}
}
}