use oxidelm::gpu::GpuContext;
use oxidelm::model::{CausalLM, CausalLmConfig, KvCache};
use oxidelm::{cpu_memory_stats, reset_cpu_peak_memory, IndexTensor, Tensor};
use serde::Serialize;
use std::env;
use std::error::Error;
use std::fs;
use std::hint::black_box;
use std::path::PathBuf;
use std::process::ExitCode;
const PROMPT_LENGTH: usize = 64;
const DECODE_LENGTH: usize = 32;
#[derive(Clone, Copy, PartialEq, Eq)]
enum DeviceSelection {
All,
Cpu,
Gpu,
}
#[derive(Clone, Copy)]
enum Backend<'a> {
Cpu,
Gpu(&'a GpuContext),
}
struct Arguments {
device: DeviceSelection,
json_path: Option<PathBuf>,
}
#[derive(Clone, Copy, Debug)]
struct MemoryReport {
peak_tensor_bytes: usize,
live_tensor_bytes: usize,
}
#[derive(Serialize)]
struct JsonResults {
model: ModelMetadata,
prompt_length: usize,
decode_length: usize,
cpu: Option<MemoryReportJson>,
gpu: Option<MemoryReportJson>,
}
#[derive(Serialize)]
struct ModelMetadata {
vocab_size: usize,
hidden_size: usize,
num_layers: usize,
num_heads: usize,
intermediate_size: usize,
}
#[derive(Serialize)]
struct MemoryReportJson {
peak_tensor_bytes: usize,
live_tensor_bytes: usize,
}
fn main() -> ExitCode {
match run() {
Ok(()) => ExitCode::SUCCESS,
Err(error) => {
eprintln!("error: {error}");
ExitCode::from(2)
}
}
}
fn run() -> Result<(), Box<dyn Error>> {
let arguments = parse_arguments(&env::args().skip(1).collect::<Vec<_>>())?;
let config = CausalLmConfig {
vocab_size: 320,
hidden_size: 64,
num_layers: 2,
num_heads: 4,
num_key_value_heads: 2,
intermediate_size: 128,
max_sequence_length: PROMPT_LENGTH + DECODE_LENGTH,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: 319,
seed: 42,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
};
let model = CausalLM::new(config.clone())?;
let prompt = token_sequence(0, PROMPT_LENGTH);
println!("Allocation profiling during decode (FP32, batch=1)");
println!(
"model: vocab={} hidden={} layers={} heads={} intermediate={} prompt={} decode={}",
config.vocab_size,
config.hidden_size,
config.num_layers,
config.num_heads,
config.intermediate_size,
PROMPT_LENGTH,
DECODE_LENGTH,
);
let mut cpu_report = None;
let mut gpu_report = None;
if arguments.device != DeviceSelection::Gpu {
let report = profile_backend(&model, &prompt, Backend::Cpu)?;
print_report("CPU", None, report);
cpu_report = Some(report);
}
if arguments.device != DeviceSelection::Cpu {
match GpuContext::new() {
Ok(gpu) => {
let report = profile_backend(&model, &prompt, Backend::Gpu(&gpu))?;
print_report("GPU", Some(gpu.adapter_name()), report);
gpu_report = Some(report);
}
Err(error) if arguments.device == DeviceSelection::All => {
println!("\nGPU: skipped ({error})");
}
Err(error) => return Err(error.into()),
}
}
if let Some(path) = arguments.json_path {
let json = JsonResults {
model: ModelMetadata {
vocab_size: config.vocab_size,
hidden_size: config.hidden_size,
num_layers: config.num_layers,
num_heads: config.num_heads,
intermediate_size: config.intermediate_size,
},
prompt_length: PROMPT_LENGTH,
decode_length: DECODE_LENGTH,
cpu: cpu_report.map(memory_report_json),
gpu: gpu_report.map(memory_report_json),
};
fs::write(&path, serde_json::to_string_pretty(&json)?)?;
println!("\nwrote results JSON: {}", path.display());
}
Ok(())
}
fn profile_backend(
model: &CausalLM,
prompt: &[usize],
backend: Backend<'_>,
) -> Result<MemoryReport, Box<dyn Error>> {
if let Backend::Gpu(gpu) = backend {
model.prepare_gpu(gpu)?;
}
reset_cpu_peak_memory();
if let Backend::Gpu(gpu) = backend {
gpu.reset_memory_stats();
}
let (mut cache, mut next_token) = prefill_next_token(model, prompt, backend)?;
for _ in 0..DECODE_LENGTH {
let logits = forward_cached(model, &[next_token], &mut cache, backend)?;
next_token = readable_argmax(&logits, model.config.vocab_size)?;
black_box(next_token);
}
match backend {
Backend::Cpu => {
let stats = cpu_memory_stats();
Ok(MemoryReport {
peak_tensor_bytes: stats.peak_tensor_bytes,
live_tensor_bytes: stats.live_tensor_bytes,
})
}
Backend::Gpu(gpu) => {
let stats = gpu.memory_stats()?;
Ok(MemoryReport {
peak_tensor_bytes: stats.peak_tensor_bytes,
live_tensor_bytes: stats.live_tensor_bytes,
})
}
}
}
fn prefill_next_token(
model: &CausalLM,
prompt: &[usize],
backend: Backend<'_>,
) -> Result<(KvCache, usize), Box<dyn Error>> {
let mut cache = KvCache::new(model.layers.len());
let logits = forward_cached(model, prompt, &mut cache, backend)?;
let next_token = readable_argmax(&logits, model.config.vocab_size)?;
Ok((cache, next_token))
}
fn forward_cached(
model: &CausalLM,
tokens: &[usize],
cache: &mut KvCache,
backend: Backend<'_>,
) -> Result<Tensor, Box<dyn Error>> {
let input = input(tokens);
match backend {
Backend::Cpu => Ok(model.forward_cached(&input, cache)?),
Backend::Gpu(gpu) => Ok(model.forward_cached_gpu(&input, cache, gpu)?),
}
}
fn readable_argmax(logits: &Tensor, width: usize) -> Result<usize, Box<dyn Error>> {
let readable = logits.to_cpu()?;
Ok(argmax_last_row(&readable, width))
}
fn print_report(label: &str, adapter: Option<&str>, report: MemoryReport) {
println!();
match adapter {
Some(name) => println!("{label} ({name}):"),
None => println!("{label}:"),
}
println!(
" peak tensor bytes: {} ({})",
report.peak_tensor_bytes,
format_bytes(report.peak_tensor_bytes)
);
println!(
" live tensor bytes: {} ({})",
report.live_tensor_bytes,
format_bytes(report.live_tensor_bytes)
);
}
fn memory_report_json(report: MemoryReport) -> MemoryReportJson {
MemoryReportJson {
peak_tensor_bytes: report.peak_tensor_bytes,
live_tensor_bytes: report.live_tensor_bytes,
}
}
fn format_bytes(bytes: usize) -> String {
const KIB: f64 = 1024.0;
const MIB: f64 = KIB * 1024.0;
const GIB: f64 = MIB * 1024.0;
let value = bytes as f64;
if value >= GIB {
format!("{:.2} GiB", value / GIB)
} else if value >= MIB {
format!("{:.2} MiB", value / MIB)
} else if value >= KIB {
format!("{:.2} KiB", value / KIB)
} else {
format!("{bytes} B")
}
}
fn parse_arguments(arguments: &[String]) -> Result<Arguments, Box<dyn Error>> {
let mut parsed = Arguments {
device: DeviceSelection::All,
json_path: None,
};
let mut index = 0;
while index < arguments.len() {
match arguments[index].as_str() {
"--device" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --device")?;
parsed.device = match value.as_str() {
"all" => DeviceSelection::All,
"cpu" => DeviceSelection::Cpu,
"gpu" => DeviceSelection::Gpu,
_ => {
return Err(format!("invalid device '{value}': use all, cpu, or gpu").into())
}
};
index += 2;
}
"--json" => {
let path = arguments.get(index + 1).ok_or("missing value for --json")?;
parsed.json_path = Some(PathBuf::from(path));
index += 2;
}
other => return Err(format!("unknown argument: {other}").into()),
}
}
Ok(parsed)
}
fn input(tokens: &[usize]) -> IndexTensor {
IndexTensor::new(tokens.to_vec(), vec![1, tokens.len()])
}
fn token_sequence(offset: usize, length: usize) -> Vec<usize> {
(offset..offset + length)
.map(|index| (index * 37 + 11) % 319)
.collect()
}
fn argmax_last_row(logits: &Tensor, width: usize) -> usize {
let data = logits.data();
data[data.len() - width..]
.iter()
.enumerate()
.max_by(|(_, left), (_, right)| left.total_cmp(right))
.map(|(index, _)| index)
.unwrap()
}