oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
use oxidelm::gpu::GpuContext;
use oxidelm::model::{CausalLM, CausalLmConfig, KvCache};
use oxidelm::{cpu_memory_stats, reset_cpu_peak_memory, IndexTensor, Tensor};
use serde::Serialize;
use std::env;
use std::error::Error;
use std::fs;
use std::hint::black_box;
use std::path::PathBuf;
use std::process::ExitCode;

const PROMPT_LENGTH: usize = 64;
const DECODE_LENGTH: usize = 32;

#[derive(Clone, Copy, PartialEq, Eq)]
enum DeviceSelection {
    All,
    Cpu,
    Gpu,
}

#[derive(Clone, Copy)]
enum Backend<'a> {
    Cpu,
    Gpu(&'a GpuContext),
}

struct Arguments {
    device: DeviceSelection,
    json_path: Option<PathBuf>,
}

#[derive(Clone, Copy, Debug)]
struct MemoryReport {
    peak_tensor_bytes: usize,
    live_tensor_bytes: usize,
}

#[derive(Serialize)]
struct JsonResults {
    model: ModelMetadata,
    prompt_length: usize,
    decode_length: usize,
    cpu: Option<MemoryReportJson>,
    gpu: Option<MemoryReportJson>,
}

#[derive(Serialize)]
struct ModelMetadata {
    vocab_size: usize,
    hidden_size: usize,
    num_layers: usize,
    num_heads: usize,
    intermediate_size: usize,
}

#[derive(Serialize)]
struct MemoryReportJson {
    peak_tensor_bytes: usize,
    live_tensor_bytes: usize,
}

fn main() -> ExitCode {
    match run() {
        Ok(()) => ExitCode::SUCCESS,
        Err(error) => {
            eprintln!("error: {error}");
            ExitCode::from(2)
        }
    }
}

fn run() -> Result<(), Box<dyn Error>> {
    let arguments = parse_arguments(&env::args().skip(1).collect::<Vec<_>>())?;
    let config = CausalLmConfig {
        vocab_size: 320,
        hidden_size: 64,
        num_layers: 2,
        num_heads: 4,
        num_key_value_heads: 2,
        intermediate_size: 128,
        max_sequence_length: PROMPT_LENGTH + DECODE_LENGTH,
        rms_norm_epsilon: 1e-5,
        rope_theta: 10_000.0,
        rope_scaling: None,
        eos_token_id: 319,
        seed: 42,
        initializer_std: 0.02,
        tie_word_embeddings: true,
        attention_dropout: 0.0,
        residual_dropout: 0.0,
        embedding_dropout: 0.0,
        use_bias: false,
    };
    let model = CausalLM::new(config.clone())?;
    let prompt = token_sequence(0, PROMPT_LENGTH);

    println!("Allocation profiling during decode (FP32, batch=1)");
    println!(
        "model: vocab={} hidden={} layers={} heads={} intermediate={} prompt={} decode={}",
        config.vocab_size,
        config.hidden_size,
        config.num_layers,
        config.num_heads,
        config.intermediate_size,
        PROMPT_LENGTH,
        DECODE_LENGTH,
    );

    let mut cpu_report = None;
    let mut gpu_report = None;

    if arguments.device != DeviceSelection::Gpu {
        let report = profile_backend(&model, &prompt, Backend::Cpu)?;
        print_report("CPU", None, report);
        cpu_report = Some(report);
    }

    if arguments.device != DeviceSelection::Cpu {
        match GpuContext::new() {
            Ok(gpu) => {
                let report = profile_backend(&model, &prompt, Backend::Gpu(&gpu))?;
                print_report("GPU", Some(gpu.adapter_name()), report);
                gpu_report = Some(report);
            }
            Err(error) if arguments.device == DeviceSelection::All => {
                println!("\nGPU: skipped ({error})");
            }
            Err(error) => return Err(error.into()),
        }
    }

    if let Some(path) = arguments.json_path {
        let json = JsonResults {
            model: ModelMetadata {
                vocab_size: config.vocab_size,
                hidden_size: config.hidden_size,
                num_layers: config.num_layers,
                num_heads: config.num_heads,
                intermediate_size: config.intermediate_size,
            },
            prompt_length: PROMPT_LENGTH,
            decode_length: DECODE_LENGTH,
            cpu: cpu_report.map(memory_report_json),
            gpu: gpu_report.map(memory_report_json),
        };
        fs::write(&path, serde_json::to_string_pretty(&json)?)?;
        println!("\nwrote results JSON: {}", path.display());
    }

    Ok(())
}

fn profile_backend(
    model: &CausalLM,
    prompt: &[usize],
    backend: Backend<'_>,
) -> Result<MemoryReport, Box<dyn Error>> {
    if let Backend::Gpu(gpu) = backend {
        model.prepare_gpu(gpu)?;
    }

    reset_cpu_peak_memory();
    if let Backend::Gpu(gpu) = backend {
        gpu.reset_memory_stats();
    }

    let (mut cache, mut next_token) = prefill_next_token(model, prompt, backend)?;
    for _ in 0..DECODE_LENGTH {
        let logits = forward_cached(model, &[next_token], &mut cache, backend)?;
        next_token = readable_argmax(&logits, model.config.vocab_size)?;
        black_box(next_token);
    }

    match backend {
        Backend::Cpu => {
            let stats = cpu_memory_stats();
            Ok(MemoryReport {
                peak_tensor_bytes: stats.peak_tensor_bytes,
                live_tensor_bytes: stats.live_tensor_bytes,
            })
        }
        Backend::Gpu(gpu) => {
            let stats = gpu.memory_stats()?;
            Ok(MemoryReport {
                peak_tensor_bytes: stats.peak_tensor_bytes,
                live_tensor_bytes: stats.live_tensor_bytes,
            })
        }
    }
}

fn prefill_next_token(
    model: &CausalLM,
    prompt: &[usize],
    backend: Backend<'_>,
) -> Result<(KvCache, usize), Box<dyn Error>> {
    let mut cache = KvCache::new(model.layers.len());
    let logits = forward_cached(model, prompt, &mut cache, backend)?;
    let next_token = readable_argmax(&logits, model.config.vocab_size)?;
    Ok((cache, next_token))
}

fn forward_cached(
    model: &CausalLM,
    tokens: &[usize],
    cache: &mut KvCache,
    backend: Backend<'_>,
) -> Result<Tensor, Box<dyn Error>> {
    let input = input(tokens);
    match backend {
        Backend::Cpu => Ok(model.forward_cached(&input, cache)?),
        Backend::Gpu(gpu) => Ok(model.forward_cached_gpu(&input, cache, gpu)?),
    }
}

fn readable_argmax(logits: &Tensor, width: usize) -> Result<usize, Box<dyn Error>> {
    let readable = logits.to_cpu()?;
    Ok(argmax_last_row(&readable, width))
}

fn print_report(label: &str, adapter: Option<&str>, report: MemoryReport) {
    println!();
    match adapter {
        Some(name) => println!("{label} ({name}):"),
        None => println!("{label}:"),
    }
    println!(
        "  peak tensor bytes: {} ({})",
        report.peak_tensor_bytes,
        format_bytes(report.peak_tensor_bytes)
    );
    println!(
        "  live tensor bytes: {} ({})",
        report.live_tensor_bytes,
        format_bytes(report.live_tensor_bytes)
    );
}

fn memory_report_json(report: MemoryReport) -> MemoryReportJson {
    MemoryReportJson {
        peak_tensor_bytes: report.peak_tensor_bytes,
        live_tensor_bytes: report.live_tensor_bytes,
    }
}

fn format_bytes(bytes: usize) -> String {
    const KIB: f64 = 1024.0;
    const MIB: f64 = KIB * 1024.0;
    const GIB: f64 = MIB * 1024.0;
    let value = bytes as f64;
    if value >= GIB {
        format!("{:.2} GiB", value / GIB)
    } else if value >= MIB {
        format!("{:.2} MiB", value / MIB)
    } else if value >= KIB {
        format!("{:.2} KiB", value / KIB)
    } else {
        format!("{bytes} B")
    }
}

fn parse_arguments(arguments: &[String]) -> Result<Arguments, Box<dyn Error>> {
    let mut parsed = Arguments {
        device: DeviceSelection::All,
        json_path: None,
    };
    let mut index = 0;
    while index < arguments.len() {
        match arguments[index].as_str() {
            "--device" => {
                let value = arguments
                    .get(index + 1)
                    .ok_or("missing value for --device")?;
                parsed.device = match value.as_str() {
                    "all" => DeviceSelection::All,
                    "cpu" => DeviceSelection::Cpu,
                    "gpu" => DeviceSelection::Gpu,
                    _ => {
                        return Err(format!("invalid device '{value}': use all, cpu, or gpu").into())
                    }
                };
                index += 2;
            }
            "--json" => {
                let path = arguments.get(index + 1).ok_or("missing value for --json")?;
                parsed.json_path = Some(PathBuf::from(path));
                index += 2;
            }
            other => return Err(format!("unknown argument: {other}").into()),
        }
    }
    Ok(parsed)
}

fn input(tokens: &[usize]) -> IndexTensor {
    IndexTensor::new(tokens.to_vec(), vec![1, tokens.len()])
}

fn token_sequence(offset: usize, length: usize) -> Vec<usize> {
    (offset..offset + length)
        .map(|index| (index * 37 + 11) % 319)
        .collect()
}

fn argmax_last_row(logits: &Tensor, width: usize) -> usize {
    let data = logits.data();
    data[data.len() - width..]
        .iter()
        .enumerate()
        .max_by(|(_, left), (_, right)| left.total_cmp(right))
        .map(|(index, _)| index)
        .unwrap()
}