use crate::model::CausalLmConfig;
use crate::trainer::TrainerConfig;
use std::error::Error;
use std::fmt;
pub const BYTES_PER_F32: usize = 4;
pub const DEFAULT_CPU_BUDGET_BYTES: usize = 32 * 1024 * 1024 * 1024;
pub const DEFAULT_HOST_MEMORY_FRACTION_NUMERATOR: usize = 70;
pub const DEFAULT_HOST_MEMORY_FRACTION_DENOMINATOR: usize = 100;
pub const DEFAULT_GPU_BUDGET_CAP_BYTES: usize = 16 * 1024 * 1024 * 1024;
pub const TINY_HOST_MEMORY_BYTES: usize = 16 * 1024 * 1024 * 1024;
const GPU_BUDGET_FROM_MAX_BUFFER_FACTOR: usize = 4;
const OBVIOUS_OVERFLOW_NUMERATOR: usize = 95;
const OBVIOUS_OVERFLOW_DENOMINATOR: usize = 100;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MemoryMode {
Inference,
Training,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MemoryEstimate {
pub parameter_bytes: usize,
pub amp_master_bytes: usize,
pub activation_bytes: usize,
pub cache_bytes: usize,
pub staging_bytes: usize,
pub optimizer_bytes: usize,
pub gradient_bytes: usize,
pub total_bytes: usize,
pub largest_tensor_bytes: usize,
pub batch_size: usize,
pub sequence_length: usize,
pub mode: MemoryMode,
pub prefetch_batches: usize,
pub data_parallel_workers: usize,
pub activation_checkpoint_interval: usize,
pub optimizer_moment_bytes: usize,
pub notes: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DeviceBudget {
pub label: String,
pub total_bytes: usize,
pub max_buffer_bytes: usize,
}
impl DeviceBudget {
pub fn cpu(total_bytes: Option<usize>) -> Self {
Self {
label: "cpu".into(),
total_bytes: total_bytes.unwrap_or_else(default_cpu_budget_bytes),
max_buffer_bytes: usize::MAX,
}
}
pub fn gpu(
label: impl Into<String>,
max_buffer_bytes: usize,
total_bytes: Option<usize>,
) -> Self {
let derived = max_buffer_bytes
.saturating_mul(GPU_BUDGET_FROM_MAX_BUFFER_FACTOR)
.min(DEFAULT_GPU_BUDGET_CAP_BYTES)
.max(max_buffer_bytes);
Self {
label: label.into(),
total_bytes: total_bytes.unwrap_or(derived),
max_buffer_bytes: max_buffer_bytes.max(1),
}
}
}
#[derive(Debug)]
pub enum MemoryBudgetError {
ExceedsDeviceBudget {
label: String,
estimated_bytes: usize,
budget_bytes: usize,
estimate: MemoryEstimate,
},
ExceedsMaxBuffer {
label: String,
required_bytes: usize,
max_buffer_bytes: usize,
estimate: MemoryEstimate,
},
}
impl fmt::Display for MemoryBudgetError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::ExceedsDeviceBudget {
label,
estimated_bytes,
budget_bytes,
estimate,
} => write!(
formatter,
"memory estimate {} exceeds {} budget {} ({})\n{}",
format_bytes(*estimated_bytes),
label,
format_bytes(*budget_bytes),
summarize_estimate(estimate),
format_memory_remediation(estimate, *budget_bytes)
),
Self::ExceedsMaxBuffer {
label,
required_bytes,
max_buffer_bytes,
estimate,
} => write!(
formatter,
"largest tensor {} exceeds {} max buffer {} ({})\n{}",
format_bytes(*required_bytes),
label,
format_bytes(*max_buffer_bytes),
summarize_estimate(estimate),
format_memory_remediation(estimate, *max_buffer_bytes)
),
}
}
}
impl Error for MemoryBudgetError {}
pub fn default_cpu_budget_bytes() -> usize {
detect_usable_host_memory_bytes()
.map(|bytes| {
bytes
.saturating_mul(DEFAULT_HOST_MEMORY_FRACTION_NUMERATOR)
/ DEFAULT_HOST_MEMORY_FRACTION_DENOMINATOR
})
.filter(|bytes| *bytes > 0)
.unwrap_or(DEFAULT_CPU_BUDGET_BYTES)
}
pub fn host_memory_is_tiny() -> bool {
detect_usable_host_memory_bytes()
.map(|bytes| bytes > 0 && bytes <= TINY_HOST_MEMORY_BYTES)
.unwrap_or(false)
}
pub fn cap_gpu_budget_for_tiny_host(budget_bytes: Option<usize>) -> Option<usize> {
if !host_memory_is_tiny() {
return budget_bytes;
}
let host_cap = default_cpu_budget_bytes();
Some(budget_bytes.map(|bytes| bytes.min(host_cap)).unwrap_or(host_cap))
}
pub fn detect_usable_host_memory_bytes() -> Option<usize> {
#[cfg(target_os = "linux")]
{
return parse_linux_meminfo_usable();
}
#[cfg(target_os = "macos")]
{
return macos_hw_memsize();
}
#[cfg(target_os = "windows")]
{
return windows_available_memory();
}
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
{
None
}
}
#[cfg(target_os = "linux")]
fn parse_linux_meminfo_usable() -> Option<usize> {
let text = std::fs::read_to_string("/proc/meminfo").ok()?;
let mut available_kb = None;
let mut total_kb = None;
for line in text.lines() {
let mut parts = line.split_whitespace();
let key = parts.next()?;
let value = parts.next()?.parse::<usize>().ok()?;
match key {
"MemAvailable:" => available_kb = Some(value),
"MemTotal:" => total_kb = Some(value),
_ => {}
}
}
available_kb
.or(total_kb)
.map(|kib| kib.saturating_mul(1024))
}
#[cfg(target_os = "macos")]
fn macos_hw_memsize() -> Option<usize> {
let mut size: u64 = 0;
let mut length = std::mem::size_of::<u64>();
let name = std::ffi::CString::new("hw.memsize").ok()?;
let status = unsafe {
sysctlbyname(
name.as_ptr(),
&mut size as *mut u64 as *mut std::ffi::c_void,
&mut length,
std::ptr::null_mut(),
0,
)
};
if status == 0 && size > 0 {
Some(size as usize)
} else {
None
}
}
#[cfg(target_os = "macos")]
extern "C" {
fn sysctlbyname(
name: *const std::ffi::c_char,
oldp: *mut std::ffi::c_void,
oldlenp: *mut usize,
newp: *mut std::ffi::c_void,
newlen: usize,
) -> i32;
}
#[cfg(target_os = "windows")]
fn windows_available_memory() -> Option<usize> {
#[repr(C)]
struct MemoryStatusEx {
length: u32,
memory_load: u32,
total_phys: u64,
avail_phys: u64,
total_page_file: u64,
avail_page_file: u64,
total_virtual: u64,
avail_virtual: u64,
avail_extended_virtual: u64,
}
extern "system" {
fn GlobalMemoryStatusEx(status: *mut MemoryStatusEx) -> i32;
}
let mut status = MemoryStatusEx {
length: std::mem::size_of::<MemoryStatusEx>() as u32,
memory_load: 0,
total_phys: 0,
avail_phys: 0,
total_page_file: 0,
avail_page_file: 0,
total_virtual: 0,
avail_virtual: 0,
avail_extended_virtual: 0,
};
let ok = unsafe { GlobalMemoryStatusEx(&mut status) };
if ok == 0 {
return None;
}
let bytes = if status.avail_phys > 0 {
status.avail_phys
} else {
status.total_phys
};
if bytes == 0 {
None
} else {
Some(bytes as usize)
}
}
pub fn process_peak_rss_bytes() -> Option<usize> {
#[cfg(target_os = "linux")]
{
return parse_linux_status_kb("VmHWM:").map(|kib| kib.saturating_mul(1024));
}
#[cfg(target_os = "macos")]
{
return macos_peak_rss_bytes();
}
#[cfg(target_os = "windows")]
{
return windows_peak_working_set_bytes();
}
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
{
None
}
}
#[cfg(target_os = "linux")]
fn parse_linux_status_kb(key: &str) -> Option<usize> {
let text = std::fs::read_to_string("/proc/self/status").ok()?;
for line in text.lines() {
let mut parts = line.split_whitespace();
if parts.next()? != key {
continue;
}
return parts.next()?.parse::<usize>().ok();
}
None
}
#[cfg(target_os = "macos")]
fn macos_peak_rss_bytes() -> Option<usize> {
#[repr(C)]
#[derive(Clone, Copy, Default)]
struct TimeValue {
seconds: i32,
microseconds: i32,
}
#[repr(C)]
#[derive(Clone, Copy, Default)]
struct MachTaskBasicInfo {
virtual_size: u64,
resident_size: u64,
resident_size_max: u64,
user_time: TimeValue,
system_time: TimeValue,
policy: i32,
suspend_count: i32,
}
const MACH_TASK_BASIC_INFO: u32 = 20;
const COUNT: u32 =
(std::mem::size_of::<MachTaskBasicInfo>() / std::mem::size_of::<u32>()) as u32;
extern "C" {
fn mach_task_self() -> u32;
fn task_info(
target_task: u32,
flavor: u32,
task_info_out: *mut MachTaskBasicInfo,
task_info_count: *mut u32,
) -> i32;
}
let mut info = MachTaskBasicInfo::default();
let mut count = COUNT;
let status = unsafe { task_info(mach_task_self(), MACH_TASK_BASIC_INFO, &mut info, &mut count) };
if status != 0 {
return None;
}
let bytes = if info.resident_size_max > 0 {
info.resident_size_max
} else {
info.resident_size
};
(bytes > 0).then_some(bytes as usize)
}
#[cfg(target_os = "windows")]
fn windows_peak_working_set_bytes() -> Option<usize> {
#[repr(C)]
struct ProcessMemoryCounters {
cb: u32,
page_fault_count: u32,
peak_working_set_size: usize,
working_set_size: usize,
quota_peak_paged_pool_usage: usize,
quota_paged_pool_usage: usize,
quota_peak_non_paged_pool_usage: usize,
quota_non_paged_pool_usage: usize,
pagefile_usage: usize,
peak_pagefile_usage: usize,
}
extern "system" {
fn GetCurrentProcess() -> *mut core::ffi::c_void;
fn GetProcessMemoryInfo(
process: *mut core::ffi::c_void,
counters: *mut ProcessMemoryCounters,
size: u32,
) -> i32;
}
let mut counters = ProcessMemoryCounters {
cb: std::mem::size_of::<ProcessMemoryCounters>() as u32,
page_fault_count: 0,
peak_working_set_size: 0,
working_set_size: 0,
quota_peak_paged_pool_usage: 0,
quota_paged_pool_usage: 0,
quota_peak_non_paged_pool_usage: 0,
quota_non_paged_pool_usage: 0,
pagefile_usage: 0,
peak_pagefile_usage: 0,
};
let ok = unsafe {
GetProcessMemoryInfo(
GetCurrentProcess(),
&mut counters,
counters.cb,
)
};
if ok == 0 || counters.peak_working_set_size == 0 {
None
} else {
Some(counters.peak_working_set_size)
}
}
pub fn format_memory_remediation(estimate: &MemoryEstimate, budget_bytes: usize) -> String {
let mut lines = vec![
"remediation: reduce peak training memory with one or more of:".into(),
" CLI: --fit-memory [<budget>] (auto-tune until the plan fits)".into(),
" CLI: --activation-checkpointing".into(),
" CLI: --gradient-accumulation-steps 8 (then lower --batch-size)".into(),
" CLI: --batch-size 1 --sequence-length 64 --prefetch-batches 1".into(),
" CLI: --mixed-precision --amp-format bf16".into(),
" CLI: --layer-offload --offload-moments --trim-caches --batch-size 1".into(),
" CLI: --hidden-size 64 --num-layers 2 --intermediate-size 128".into(),
" JSON: trainer.activation_checkpointing=true, trainer.gradient_accumulation_steps>=8,".into(),
" trainer.batch_size=1, trainer.sequence_length (short), data.prefetch_batches=1,".into(),
" trainer.mixed_precision=true, trainer.amp_format=\"bf16\"".into(),
" trainer.layer_offload=true, optimizer.offload_moments=true".into(),
" recipe: --config training/low-ram-8gb.json or --config training/cpu-laptop.json".into(),
];
if estimate.total_bytes > budget_bytes && budget_bytes > 0 {
lines.push(format!(
" tip: plan needs ~{}; try --memory-budget {} or a smaller recipe",
format_bytes(estimate.total_bytes),
format_bytes(budget_bytes)
));
}
lines.join("\n")
}
pub fn count_parameters(config: &CausalLmConfig) -> usize {
let embeddings = config.vocab_size.saturating_mul(config.hidden_size);
let layers = config
.num_layers
.saturating_mul(count_transformer_block_parameters(config));
let final_norm = config.hidden_size;
let lm_head = if config.tie_word_embeddings {
0
} else {
let weights = config.hidden_size.saturating_mul(config.vocab_size);
if config.use_bias {
weights.saturating_add(config.vocab_size)
} else {
weights
}
};
embeddings
.saturating_add(layers)
.saturating_add(final_norm)
.saturating_add(lm_head)
}
pub fn count_transformer_block_parameters(config: &CausalLmConfig) -> usize {
let d = config.hidden_size;
let i = config.intermediate_size;
let key_value_heads = effective_key_value_heads(config);
let key_value_width = d
.saturating_div(config.num_heads.max(1))
.saturating_mul(key_value_heads);
let per_layer = d .saturating_add(d.saturating_mul(d).saturating_mul(2)) .saturating_add(d.saturating_mul(key_value_width).saturating_mul(2)) .saturating_add(d) .saturating_add(d.saturating_mul(i).saturating_mul(2)) .saturating_add(i.saturating_mul(d)); let per_layer_bias = if config.use_bias {
d.saturating_mul(3) .saturating_add(key_value_width.saturating_mul(2)) .saturating_add(i.saturating_mul(2)) } else {
0
};
per_layer.saturating_add(per_layer_bias)
}
pub fn estimate_gpu_parameter_residency_bytes(
config: &CausalLmConfig,
layer_offload: bool,
) -> usize {
let full = count_parameters(config).saturating_mul(BYTES_PER_F32);
if !layer_offload {
return full;
}
let block = count_transformer_block_parameters(config).saturating_mul(BYTES_PER_F32);
let embeddings = config
.vocab_size
.saturating_mul(config.hidden_size)
.saturating_mul(BYTES_PER_F32);
let lm_head = if config.tie_word_embeddings {
0
} else {
embeddings
};
block.max(embeddings).max(lm_head)
}
pub fn apply_gpu_parameter_residency(
estimate: &mut MemoryEstimate,
config: &CausalLmConfig,
layer_offload: bool,
) {
let gpu_extra = estimate_gpu_parameter_residency_bytes(config, layer_offload);
estimate.parameter_bytes = estimate.parameter_bytes.saturating_add(gpu_extra);
if layer_offload {
estimate.notes.push(format!(
"layer_offload: host params + ~{} peak GPU layer copy",
format_bytes(gpu_extra)
));
} else {
estimate.notes.push(
"GPU residency: counting CPU + GPU parameter copies (enable --layer-offload to reduce)"
.into(),
);
}
estimate.total_bytes = estimate
.parameter_bytes
.saturating_add(estimate.amp_master_bytes)
.saturating_add(estimate.activation_bytes)
.saturating_add(estimate.cache_bytes)
.saturating_add(estimate.staging_bytes)
.saturating_add(estimate.optimizer_bytes);
}
pub fn estimate_memory(
config: &CausalLmConfig,
batch_size: usize,
sequence_length: usize,
mode: MemoryMode,
) -> MemoryEstimate {
let batch_size = batch_size.max(1);
let sequence_length = sequence_length.min(config.max_sequence_length).max(1);
let parameter_bytes = count_parameters(config).saturating_mul(BYTES_PER_F32);
let activation_bytes = estimate_activation_bytes(config, batch_size, sequence_length, mode);
let cache_bytes = estimate_kv_cache_bytes(config, batch_size, sequence_length);
let gradient_bytes = match mode {
MemoryMode::Training => parameter_bytes,
MemoryMode::Inference => 0,
};
let moment_bytes = match mode {
MemoryMode::Training => parameter_bytes.saturating_mul(2),
MemoryMode::Inference => 0,
};
let optimizer_bytes = gradient_bytes.saturating_add(moment_bytes);
let largest_tensor_bytes =
estimate_largest_tensor_bytes(config, batch_size, sequence_length, parameter_bytes);
let staging_bytes = match mode {
MemoryMode::Training | MemoryMode::Inference => largest_tensor_bytes,
};
let total_bytes = parameter_bytes
.saturating_add(activation_bytes)
.saturating_add(cache_bytes)
.saturating_add(staging_bytes)
.saturating_add(optimizer_bytes);
MemoryEstimate {
parameter_bytes,
amp_master_bytes: 0,
activation_bytes,
cache_bytes,
staging_bytes,
optimizer_bytes,
gradient_bytes,
total_bytes,
largest_tensor_bytes,
batch_size,
sequence_length,
mode,
prefetch_batches: 0,
data_parallel_workers: 1,
activation_checkpoint_interval: 1,
optimizer_moment_bytes: moment_bytes,
notes: Vec::new(),
}
}
pub fn estimate_training_memory(
config: &CausalLmConfig,
trainer: &TrainerConfig,
) -> MemoryEstimate {
let mut estimate = estimate_memory(
config,
trainer.batch_size,
trainer.sequence_length,
MemoryMode::Training,
);
estimate.prefetch_batches = 0;
estimate.data_parallel_workers = trainer.data_parallel_workers.max(1);
estimate.activation_checkpoint_interval = trainer
.low_memory
.activation_checkpoint_interval
.max(1);
if trainer.activation_checkpointing {
let checkpointed = estimate_checkpointed_activation_bytes_with_interval(
config,
trainer.batch_size.max(1),
trainer
.sequence_length
.min(config.max_sequence_length)
.max(1),
estimate.activation_checkpoint_interval,
trainer.low_memory.selective_checkpoint,
trainer.low_memory.attention_chunk_size,
);
let saved = estimate.activation_bytes.saturating_sub(checkpointed);
estimate.activation_bytes = checkpointed;
estimate.total_bytes = estimate.total_bytes.saturating_sub(saved);
estimate.notes.push(format!(
"activation_checkpointing interval={} selective={}",
estimate.activation_checkpoint_interval,
trainer.low_memory.selective_checkpoint.as_str()
));
}
let parameter_count = count_parameters(config);
let base_moment_bytes = parameter_count.saturating_mul(BYTES_PER_F32).saturating_mul(2);
let mut moment_bytes = base_moment_bytes;
if trainer.low_memory.lora.is_some() {
let lora = trainer.low_memory.lora.as_ref().unwrap();
let lora_params = crate::peft::estimate_lora_parameter_count(config, lora);
moment_bytes = lora_params.saturating_mul(BYTES_PER_F32).saturating_mul(2);
estimate.gradient_bytes = lora_params.saturating_mul(BYTES_PER_F32);
estimate.notes.push(format!(
"LoRA rank={} trainable_params≈{}",
lora.rank, lora_params
));
}
estimate.optimizer_moment_bytes = moment_bytes;
estimate.optimizer_bytes = estimate.gradient_bytes.saturating_add(moment_bytes);
if trainer.mixed_precision.enabled {
estimate.amp_master_bytes = parameter_count.saturating_mul(BYTES_PER_F32);
estimate.notes.push(format!(
"AMP {}: working params in parameter_bytes; masters counted in amp_master_bytes",
trainer.mixed_precision.format.as_str()
));
}
if trainer.data_parallel_workers > 1 {
let workers = trainer.data_parallel_workers;
let shard_batch = (trainer.batch_size / workers).max(1);
let sequence_length = trainer
.sequence_length
.min(config.max_sequence_length)
.max(1);
let mut shard_activations =
estimate_activation_bytes(config, shard_batch, sequence_length, MemoryMode::Training);
if trainer.activation_checkpointing {
shard_activations = estimate_checkpointed_activation_bytes_with_interval(
config,
shard_batch,
sequence_length,
estimate.activation_checkpoint_interval,
trainer.low_memory.selective_checkpoint,
trainer.low_memory.attention_chunk_size,
);
}
estimate.activation_bytes = shard_activations.saturating_mul(workers);
let replica_params = parameter_count
.saturating_mul(BYTES_PER_F32)
.saturating_mul(workers.saturating_sub(1));
estimate.parameter_bytes = estimate.parameter_bytes.saturating_add(replica_params);
estimate.notes.push(format!("data_parallel_workers={workers}"));
}
if trainer.low_memory.layer_offload {
estimate.notes.push(
"layer_offload enabled (host holds params; GPU keeps one active layer)".into(),
);
}
if let Some(qat) = &trainer.low_memory.qat {
estimate.notes.push(format!(
"QAT {}: fake-quant STE on Linear weights; FP32/BF16 masters still resident{}",
qat.as_str(),
if qat.learnable_scales {
" (learnable scales)"
} else {
""
}
));
}
if let Some(lora) = &trainer.low_memory.lora {
if lora.is_qlora() {
estimate.notes.push(format!(
"QLoRA base={} frozen quantized; adapters train in higher precision",
lora.base_quant.unwrap().as_str()
));
}
}
estimate.total_bytes = estimate
.parameter_bytes
.saturating_add(estimate.amp_master_bytes)
.saturating_add(estimate.activation_bytes)
.saturating_add(estimate.cache_bytes)
.saturating_add(estimate.staging_bytes)
.saturating_add(estimate.optimizer_bytes);
estimate
}
pub fn estimate_training_memory_detailed(
config: &CausalLmConfig,
trainer: &TrainerConfig,
optimizer: &crate::optimizer::AdamWConfig,
prefetch_batches: usize,
) -> MemoryEstimate {
let mut estimate = estimate_training_memory(config, trainer);
estimate.prefetch_batches = prefetch_batches;
let parameter_count = if let Some(lora) = &trainer.low_memory.lora {
crate::peft::estimate_lora_parameter_count(config, lora)
} else {
count_parameters(config)
};
let moment_bytes = match optimizer.kind {
crate::optimizer::OptimizerKind::AdamW => {
let bytes_per = match optimizer.moment_dtype {
crate::optimizer::MomentDtype::Fp32 => BYTES_PER_F32,
crate::optimizer::MomentDtype::Fp16 => 2,
};
parameter_count.saturating_mul(bytes_per).saturating_mul(2)
}
crate::optimizer::OptimizerKind::Lion => {
let bytes_per = match optimizer.moment_dtype {
crate::optimizer::MomentDtype::Fp32 => BYTES_PER_F32,
crate::optimizer::MomentDtype::Fp16 => 2,
};
parameter_count.saturating_mul(bytes_per)
}
crate::optimizer::OptimizerKind::Sgd => {
if optimizer.momentum > 0.0 {
parameter_count.saturating_mul(BYTES_PER_F32)
} else {
0
}
}
crate::optimizer::OptimizerKind::Adafactor => {
let d = config.hidden_size.max(1);
d.saturating_mul(2)
.saturating_mul(config.num_layers.max(1).saturating_add(2))
.saturating_mul(BYTES_PER_F32)
.max(parameter_count.saturating_mul(BYTES_PER_F32) / 16)
}
};
let live_moment_bytes = if optimizer.offload_moments { 0 } else { moment_bytes };
estimate.optimizer_moment_bytes = live_moment_bytes;
estimate.gradient_bytes = parameter_count.saturating_mul(BYTES_PER_F32);
estimate.optimizer_bytes = estimate
.gradient_bytes
.saturating_add(live_moment_bytes);
estimate.notes.push(format!(
"optimizer kind={:?} moment_dtype={:?} offload_moments={}",
optimizer.kind, optimizer.moment_dtype, optimizer.offload_moments
));
if prefetch_batches > 0 {
let packed = trainer
.batch_size
.saturating_mul(trainer.sequence_length)
.saturating_mul(std::mem::size_of::<u32>() * 2)
.saturating_mul(prefetch_batches);
estimate.staging_bytes = estimate.staging_bytes.saturating_add(packed);
estimate.notes.push(format!(
"prefetch_batches={prefetch_batches} holds ~{} packed batch bytes",
format_bytes(packed)
));
} else {
estimate.notes.push("prefetch_batches=0 (sync loader)".into());
}
estimate.total_bytes = estimate
.parameter_bytes
.saturating_add(estimate.amp_master_bytes)
.saturating_add(estimate.activation_bytes)
.saturating_add(estimate.cache_bytes)
.saturating_add(estimate.staging_bytes)
.saturating_add(estimate.optimizer_bytes);
estimate
}
fn estimate_checkpointed_activation_bytes_with_interval(
config: &CausalLmConfig,
batch_size: usize,
sequence_length: usize,
interval: usize,
selective: crate::trainer::SelectiveCheckpoint,
attention_chunk_size: usize,
) -> usize {
let b = batch_size;
let s = sequence_length;
let d = config.hidden_size;
let h = config.num_heads;
let key_value_width = d
.saturating_div(h.max(1))
.saturating_mul(effective_key_value_heads(config));
let i = config.intermediate_size;
let v = config.vocab_size;
let tokens = b.saturating_mul(s);
let residual = tokens.saturating_mul(d);
let attn_seq = if attention_chunk_size == 0 {
s
} else {
attention_chunk_size.min(s)
};
let attn_tokens = b.saturating_mul(attn_seq);
let qkv = residual.saturating_add(attn_tokens.saturating_mul(key_value_width).saturating_mul(2));
let attention_rows = b.saturating_mul(h).saturating_mul(attn_seq);
let ffn = tokens.saturating_mul(i).saturating_mul(3);
let attention_working = residual
.saturating_add(qkv)
.saturating_add(attention_rows)
.saturating_add(residual);
let ffn_working = residual.saturating_add(ffn).saturating_add(residual);
let layer_working = match selective {
crate::trainer::SelectiveCheckpoint::Full => attention_working
.saturating_add(ffn_working)
.saturating_sub(residual),
crate::trainer::SelectiveCheckpoint::Attention => {
attention_working.saturating_add(ffn)
}
crate::trainer::SelectiveCheckpoint::Ffn => {
ffn_working.saturating_add(qkv).saturating_add(attention_rows)
}
};
let logits = tokens.saturating_mul(v);
let embedding_dropout =
usize::from(config.embedding_dropout > 0.0).saturating_mul(residual);
let residual_dropout = usize::from(config.residual_dropout > 0.0)
.saturating_mul(2)
.saturating_mul(residual);
let retained_blocks = interval.max(1).min(config.num_layers.max(1));
let elements = residual
.saturating_add(retained_blocks.saturating_mul(residual))
.saturating_add(layer_working)
.saturating_add(embedding_dropout)
.saturating_add(residual_dropout)
.saturating_add(logits);
elements.saturating_mul(BYTES_PER_F32)
}
pub fn estimate_inference_memory(
config: &CausalLmConfig,
batch_size: usize,
sequence_length: usize,
) -> MemoryEstimate {
estimate_memory(config, batch_size, sequence_length, MemoryMode::Inference)
}
pub fn check_memory_budget(
estimate: &MemoryEstimate,
budget: &DeviceBudget,
) -> Result<(), MemoryBudgetError> {
if estimate.largest_tensor_bytes > budget.max_buffer_bytes {
return Err(MemoryBudgetError::ExceedsMaxBuffer {
label: budget.label.clone(),
required_bytes: estimate.largest_tensor_bytes,
max_buffer_bytes: budget.max_buffer_bytes,
estimate: estimate.clone(),
});
}
let obvious_limit = budget
.total_bytes
.saturating_mul(OBVIOUS_OVERFLOW_NUMERATOR)
/ OBVIOUS_OVERFLOW_DENOMINATOR;
if estimate.total_bytes > obvious_limit {
return Err(MemoryBudgetError::ExceedsDeviceBudget {
label: budget.label.clone(),
estimated_bytes: estimate.total_bytes,
budget_bytes: budget.total_bytes,
estimate: estimate.clone(),
});
}
Ok(())
}
pub fn format_bytes(bytes: usize) -> String {
if bytes == usize::MAX {
return "unlimited".into();
}
const KIB: f64 = 1024.0;
const MIB: f64 = KIB * 1024.0;
const GIB: f64 = MIB * 1024.0;
let bytes_f = bytes as f64;
if bytes_f >= GIB {
format!("{bytes} ({:.2} GiB)", bytes_f / GIB)
} else if bytes_f >= MIB {
format!("{bytes} ({:.2} MiB)", bytes_f / MIB)
} else if bytes_f >= KIB {
format!("{bytes} ({:.2} KiB)", bytes_f / KIB)
} else {
format!("{bytes} B")
}
}
pub fn format_bytes_compact(bytes: usize) -> String {
if bytes == usize::MAX {
return "unlimited".into();
}
const KIB: f64 = 1024.0;
const MIB: f64 = KIB * 1024.0;
const GIB: f64 = MIB * 1024.0;
let bytes_f = bytes as f64;
if bytes_f >= GIB {
format!("{:.2}GiB", bytes_f / GIB)
} else if bytes_f >= MIB {
format!("{:.2}MiB", bytes_f / MIB)
} else if bytes_f >= KIB {
format!("{:.2}KiB", bytes_f / KIB)
} else {
format!("{bytes}B")
}
}
pub fn format_eta(seconds: f64) -> String {
if !seconds.is_finite() || seconds <= 0.0 {
return "0s".into();
}
let total = seconds.round() as u64;
let hours = total / 3600;
let minutes = (total % 3600) / 60;
let secs = total % 60;
if hours > 0 {
format!("{hours}h{minutes:02}m")
} else if minutes > 0 {
format!("{minutes}m{secs:02}s")
} else {
format!("{secs}s")
}
}
pub fn format_memory_report(estimate: &MemoryEstimate, budget: Option<&DeviceBudget>) -> String {
let mode = match estimate.mode {
MemoryMode::Training => "training",
MemoryMode::Inference => "inference",
};
let mut lines = vec![
format!(
"memory mode={} batch={} sequence_length={}",
mode, estimate.batch_size, estimate.sequence_length
),
format!(
"memory parameters={} amp_masters={} activations={} cache={} staging={} optimizer={}",
format_bytes(estimate.parameter_bytes),
format_bytes(estimate.amp_master_bytes),
format_bytes(estimate.activation_bytes),
format_bytes(estimate.cache_bytes),
format_bytes(estimate.staging_bytes),
format_bytes(estimate.optimizer_bytes)
),
format!(
"memory optimizer_moments={} grads={} checkpoint_interval={} prefetch={} dp_workers={}",
format_bytes(estimate.optimizer_moment_bytes),
format_bytes(estimate.gradient_bytes),
estimate.activation_checkpoint_interval,
estimate.prefetch_batches,
estimate.data_parallel_workers
),
format!(
"memory total={} largest_tensor={}",
format_bytes(estimate.total_bytes),
format_bytes(estimate.largest_tensor_bytes)
),
];
for note in &estimate.notes {
lines.push(format!("memory note: {note}"));
}
if let Some(budget) = budget {
lines.push(format!(
"memory budget device={} total={} max_buffer={}",
budget.label,
format_bytes(budget.total_bytes),
format_bytes(budget.max_buffer_bytes)
));
}
lines.join("\n")
}
fn summarize_estimate(estimate: &MemoryEstimate) -> String {
format!(
"parameters={} activations={} cache={} staging={} optimizer={} total={}",
format_bytes(estimate.parameter_bytes),
format_bytes(estimate.activation_bytes),
format_bytes(estimate.cache_bytes),
format_bytes(estimate.staging_bytes),
format_bytes(estimate.optimizer_bytes),
format_bytes(estimate.total_bytes)
)
}
pub fn estimate_kv_cache_bytes(
config: &CausalLmConfig,
batch_size: usize,
sequence_length: usize,
) -> usize {
let head_dim = config.hidden_size / config.num_heads.max(1);
config
.num_layers
.saturating_mul(2)
.saturating_mul(batch_size)
.saturating_mul(sequence_length)
.saturating_mul(effective_key_value_heads(config))
.saturating_mul(head_dim)
.saturating_mul(BYTES_PER_F32)
}
fn estimate_activation_bytes(
config: &CausalLmConfig,
batch_size: usize,
sequence_length: usize,
mode: MemoryMode,
) -> usize {
let b = batch_size;
let s = sequence_length;
let d = config.hidden_size;
let h = config.num_heads;
let key_value_width = d
.saturating_div(h.max(1))
.saturating_mul(effective_key_value_heads(config));
let i = config.intermediate_size;
let v = config.vocab_size;
let tokens = b.saturating_mul(s);
let residual = tokens.saturating_mul(d);
let qkv = residual.saturating_add(tokens.saturating_mul(key_value_width).saturating_mul(2));
let attention_rows = b.saturating_mul(h).saturating_mul(s); let ffn = tokens.saturating_mul(i).saturating_mul(3); let layer_working = residual .saturating_add(qkv)
.saturating_add(attention_rows)
.saturating_add(ffn)
.saturating_add(residual); let logits = tokens.saturating_mul(v);
let elements = match mode {
MemoryMode::Training => {
let embedding_dropout =
usize::from(config.embedding_dropout > 0.0).saturating_mul(residual);
let residual_dropout = usize::from(config.residual_dropout > 0.0)
.saturating_mul(config.num_layers)
.saturating_mul(2)
.saturating_mul(residual);
residual
.saturating_add(config.num_layers.saturating_mul(layer_working))
.saturating_add(embedding_dropout)
.saturating_add(residual_dropout)
.saturating_add(logits)
}
MemoryMode::Inference => residual
.saturating_add(layer_working)
.saturating_add(logits),
};
elements.saturating_mul(BYTES_PER_F32)
}
fn effective_key_value_heads(config: &CausalLmConfig) -> usize {
if config.num_key_value_heads == 0 {
config.num_heads
} else {
config.num_key_value_heads
}
}
fn estimate_largest_tensor_bytes(
config: &CausalLmConfig,
batch_size: usize,
sequence_length: usize,
_parameter_bytes: usize,
) -> usize {
let b = batch_size;
let s = sequence_length;
let d = config.hidden_size;
let h = config.num_heads;
let i = config.intermediate_size;
let v = config.vocab_size;
let candidates = [
b.saturating_mul(s)
.saturating_mul(v)
.saturating_mul(BYTES_PER_F32), b.saturating_mul(h)
.saturating_mul(s)
.saturating_mul(BYTES_PER_F32), b.saturating_mul(s)
.saturating_mul(i)
.saturating_mul(BYTES_PER_F32), b.saturating_mul(s)
.saturating_mul(d)
.saturating_mul(BYTES_PER_F32), d.saturating_mul(d).saturating_mul(BYTES_PER_F32), d.saturating_mul(i).saturating_mul(BYTES_PER_F32), d.saturating_mul(v).saturating_mul(BYTES_PER_F32), ];
candidates.into_iter().max().unwrap_or(BYTES_PER_F32)
}
#[cfg(test)]
mod tests {
use super::{
apply_gpu_parameter_residency, check_memory_budget, count_parameters, estimate_memory,
estimate_gpu_parameter_residency_bytes, estimate_training_memory, format_bytes,
format_memory_report, DeviceBudget, MemoryBudgetError, MemoryMode, BYTES_PER_F32,
};
use crate::model::{CausalLM, CausalLmConfig};
use crate::trainer::TrainerConfig;
fn tiny_config(tie_word_embeddings: bool) -> CausalLmConfig {
CausalLmConfig {
vocab_size: 16,
hidden_size: 8,
num_layers: 2,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 8,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: 15,
seed: 11,
initializer_std: 0.02,
tie_word_embeddings,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
}
}
#[test]
fn parameter_count_matches_constructed_model() {
for tie in [true, false] {
for use_bias in [false, true] {
let mut config = tiny_config(tie);
config.use_bias = use_bias;
let model = CausalLM::new(config.clone()).unwrap();
let expected: usize = model.parameters().iter().map(|p| p.numel()).sum();
assert_eq!(count_parameters(&config), expected);
}
}
}
#[test]
fn training_estimate_reports_five_categories() {
let config = tiny_config(true);
let trainer = TrainerConfig {
batch_size: 2,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
let memory = estimate_training_memory(&config, &trainer);
assert_eq!(memory.mode, MemoryMode::Training);
assert_eq!(memory.batch_size, 2);
assert_eq!(memory.sequence_length, 4);
assert_eq!(
memory.parameter_bytes,
count_parameters(&config) * BYTES_PER_F32
);
assert!(memory.activation_bytes > 0);
assert_eq!(
memory.cache_bytes,
2 * 2 * 2 * 4 * 4 * BYTES_PER_F32 );
assert!(memory.staging_bytes > 0);
assert_eq!(
memory.optimizer_bytes,
memory.parameter_bytes * 3 );
assert_eq!(
memory.total_bytes,
memory
.parameter_bytes
.saturating_add(memory.activation_bytes)
.saturating_add(memory.cache_bytes)
.saturating_add(memory.staging_bytes)
.saturating_add(memory.optimizer_bytes)
);
let report = format_memory_report(&memory, None);
assert!(report.contains("parameters="));
assert!(report.contains("activations="));
assert!(report.contains("cache="));
assert!(report.contains("staging="));
assert!(report.contains("optimizer="));
}
#[test]
fn inference_omits_optimizer_state() {
let memory = estimate_memory(&tiny_config(true), 1, 8, MemoryMode::Inference);
assert_eq!(memory.optimizer_bytes, 0);
assert_eq!(memory.gradient_bytes, 0);
assert!(memory.activation_bytes > 0);
assert!(memory.cache_bytes > 0);
}
#[test]
fn refuses_obvious_device_budget_overflow() {
let memory = estimate_memory(&tiny_config(true), 1, 4, MemoryMode::Training);
let budget = DeviceBudget {
label: "test".into(),
total_bytes: memory.total_bytes / 2,
max_buffer_bytes: usize::MAX,
};
let error = check_memory_budget(&memory, &budget).unwrap_err();
match error {
MemoryBudgetError::ExceedsDeviceBudget {
estimated_bytes,
budget_bytes,
..
} => {
assert_eq!(estimated_bytes, memory.total_bytes);
assert_eq!(budget_bytes, budget.total_bytes);
}
other => panic!("unexpected error: {other}"),
}
}
#[test]
fn refuses_tensor_larger_than_max_buffer() {
let memory = estimate_memory(&tiny_config(true), 1, 4, MemoryMode::Inference);
let budget = DeviceBudget {
label: "gpu".into(),
total_bytes: usize::MAX,
max_buffer_bytes: memory.largest_tensor_bytes / 2,
};
let error = check_memory_budget(&memory, &budget).unwrap_err();
assert!(matches!(error, MemoryBudgetError::ExceedsMaxBuffer { .. }));
}
#[test]
fn accepts_configuration_within_budget() {
let memory = estimate_memory(&tiny_config(true), 1, 4, MemoryMode::Training);
let budget = DeviceBudget::cpu(Some(memory.total_bytes.saturating_mul(2)));
check_memory_budget(&memory, &budget).unwrap();
}
#[test]
fn format_bytes_uses_binary_units() {
assert_eq!(format_bytes(512), "512 B");
assert!(format_bytes(2048).contains("KiB"));
assert!(format_bytes(2 * 1024 * 1024).contains("MiB"));
}
#[test]
fn format_bytes_compact_and_eta() {
assert_eq!(super::format_bytes_compact(512), "512B");
assert_eq!(super::format_bytes_compact(2 * 1024 * 1024), "2.00MiB");
assert_eq!(super::format_eta(0.0), "0s");
assert_eq!(super::format_eta(45.2), "45s");
assert_eq!(super::format_eta(192.0), "3m12s");
assert_eq!(super::format_eta(3725.0), "1h02m");
}
#[test]
fn process_peak_rss_is_available_on_supported_hosts() {
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
{
let rss = super::process_peak_rss_bytes().expect("peak RSS should be readable");
assert!(rss > 0);
}
}
#[test]
fn remediation_mentions_fit_memory_and_recipes() {
let memory = estimate_memory(&tiny_config(true), 1, 4, MemoryMode::Training);
let text = super::format_memory_remediation(&memory, 1024);
assert!(text.contains("--fit-memory"));
assert!(text.contains("low-ram-8gb.json"));
assert!(text.contains("activation_checkpointing"));
assert!(text.contains("--layer-offload"));
}
#[test]
fn default_cpu_budget_is_positive() {
assert!(super::default_cpu_budget_bytes() > 0);
}
#[test]
fn gpu_residency_prefers_layer_offload_over_full_mirror() {
let config = tiny_config(true);
let full_mirror = estimate_gpu_parameter_residency_bytes(&config, false);
let offload = estimate_gpu_parameter_residency_bytes(&config, true);
assert_eq!(full_mirror, count_parameters(&config) * BYTES_PER_F32);
assert!(offload < full_mirror);
let trainer = TrainerConfig {
batch_size: 2,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
let mut mirrored = estimate_training_memory(&config, &trainer);
let baseline_total = mirrored.total_bytes;
apply_gpu_parameter_residency(&mut mirrored, &config, false);
let mut offloaded = estimate_training_memory(&config, &trainer);
apply_gpu_parameter_residency(&mut offloaded, &config, true);
assert!(mirrored.total_bytes > baseline_total);
assert!(offloaded.total_bytes < mirrored.total_bytes);
assert!(offloaded.total_bytes > baseline_total);
}
}