use crate::memory::{estimate_training_memory, format_bytes, format_memory_report, MemoryEstimate};
use crate::model::{CausalLM, CausalLmConfig};
use crate::optimizer::AdamWConfig;
use crate::tokenizer::{BpeTokenizer, SpecialTokens, BYTE_VOCAB_SIZE, FIRST_MERGE_ID};
use crate::trainer::{
load_checkpoint, LearningRateSchedule, TrainerConfig, TrainerError,
};
use crate::Tensor;
use std::fmt::Write as _;
use std::path::{Path, PathBuf};
pub const TOKENIZER_SAMPLE: &str = "The quick brown fox jumps over the lazy dog. 0123456789";
#[derive(Clone, Debug)]
pub struct ParameterInfo {
pub name: String,
pub shape: Vec<usize>,
pub numel: usize,
pub dtype: &'static str,
}
#[derive(Clone, Debug)]
pub struct TokenizerStats {
pub vocab_size: usize,
pub merge_count: usize,
pub special_tokens: SpecialTokens,
pub byte_token_count: usize,
pub special_token_count: usize,
pub merge_token_count: usize,
pub min_token_bytes: usize,
pub max_token_bytes: usize,
pub average_token_bytes: f64,
pub sample_characters: usize,
pub sample_bytes: usize,
pub sample_tokens: usize,
pub sample_tokens_per_character: f64,
pub sample_bytes_per_token: f64,
}
#[derive(Clone, Debug)]
pub struct CheckpointReport {
pub path: PathBuf,
pub file_bytes: u64,
pub version: u32,
pub model_config: CausalLmConfig,
pub parameters: Vec<ParameterInfo>,
pub total_parameters: usize,
pub tie_word_embeddings: bool,
pub memory: MemoryEstimate,
pub tokenizer: TokenizerStats,
pub optimizer_config: AdamWConfig,
pub optimizer_step: usize,
pub schedule: LearningRateSchedule,
pub trainer_config: TrainerConfig,
pub training_step: usize,
pub cursor: usize,
pub tokens_processed: usize,
pub best_validation_loss: Option<f32>,
pub rng_model_seed: u64,
pub rng_dropout_stream: usize,
pub data_loader_seed: u64,
pub data_loader_batch_size: usize,
pub data_loader_sequence_length: usize,
}
pub fn named_parameters(model: &CausalLM) -> Vec<(String, Tensor)> {
model.named_parameters()
}
pub fn parameter_infos(model: &CausalLM) -> Vec<ParameterInfo> {
named_parameters(model)
.into_iter()
.map(|(name, tensor)| ParameterInfo {
name,
shape: tensor.shape(),
numel: tensor.numel(),
dtype: "F32",
})
.collect()
}
pub fn estimate_memory(
_total_parameters: usize,
config: &CausalLmConfig,
trainer_config: &TrainerConfig,
) -> MemoryEstimate {
estimate_training_memory(config, trainer_config)
}
pub fn tokenizer_stats(tokenizer: &BpeTokenizer) -> TokenizerStats {
let mut min_token_bytes = usize::MAX;
let mut max_token_bytes = 0usize;
let mut total_token_bytes = 0usize;
let mut byte_token_count = 0usize;
let mut special_token_count = 0usize;
let mut merge_token_count = 0usize;
for (token_id, bytes) in tokenizer.vocab.iter().enumerate() {
let length = bytes.len();
min_token_bytes = min_token_bytes.min(length);
max_token_bytes = max_token_bytes.max(length);
total_token_bytes = total_token_bytes.saturating_add(length);
if token_id < BYTE_VOCAB_SIZE {
byte_token_count += 1;
} else if token_id < FIRST_MERGE_ID {
special_token_count += 1;
} else {
merge_token_count += 1;
}
}
if tokenizer.vocab.is_empty() {
min_token_bytes = 0;
}
let vocab_size = tokenizer.vocab_size();
let average_token_bytes = if vocab_size == 0 {
0.0
} else {
total_token_bytes as f64 / vocab_size as f64
};
let sample_tokens = tokenizer.encode(TOKENIZER_SAMPLE);
let sample_characters = TOKENIZER_SAMPLE.chars().count();
let sample_bytes = TOKENIZER_SAMPLE.len();
let sample_token_count = sample_tokens.len();
let sample_tokens_per_character = if sample_characters == 0 {
0.0
} else {
sample_token_count as f64 / sample_characters as f64
};
let sample_bytes_per_token = if sample_token_count == 0 {
0.0
} else {
sample_bytes as f64 / sample_token_count as f64
};
TokenizerStats {
vocab_size,
merge_count: tokenizer.merges.len(),
special_tokens: tokenizer.special_tokens.clone(),
byte_token_count,
special_token_count,
merge_token_count,
min_token_bytes,
max_token_bytes,
average_token_bytes,
sample_characters,
sample_bytes,
sample_tokens: sample_token_count,
sample_tokens_per_character,
sample_bytes_per_token,
}
}
pub fn inspect_checkpoint(path: impl AsRef<Path>) -> Result<CheckpointReport, TrainerError> {
let path = path.as_ref();
let file_bytes = std::fs::metadata(path)?.len();
let checkpoint = load_checkpoint(path)?;
let parameters = parameter_infos(&checkpoint.model);
let total_parameters = parameters.iter().map(|parameter| parameter.numel).sum();
let memory = estimate_training_memory(&checkpoint.model.config, &checkpoint.trainer_config);
let tokenizer = tokenizer_stats(&checkpoint.tokenizer);
Ok(CheckpointReport {
path: path.to_path_buf(),
file_bytes,
version: checkpoint.version,
model_config: checkpoint.model.config.clone(),
parameters,
total_parameters,
tie_word_embeddings: checkpoint.model.config.tie_word_embeddings,
memory,
tokenizer,
optimizer_config: checkpoint.optimizer.config().clone(),
optimizer_step: checkpoint.optimizer.state().step,
schedule: checkpoint.schedule,
trainer_config: checkpoint.trainer_config,
training_step: checkpoint.training_step,
cursor: checkpoint.cursor,
tokens_processed: checkpoint.tokens_processed,
best_validation_loss: checkpoint.best_validation_loss,
rng_model_seed: checkpoint.rng.model_seed,
rng_dropout_stream: checkpoint.rng.dropout_stream,
data_loader_seed: checkpoint.data_loader.seed,
data_loader_batch_size: checkpoint.data_loader.batch_size,
data_loader_sequence_length: checkpoint.data_loader.sequence_length,
})
}
pub fn format_report(report: &CheckpointReport) -> String {
let mut output = String::new();
let _ = writeln!(output, "checkpoint path={}", report.path.display());
let _ = writeln!(output, "checkpoint file_bytes={}", report.file_bytes);
let _ = writeln!(output, "checkpoint version={}", report.version);
let _ = writeln!(
output,
"checkpoint training_step={} cursor={} tokens_processed={}",
report.training_step, report.cursor, report.tokens_processed
);
let _ = writeln!(
output,
"checkpoint rng model_seed={} dropout_stream={}",
report.rng_model_seed, report.rng_dropout_stream
);
let _ = writeln!(
output,
"checkpoint data_loader seed={} cursor={} batch_size={} sequence_length={}",
report.data_loader_seed,
report.cursor,
report.data_loader_batch_size,
report.data_loader_sequence_length
);
match report.best_validation_loss {
Some(loss) => {
let _ = writeln!(output, "checkpoint best_validation_loss={loss:.6}");
}
None => {
let _ = writeln!(output, "checkpoint best_validation_loss=none");
}
}
let config = &report.model_config;
let _ = writeln!(
output,
"model vocab_size={} hidden_size={} num_layers={} num_heads={} num_key_value_heads={} intermediate_size={}",
config.vocab_size,
config.hidden_size,
config.num_layers,
config.num_heads,
config.num_key_value_heads,
config.intermediate_size
);
let _ = writeln!(
output,
"model max_sequence_length={} rms_norm_epsilon={} rope_theta={} eos_token_id={} seed={} tie_word_embeddings={}",
config.max_sequence_length,
config.rms_norm_epsilon,
config.rope_theta,
config.eos_token_id,
config.seed,
config.tie_word_embeddings
);
let _ = writeln!(
output,
"model attention_dropout={} residual_dropout={} embedding_dropout={} use_bias={}",
config.attention_dropout,
config.residual_dropout,
config.embedding_dropout,
config.use_bias
);
if let Some(scaling) = &config.rope_scaling {
let _ = writeln!(
output,
"model rope_scaling={:?} factor={} original_max_position_embeddings={} attention_factor={:?} beta_fast={} beta_slow={}",
scaling.kind,
scaling.factor,
scaling.original_max_position_embeddings,
scaling.attention_factor,
scaling.beta_fast,
scaling.beta_slow
);
}
let _ = writeln!(output, "parameters total={}", report.total_parameters);
if report.tie_word_embeddings {
let _ = writeln!(
output,
"parameters note=lm_head.weight tied to tok_embeddings.weight"
);
}
for parameter in &report.parameters {
let _ = writeln!(
output,
"parameter name={} shape={} dtype={} numel={}",
parameter.name,
format_shape(¶meter.shape),
parameter.dtype,
parameter.numel
);
}
let _ = writeln!(output, "{}", format_memory_report(&report.memory, None));
let _ = writeln!(
output,
"memory gradients={} (included in optimizer)",
format_bytes(report.memory.gradient_bytes)
);
let tokenizer = &report.tokenizer;
let _ = writeln!(
output,
"tokenizer vocab_size={} merges={} byte_tokens={} special_tokens={} merge_tokens={}",
tokenizer.vocab_size,
tokenizer.merge_count,
tokenizer.byte_token_count,
tokenizer.special_token_count,
tokenizer.merge_token_count
);
let _ = writeln!(
output,
"tokenizer specials pad={} unk={} bos={} eos={}",
tokenizer.special_tokens.pad,
tokenizer.special_tokens.unk,
tokenizer.special_tokens.bos,
tokenizer.special_tokens.eos
);
let _ = writeln!(
output,
"tokenizer token_bytes min={} max={} avg={:.3}",
tokenizer.min_token_bytes, tokenizer.max_token_bytes, tokenizer.average_token_bytes
);
let _ = writeln!(
output,
"tokenizer sample_bytes={} sample_chars={} sample_tokens={} tokens_per_char={:.3} bytes_per_token={:.3}",
tokenizer.sample_bytes,
tokenizer.sample_characters,
tokenizer.sample_tokens,
tokenizer.sample_tokens_per_character,
tokenizer.sample_bytes_per_token
);
let optimizer = &report.optimizer_config;
let _ = writeln!(
output,
"optimizer step={} beta1={} beta2={} epsilon={} weight_decay={} no_decay_rms_norm={} no_decay_embeddings=[{}] rms_norm_lr_scale={} embedding_lr_scale={}",
report.optimizer_step,
optimizer.beta1,
optimizer.beta2,
optimizer.epsilon,
optimizer.weight_decay,
optimizer.no_decay_rms_norm,
optimizer.no_decay_embeddings.join(","),
optimizer.rms_norm_lr_scale,
optimizer.embedding_lr_scale
);
let schedule = &report.schedule;
let _ = writeln!(
output,
"schedule base_lr={} min_lr={} warmup_steps={} total_steps={}",
schedule.base_learning_rate,
schedule.minimum_learning_rate,
schedule.warmup_steps,
schedule.total_steps
);
let trainer = &report.trainer_config;
let _ = writeln!(
output,
"trainer batch_size={} sequence_length={} max_gradient_norm={} log_every={} gradient_accumulation_steps={} activation_checkpointing={} mixed_precision={} amp_format={} initial_loss_scale={}",
trainer.batch_size,
trainer.sequence_length,
trainer.max_gradient_norm,
trainer.log_every,
trainer.gradient_accumulation_steps,
trainer.activation_checkpointing,
trainer.mixed_precision.enabled,
trainer.mixed_precision.format.as_str(),
trainer.mixed_precision.initial_loss_scale
);
output
}
fn format_shape(shape: &[usize]) -> String {
format!(
"[{}]",
shape
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(", ")
)
}
#[cfg(test)]
mod tests {
use super::{
estimate_memory, format_report, inspect_checkpoint, named_parameters, parameter_infos,
tokenizer_stats,
};
use crate::memory::count_parameters;
use crate::model::{CausalLM, CausalLmConfig};
use crate::optimizer::{AdamW, AdamWConfig};
use crate::tokenizer::{BpeTokenizer, FIRST_MERGE_ID};
use crate::trainer::{save_checkpoint, LearningRateSchedule, TrainerConfig};
fn tiny_config(tie_word_embeddings: bool) -> CausalLmConfig {
CausalLmConfig {
vocab_size: 16,
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 8,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: 15,
seed: 11,
initializer_std: 0.02,
tie_word_embeddings,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
}
}
#[test]
fn named_parameters_match_parameter_list() {
for tie in [true, false] {
let model = CausalLM::new(tiny_config(tie)).unwrap();
let named = named_parameters(&model);
let parameters = model.parameters();
assert_eq!(named.len(), parameters.len());
let total: usize = named.iter().map(|(_, tensor)| tensor.numel()).sum();
let expected: usize = parameters.iter().map(|parameter| parameter.numel()).sum();
assert_eq!(total, expected);
for ((_, named_tensor), parameter) in named.iter().zip(parameters.iter()) {
assert_eq!(named_tensor.shape(), parameter.shape());
assert_eq!(named_tensor.numel(), parameter.numel());
}
}
}
#[test]
fn memory_estimate_uses_fp32_footprint() {
let config = tiny_config(true);
let trainer = TrainerConfig {
batch_size: 2,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
let memory = estimate_memory(100, &config, &trainer);
assert_eq!(memory.parameter_bytes, count_parameters(&config) * 4);
assert_eq!(memory.gradient_bytes, memory.parameter_bytes);
assert_eq!(memory.optimizer_bytes, memory.parameter_bytes * 3);
let head_dim = config.hidden_size / config.num_heads;
assert_eq!(
memory.cache_bytes,
2 * config.num_layers
* trainer.batch_size
* trainer.sequence_length
* config.num_key_value_heads
* head_dim
* 4
);
assert!(memory.activation_bytes > 0);
assert!(memory.staging_bytes > 0);
assert!(memory.total_bytes >= memory.parameter_bytes);
}
#[test]
fn inspect_checkpoint_reports_totals_without_training_corpus() {
let directory = std::env::temp_dir().join(format!(
"oxidelm-inspect-{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&directory).unwrap();
let path = directory.join("model.bin");
let tokenizer = BpeTokenizer::train_from_bytes_with_min_frequency(
b"hello world hello rust",
FIRST_MERGE_ID + 8,
1,
)
.unwrap();
let mut config = tiny_config(true);
config.vocab_size = tokenizer.vocab_size();
config.eos_token_id = tokenizer.special_tokens.eos;
let model = CausalLM::new(config).unwrap();
let parameters = model.parameters();
let expected_total: usize = parameters.iter().map(|parameter| parameter.numel()).sum();
let optimizer = AdamW::new(¶meters, AdamWConfig::default()).unwrap();
let schedule = LearningRateSchedule {
base_learning_rate: 3e-4,
minimum_learning_rate: 3e-5,
warmup_steps: 1,
total_steps: 10,
};
let trainer_config = TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
save_checkpoint(
&path,
&model,
&optimizer,
&tokenizer,
&schedule,
&trainer_config,
7,
3,
42,
)
.unwrap();
let report = inspect_checkpoint(&path).unwrap();
assert_eq!(report.total_parameters, expected_total);
assert_eq!(report.parameters.len(), parameter_infos(&model).len());
assert_eq!(report.training_step, 7);
assert_eq!(report.cursor, 3);
assert_eq!(report.tokens_processed, 42);
assert_eq!(report.tokenizer.vocab_size, tokenizer.vocab_size());
assert_eq!(
report.tokenizer.merge_count,
tokenizer_stats(&tokenizer).merge_count
);
assert!(report.file_bytes > 0);
let formatted = format_report(&report);
assert!(formatted.contains("parameters total="));
assert!(formatted.contains("tok_embeddings.weight"));
assert!(formatted.contains("tokenizer vocab_size="));
assert!(formatted.contains("memory parameters="));
assert!(formatted.contains("activations="));
assert!(formatted.contains("staging="));
let _ = std::fs::remove_dir_all(directory);
}
}