#[cfg(feature = "training")]
use crate::amp::{
cast_storage, parameters_overflow, parameters_overflow_gpu, scale_parameter_grads,
DynamicLossScaler, LossScalerState,
};
use crate::amp::{AmpError, MixedPrecisionConfig};
#[cfg(feature = "training")]
use crate::anomaly;
#[cfg(all(test, feature = "training"))]
use crate::data_loader::PackedSequenceSampler;
#[cfg(feature = "training")]
use crate::data_loader::{DataLoaderError, PrefetchDataLoader, TokenDataset};
#[cfg(feature = "training")]
#[cfg(feature = "training")]
use crate::data_parallel::{
accumulate_parameter_grads, capture_parameter_weights, CpuDataParallelPool, DataParallelError,
};
#[cfg(all(feature = "training", feature = "gpu"))]
use crate::data_parallel::{accumulate_parameter_gpu_grads, GpuDataParallelPool};
#[cfg(feature = "training")]
use crate::gpu::GpuContext;
use crate::gpu::GpuError;
#[cfg(feature = "training")]
use crate::log::{self, LogValue};
#[cfg(feature = "training")]
use crate::memory::{
apply_gpu_parameter_residency, check_memory_budget, estimate_training_memory, format_eta,
process_peak_rss_bytes, DeviceBudget, MemoryEstimate,
};
use crate::memory::MemoryBudgetError;
#[cfg(feature = "training")]
use crate::model::{CausalLM, CausalLmConfig};
use crate::model::{ModelConfigError, ModelError};
#[cfg(feature = "training")]
use crate::optimizer::{AdamW, AdamWConfig};
use crate::optimizer::OptimizerError;
use crate::peft::LoraConfig;
#[cfg(feature = "training")]
use crate::profile;
#[cfg(feature = "training")]
use crate::tokenizer::BpeTokenizer;
use crate::tokenizer::TokenizerError;
#[cfg(feature = "training")]
use crate::{cpu_memory_stats, IndexTensor, Tensor};
use serde::{Deserialize, Serialize};
use serde_json::{json, Map, Value};
use std::error::Error;
use std::fmt;
#[cfg(feature = "training")]
use std::fs::File;
#[cfg(feature = "training")]
use std::io::{BufWriter, Write};
#[cfg(feature = "training")]
use std::path::Path;
use std::path::PathBuf;
#[cfg(feature = "training")]
use std::sync::Arc;
#[cfg(feature = "training")]
use std::time::Instant;
pub use crate::checkpoint::{
load_checkpoint, save_checkpoint, save_checkpoint_with_best, save_inference_checkpoint,
validate_quantization_metadata, CheckpointActivationQuant, CheckpointKvQuant,
CheckpointQuantization, DataLoaderState, LoadedCheckpoint, QuantizedLayerEntry, RngState,
CHECKPOINT_MAGIC, CHECKPOINT_VERSION, LEGACY_CHECKPOINT_VERSION, QUANTIZATION_SCHEMA_VERSION,
QUANTIZATION_SCHEMA_VERSION_MIN,
};
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct LearningRateSchedule {
pub base_learning_rate: f32,
pub minimum_learning_rate: f32,
pub warmup_steps: usize,
pub total_steps: usize,
}
impl LearningRateSchedule {
pub fn learning_rate(&self, step: usize) -> f32 {
if self.warmup_steps > 0 && step < self.warmup_steps {
return self.base_learning_rate * (step + 1) as f32 / self.warmup_steps as f32;
}
if self.total_steps <= self.warmup_steps {
return self.minimum_learning_rate;
}
let decay_step = step.saturating_sub(self.warmup_steps);
let decay_steps = self.total_steps - self.warmup_steps;
let progress = (decay_step as f32 / decay_steps as f32).clamp(0.0, 1.0);
let cosine = 0.5 * (1.0 + (std::f32::consts::PI * progress).cos());
self.minimum_learning_rate + (self.base_learning_rate - self.minimum_learning_rate) * cosine
}
}
fn default_gradient_accumulation_steps() -> usize {
1
}
fn default_mixed_precision() -> MixedPrecisionConfig {
MixedPrecisionConfig::disabled()
}
fn default_activation_checkpointing() -> bool {
false
}
fn default_data_parallel_workers() -> usize {
1
}
fn default_activation_checkpoint_interval() -> usize {
1
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum SelectiveCheckpoint {
#[default]
Full,
Attention,
Ffn,
}
impl SelectiveCheckpoint {
pub fn as_str(self) -> &'static str {
match self {
Self::Full => "full",
Self::Attention => "attention",
Self::Ffn => "ffn",
}
}
pub fn parse(value: &str) -> Result<Self, String> {
match value.trim().to_ascii_lowercase().as_str() {
"full" | "block" => Ok(Self::Full),
"attention" | "attn" => Ok(Self::Attention),
"ffn" | "mlp" | "feedforward" => Ok(Self::Ffn),
other => Err(format!(
"unknown selective checkpoint mode '{other}'; expected full, attention, or ffn"
)),
}
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct LowMemoryProfile {
#[serde(default = "default_activation_checkpoint_interval")]
pub activation_checkpoint_interval: usize,
#[serde(default)]
pub selective_checkpoint: SelectiveCheckpoint,
#[serde(default)]
pub attention_chunk_size: usize,
#[serde(default)]
pub layer_offload: bool,
#[serde(default)]
pub trim_caches_each_microbatch: bool,
#[serde(default)]
pub lora: Option<LoraConfig>,
#[serde(default)]
pub freeze: Vec<String>,
#[serde(default)]
pub qat: Option<crate::qat::QatConfig>,
#[serde(default)]
pub rebuild_token_cache: bool,
}
impl Default for LowMemoryProfile {
fn default() -> Self {
Self {
activation_checkpoint_interval: default_activation_checkpoint_interval(),
selective_checkpoint: SelectiveCheckpoint::Full,
attention_chunk_size: 0,
layer_offload: false,
trim_caches_each_microbatch: false,
lora: None,
freeze: Vec::new(),
qat: None,
rebuild_token_cache: false,
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct TrainerConfig {
pub batch_size: usize,
pub sequence_length: usize,
pub max_gradient_norm: f32,
pub log_every: usize,
#[serde(default = "default_gradient_accumulation_steps")]
pub gradient_accumulation_steps: usize,
#[serde(default = "default_mixed_precision")]
pub mixed_precision: MixedPrecisionConfig,
#[serde(default = "default_activation_checkpointing")]
pub activation_checkpointing: bool,
#[serde(default = "default_data_parallel_workers")]
pub data_parallel_workers: usize,
#[serde(default)]
pub low_memory: LowMemoryProfile,
}
#[derive(Clone, Debug)]
pub struct TrainingMetrics {
pub step: usize,
pub micro_batches: usize,
pub micro_batches_processed: usize,
pub loss: f32,
pub perplexity: f32,
pub learning_rate: f32,
pub gradient_norm: f32,
pub tokens_per_second: f64,
pub loss_scale: f32,
pub overflow: bool,
pub optimizer_stepped: bool,
}
pub fn training_metrics_jsonl_line(metrics: &TrainingMetrics, rss_bytes: Option<usize>) -> String {
let mut object = Map::new();
object.insert("step".into(), json!(metrics.step));
object.insert("loss".into(), json_f32(metrics.loss));
object.insert("ppl".into(), json_f32(metrics.perplexity));
object.insert("lr".into(), json_f32(metrics.learning_rate));
object.insert("grad_norm".into(), json_f32(metrics.gradient_norm));
object.insert("loss_scale".into(), json_f32(metrics.loss_scale));
object.insert(
"rss".into(),
match rss_bytes {
Some(bytes) => json!(bytes),
None => Value::Null,
},
);
Value::Object(object).to_string()
}
fn json_f32(value: f32) -> Value {
let number = value as f64;
if number.is_finite() {
json!(number)
} else {
Value::String(number.to_string())
}
}
#[derive(Clone, Debug)]
pub struct EvaluationMetrics {
pub step: usize,
pub loss: f32,
pub perplexity: f32,
pub tokens_processed: usize,
pub training_tokens_processed: usize,
pub wall_time_seconds: f64,
pub improved: bool,
pub best_loss: f32,
pub best_checkpoint: PathBuf,
pub evaluations_without_improvement: usize,
pub stopped_early: bool,
}
#[cfg(feature = "training")]
pub enum TrainingBackend {
Cpu,
Gpu(GpuContext),
}
#[derive(Debug)]
pub enum TrainerError {
InvalidData(String),
Io(std::io::Error),
Json(serde_json::Error),
Tokenizer(TokenizerError),
Model(ModelConfigError),
ModelRuntime(ModelError),
Optimizer(OptimizerError),
Gpu(GpuError),
#[cfg(feature = "training")]
DataLoader(DataLoaderError),
MemoryBudget(MemoryBudgetError),
Amp(AmpError),
#[cfg(feature = "training")]
DataParallel(DataParallelError),
}
impl fmt::Display for TrainerError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidData(message) => write!(formatter, "training data error: {message}"),
Self::Io(error) => error.fmt(formatter),
Self::Json(error) => error.fmt(formatter),
Self::Tokenizer(error) => error.fmt(formatter),
Self::Model(error) => error.fmt(formatter),
Self::ModelRuntime(error) => error.fmt(formatter),
Self::Optimizer(error) => error.fmt(formatter),
Self::Gpu(error) => error.fmt(formatter),
#[cfg(feature = "training")]
Self::DataLoader(error) => error.fmt(formatter),
Self::MemoryBudget(error) => error.fmt(formatter),
Self::Amp(error) => error.fmt(formatter),
#[cfg(feature = "training")]
Self::DataParallel(error) => error.fmt(formatter),
}
}
}
impl Error for TrainerError {}
macro_rules! error_conversion {
($source:ty, $variant:ident) => {
impl From<$source> for TrainerError {
fn from(error: $source) -> Self {
Self::$variant(error)
}
}
};
}
error_conversion!(std::io::Error, Io);
error_conversion!(serde_json::Error, Json);
error_conversion!(TokenizerError, Tokenizer);
error_conversion!(ModelConfigError, Model);
error_conversion!(ModelError, ModelRuntime);
error_conversion!(OptimizerError, Optimizer);
error_conversion!(GpuError, Gpu);
#[cfg(feature = "training")]
error_conversion!(DataLoaderError, DataLoader);
error_conversion!(MemoryBudgetError, MemoryBudget);
error_conversion!(AmpError, Amp);
#[cfg(feature = "training")]
error_conversion!(DataParallelError, DataParallel);
#[cfg(feature = "training")]
enum DataParallelPool {
Cpu(CpuDataParallelPool),
#[cfg(feature = "gpu")]
Gpu(GpuDataParallelPool),
}
#[cfg(feature = "training")]
pub struct Trainer {
pub model: CausalLM,
pub tokenizer: BpeTokenizer,
pub optimizer: AdamW,
pub schedule: LearningRateSchedule,
pub config: TrainerConfig,
backend: TrainingBackend,
data_loader: PrefetchDataLoader,
data_parallel: Option<DataParallelPool>,
step: usize,
micro_batches_processed: usize,
tokens_processed: usize,
validation_data: Option<TokenDataset>,
evaluation_every: usize,
best_checkpoint_path: Option<PathBuf>,
best_validation_loss: Option<f32>,
last_evaluation_step: Option<usize>,
early_stopping_patience: usize,
early_stopping_min_delta: f32,
evaluations_without_improvement: usize,
stopped_early: bool,
master_weights: Option<Vec<Vec<f32>>>,
loss_scaler: DynamicLossScaler,
metrics_jsonl: Option<BufWriter<File>>,
}
#[cfg(feature = "training")]
impl Trainer {
pub fn new(
model: CausalLM,
tokenizer: BpeTokenizer,
tokens: Vec<usize>,
optimizer_config: AdamWConfig,
schedule: LearningRateSchedule,
config: TrainerConfig,
backend: TrainingBackend,
) -> Result<Self, TrainerError> {
Self::new_with_memory_budget(
model,
tokenizer,
tokens,
optimizer_config,
schedule,
config,
backend,
None,
)
}
pub fn new_with_memory_budget(
model: CausalLM,
tokenizer: BpeTokenizer,
tokens: Vec<usize>,
optimizer_config: AdamWConfig,
schedule: LearningRateSchedule,
config: TrainerConfig,
backend: TrainingBackend,
memory_budget_bytes: Option<usize>,
) -> Result<Self, TrainerError> {
Self::new_with_dataset_and_memory_budget(
model,
tokenizer,
TokenDataset::from_tokens(tokens),
optimizer_config,
schedule,
config,
backend,
memory_budget_bytes,
2,
)
}
#[allow(clippy::too_many_arguments)]
pub fn new_with_dataset_and_memory_budget(
model: CausalLM,
tokenizer: BpeTokenizer,
training_data: TokenDataset,
optimizer_config: AdamWConfig,
schedule: LearningRateSchedule,
config: TrainerConfig,
backend: TrainingBackend,
memory_budget_bytes: Option<usize>,
prefetch_batches: usize,
) -> Result<Self, TrainerError> {
let mut model = model;
if let Some(lora) = &config.low_memory.lora {
model.install_lora(lora)?;
}
if let Some(qat) = &config.low_memory.qat {
model.enable_qat_with(qat.clone())?;
}
if !config.low_memory.freeze.is_empty() {
crate::peft::freeze_parameters(&model.named_parameters(), &config.low_memory.freeze);
}
validate_training_inputs(&model, &tokenizer, &training_data, &schedule, &config)?;
validate_data_parallel(&config, &backend)?;
enforce_memory_budget(&model.config, &config, &backend, memory_budget_bytes)?;
let named = model.named_parameters();
let trainable: Vec<(String, Tensor)> = named
.into_iter()
.filter(|(_, parameter)| parameter.requires_grad())
.collect();
let optimizer = if trainable.is_empty() {
return Err(TrainerError::InvalidData(
"no trainable parameters after freeze/LoRA setup".into(),
));
} else {
AdamW::new_grouped(&trainable, optimizer_config)?
};
let data_loader = PrefetchDataLoader::new(
training_data.clone(),
config.sequence_length,
config.batch_size,
model.config.seed,
0,
prefetch_batches,
)?;
let data_parallel = create_data_parallel_pool(&model.config, &config, &backend)?;
let (master_weights, loss_scaler) = initialize_mixed_precision(&model, &config, None)?;
Ok(Self {
model,
tokenizer,
optimizer,
schedule,
config,
backend,
data_loader,
data_parallel,
step: 0,
micro_batches_processed: 0,
tokens_processed: 0,
validation_data: None,
evaluation_every: 0,
best_checkpoint_path: None,
best_validation_loss: None,
last_evaluation_step: None,
early_stopping_patience: 0,
early_stopping_min_delta: 0.0,
evaluations_without_improvement: 0,
stopped_early: false,
master_weights,
loss_scaler,
metrics_jsonl: None,
})
}
pub fn configure_metrics_jsonl(
&mut self,
path: impl AsRef<Path>,
) -> Result<(), TrainerError> {
let path = path.as_ref();
if let Some(parent) = path.parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent)?;
}
}
let file = File::create(path)?;
self.metrics_jsonl = Some(BufWriter::new(file));
Ok(())
}
pub fn configure_evaluation(
&mut self,
validation_tokens: Vec<usize>,
every_steps: usize,
best_checkpoint_path: impl Into<PathBuf>,
) -> Result<(), TrainerError> {
self.configure_evaluation_dataset(
TokenDataset::from_tokens(validation_tokens),
every_steps,
best_checkpoint_path,
)
}
pub fn configure_evaluation_dataset(
&mut self,
validation_data: TokenDataset,
every_steps: usize,
best_checkpoint_path: impl Into<PathBuf>,
) -> Result<(), TrainerError> {
if every_steps == 0 {
return Err(TrainerError::InvalidData(
"evaluation interval must be positive".into(),
));
}
if validation_data.token_count() <= self.config.sequence_length {
return Err(TrainerError::InvalidData(
"validation corpus is too short for one evaluation sequence".into(),
));
}
validation_data.validate_token_ids(self.model.config.vocab_size)?;
self.validation_data = Some(validation_data);
self.evaluation_every = every_steps;
self.best_checkpoint_path = Some(best_checkpoint_path.into());
Ok(())
}
pub fn configure_early_stopping(
&mut self,
patience: usize,
min_delta: f32,
) -> Result<(), TrainerError> {
if !min_delta.is_finite() || min_delta < 0.0 {
return Err(TrainerError::InvalidData(
"early stopping min delta must be finite and non-negative".into(),
));
}
self.early_stopping_patience = patience;
self.early_stopping_min_delta = min_delta;
self.stopped_early = false;
Ok(())
}
pub fn from_text(
model: CausalLM,
tokenizer: BpeTokenizer,
text: &str,
optimizer_config: AdamWConfig,
schedule: LearningRateSchedule,
config: TrainerConfig,
backend: TrainingBackend,
) -> Result<Self, TrainerError> {
let tokens = tokenizer.encode(text);
Self::new(
model,
tokenizer,
tokens,
optimizer_config,
schedule,
config,
backend,
)
}
pub fn train_steps(&mut self, count: usize) -> Result<Vec<TrainingMetrics>, TrainerError> {
let mut metrics = Vec::with_capacity(count);
let session_started = Instant::now();
for completed in 1..=count {
let current = self.train_step()?;
self.write_metrics_jsonl(¤t)?;
if self.config.log_every > 0 && current.step % self.config.log_every == 0 {
self.emit_train_progress(¤t, count, completed, session_started);
}
let step = current.step;
metrics.push(current);
if self.evaluation_every > 0 && step % self.evaluation_every == 0 {
let evaluation = self.evaluate_and_save_best()?;
if evaluation.stopped_early {
break;
}
}
}
if let Some(writer) = self.metrics_jsonl.as_mut() {
writer.flush()?;
}
Ok(metrics)
}
fn write_metrics_jsonl(&mut self, current: &TrainingMetrics) -> Result<(), TrainerError> {
let Some(writer) = self.metrics_jsonl.as_mut() else {
return Ok(());
};
let line = training_metrics_jsonl_line(current, process_peak_rss_bytes());
writeln!(writer, "{line}")?;
Ok(())
}
fn emit_train_progress(
&self,
current: &TrainingMetrics,
session_steps: usize,
completed: usize,
session_started: Instant,
) {
let remaining = session_steps.saturating_sub(completed);
let eta_seconds = if completed > 0 && remaining > 0 {
session_started.elapsed().as_secs_f64() / completed as f64 * remaining as f64
} else {
0.0
};
let eta = format_eta(eta_seconds);
let mut fields = vec![
("step", LogValue::Usize(current.step)),
("micro_batches", LogValue::Usize(current.micro_batches)),
(
"micro_batches_processed",
LogValue::Usize(current.micro_batches_processed),
),
("loss", LogValue::f32(current.loss, 5)),
("ppl", LogValue::f32(current.perplexity, 3)),
("lr", LogValue::f32(current.learning_rate, 7)),
("grad", LogValue::f32(current.gradient_norm, 4)),
("tok/s", LogValue::f64(current.tokens_per_second, 1)),
("eta", LogValue::Str(&eta)),
("tokens", LogValue::Usize(self.tokens_processed)),
];
if let Some(best) = self.best_validation_loss {
fields.push(("best_val", LogValue::f32(best, 5)));
}
if let Some(peak_rss) = process_peak_rss_bytes() {
fields.push(("peak_rss", LogValue::Bytes(peak_rss)));
}
fields.push((
"peak_alloc",
LogValue::Bytes(cpu_memory_stats().peak_tensor_bytes),
));
if let TrainingBackend::Gpu(gpu) = &self.backend {
if let Ok(stats) = gpu.memory_stats() {
fields.push(("gpu_live", LogValue::Bytes(stats.live_tensor_bytes)));
fields.push(("gpu_peak", LogValue::Bytes(stats.peak_tensor_bytes)));
}
}
if self.config.mixed_precision.enabled {
fields.push(("loss_scale", LogValue::f32(current.loss_scale, 1)));
fields.push(("overflow", LogValue::Bool(current.overflow)));
fields.push(("opt_step", LogValue::Bool(current.optimizer_stepped)));
}
log::emit("train", &fields);
}
pub fn train_step(&mut self) -> Result<TrainingMetrics, TrainerError> {
let started = Instant::now();
let report_step = self.step + 1;
anomaly::set_anomaly_training_step(report_step);
let accumulation_steps = self.config.gradient_accumulation_steps;
let inv_accumulation = 1.0 / accumulation_steps as f32;
let amp_enabled = self.config.mixed_precision.enabled;
let loss_scale = if amp_enabled {
self.loss_scaler.scale()
} else {
1.0
};
let backward_scale = if amp_enabled {
loss_scale * inv_accumulation
} else if accumulation_steps > 1 {
inv_accumulation
} else {
1.0
};
if amp_enabled {
self.sync_working_weights_from_masters()?;
}
self.model.zero_grad();
let parameters = trainable_parameters(&self.model);
let learning_rate = self.schedule.learning_rate(self.step);
let mut loss_sum = 0.0f32;
for micro_batch in 0..accumulation_steps {
let (input_ids, target_ids) = self.next_batch()?;
let dropout_seed = crate::derive_seed(
self.model.config.seed,
self.micro_batches_processed.saturating_add(micro_batch) as u64,
);
let checkpoint = self.config.activation_checkpointing;
let checkpoint_interval = self.config.low_memory.activation_checkpoint_interval;
let selective = self.config.low_memory.selective_checkpoint;
let attention_chunk = self.config.low_memory.attention_chunk_size;
match &self.backend {
TrainingBackend::Cpu => {
if let Some(DataParallelPool::Cpu(pool)) = &self.data_parallel {
let _span = profile::span("train.data_parallel");
let weights = Arc::new(capture_parameter_weights(¶meters)?);
let (loss, grads) = pool.forward_backward(
weights,
&input_ids,
&target_ids,
dropout_seed,
checkpoint,
backward_scale,
)?;
loss_sum += loss;
accumulate_parameter_grads(¶meters, &grads)?;
} else {
let logits = self.model.forward_training_with_low_memory(
&input_ids,
None,
dropout_seed,
checkpoint,
checkpoint_interval,
selective,
attention_chunk,
)?;
let loss = self.model.compute_loss(&logits, &target_ids)?;
loss_sum += loss.item();
if (backward_scale - 1.0).abs() < f32::EPSILON {
loss.backward();
} else {
loss.mul(&Tensor::scalar(backward_scale, false))
.backward();
}
}
}
TrainingBackend::Gpu(gpu) => {
#[cfg(feature = "gpu")]
if let Some(DataParallelPool::Gpu(pool)) = &self.data_parallel {
let _span = profile::span("train.data_parallel");
let weights = Arc::new(capture_parameter_weights(¶meters)?);
let (loss, grads) = pool.forward_backward(
weights,
&input_ids,
&target_ids,
dropout_seed,
checkpoint,
backward_scale,
)?;
loss_sum += loss;
accumulate_parameter_gpu_grads(¶meters, &grads, gpu)?;
} else {
let logits = if self.config.low_memory.layer_offload {
self.model.forward_training_gpu_with_layer_offload(
&input_ids,
dropout_seed,
checkpoint,
gpu,
)?
} else {
self.model.forward_training_gpu_with_checkpointing(
&input_ids,
dropout_seed,
checkpoint,
gpu,
)?
};
let loss = self.model.compute_loss_gpu(&logits, &target_ids, gpu)?;
loss_sum += loss.to_cpu()?.item();
if (backward_scale - 1.0).abs() < f32::EPSILON {
loss.backward_gpu(gpu)?;
} else {
loss.gpu_scale(backward_scale, gpu)?.backward_gpu(gpu)?;
}
if self.config.low_memory.trim_caches_each_microbatch {
let _ = gpu.trim_dispatch_caches();
}
}
#[cfg(not(feature = "gpu"))]
{
let _ = gpu;
return Err(TrainerError::InvalidData(
"GPU training requires the `gpu` feature".into(),
));
}
}
}
}
let overflow = if amp_enabled {
match &self.backend {
TrainingBackend::Cpu => {
parameters_overflow(¶meters, self.config.mixed_precision.format)
}
TrainingBackend::Gpu(gpu) => parameters_overflow_gpu(
¶meters,
self.config.mixed_precision.format,
gpu,
)?,
}
} else {
false
};
let (gradient_norm, optimizer_stepped) = if overflow {
self.loss_scaler.update(true);
self.model.zero_grad();
(0.0, false)
} else {
if amp_enabled && loss_scale != 1.0 {
if matches!(self.backend, TrainingBackend::Cpu) {
scale_parameter_grads(¶meters, 1.0 / loss_scale);
}
}
if amp_enabled {
self.restore_masters_into_parameters()?;
}
let gradient_norm = match &self.backend {
TrainingBackend::Cpu => {
let gradient_norm =
clip_gradient_norm(¶meters, self.config.max_gradient_norm);
self.optimizer.step(¶meters, learning_rate)?;
gradient_norm
}
TrainingBackend::Gpu(gpu) => {
let (gradient_norm, clip_scale) =
gpu_gradient_norm_and_scale(¶meters, self.config.max_gradient_norm, gpu)?;
let gradient_scale = if amp_enabled {
clip_scale / loss_scale
} else {
clip_scale
};
self.optimizer
.step_gpu(¶meters, learning_rate, gradient_scale, gpu)?;
gradient_norm
}
};
if amp_enabled {
self.capture_masters_from_parameters()?;
self.sync_working_weights_from_masters()?;
self.loss_scaler.update(false);
}
(gradient_norm, true)
};
let loss_value = loss_sum / accumulation_steps as f32;
self.step += 1;
self.micro_batches_processed += accumulation_steps;
let batch_tokens =
self.config.batch_size * self.config.sequence_length * accumulation_steps;
self.tokens_processed += batch_tokens;
let tokens_per_second = batch_tokens as f64 / started.elapsed().as_secs_f64().max(1e-9);
Ok(TrainingMetrics {
step: self.step,
micro_batches: accumulation_steps,
micro_batches_processed: self.micro_batches_processed,
loss: loss_value,
perplexity: loss_value.exp(),
learning_rate,
gradient_norm,
tokens_per_second,
loss_scale: if amp_enabled {
self.loss_scaler.scale()
} else {
1.0
},
overflow,
optimizer_stepped,
})
}
pub fn evaluate_and_save_best(&mut self) -> Result<EvaluationMetrics, TrainerError> {
anomaly::set_anomaly_training_step(self.step);
let _no_grad = crate::no_grad();
if self.config.mixed_precision.enabled {
self.restore_masters_into_parameters()?;
}
let best_checkpoint = self.best_checkpoint_path.clone().ok_or_else(|| {
TrainerError::InvalidData("evaluation has not been configured".into())
})?;
let started = Instant::now();
let sequence_length = self.config.sequence_length;
let validation_data = self.validation_data.as_ref().ok_or_else(|| {
TrainerError::InvalidData("evaluation has not been configured".into())
})?;
let sequence_count = (validation_data.token_count() - 1) / sequence_length;
let mut weighted_loss = 0.0f64;
let mut tokens_processed = 0usize;
for first_sequence in (0..sequence_count).step_by(self.config.batch_size) {
let batch_size = self.config.batch_size.min(sequence_count - first_sequence);
let batch_tokens = batch_size * sequence_length;
let mut inputs = Vec::with_capacity(batch_tokens);
let mut targets = Vec::with_capacity(batch_tokens);
for batch_index in 0..batch_size {
let start = (first_sequence + batch_index) * sequence_length;
let end = start + sequence_length;
let mut sequence = Vec::with_capacity(sequence_length + 1);
validation_data.read_range(start, end + 1, &mut sequence)?;
inputs.extend_from_slice(&sequence[..sequence_length]);
targets.extend_from_slice(&sequence[1..]);
}
let shape = vec![batch_size, sequence_length];
let input_ids = IndexTensor::new(inputs, shape.clone());
let target_ids = IndexTensor::new(targets, shape);
let batch_loss = match &self.backend {
TrainingBackend::Cpu => {
let logits = self.model.forward(&input_ids)?;
self.model.compute_loss(&logits, &target_ids)?.item()
}
TrainingBackend::Gpu(gpu) => {
let logits = self.model.forward_gpu(&input_ids, gpu)?;
self.model
.compute_loss_gpu(&logits, &target_ids, gpu)?
.to_cpu()?
.item()
}
};
weighted_loss += batch_loss as f64 * batch_tokens as f64;
tokens_processed += batch_tokens;
}
let loss = (weighted_loss / tokens_processed as f64) as f32;
let wall_time_seconds = started.elapsed().as_secs_f64();
let improved = self
.best_validation_loss
.map(|best| loss + self.early_stopping_min_delta < best)
.unwrap_or(true);
if improved {
self.best_validation_loss = Some(loss);
self.evaluations_without_improvement = 0;
self.save_checkpoint(&best_checkpoint)?;
} else {
self.evaluations_without_improvement =
self.evaluations_without_improvement.saturating_add(1);
}
let stopped_early = self.early_stopping_patience > 0
&& self.evaluations_without_improvement >= self.early_stopping_patience;
if stopped_early {
self.stopped_early = true;
}
self.last_evaluation_step = Some(self.step);
let metrics = EvaluationMetrics {
step: self.step,
loss,
perplexity: loss.exp(),
tokens_processed,
training_tokens_processed: self.tokens_processed,
wall_time_seconds,
improved,
best_loss: self.best_validation_loss.unwrap_or(loss),
best_checkpoint,
evaluations_without_improvement: self.evaluations_without_improvement,
stopped_early,
};
let best_checkpoint_display = metrics.best_checkpoint.display().to_string();
let mut fields = vec![
("step", LogValue::Usize(metrics.step)),
("val_loss", LogValue::f32(metrics.loss, 5)),
("val_ppl", LogValue::f32(metrics.perplexity, 3)),
("val_tokens", LogValue::Usize(metrics.tokens_processed)),
(
"train_tokens",
LogValue::Usize(metrics.training_tokens_processed),
),
("wall_s", LogValue::f64(metrics.wall_time_seconds, 3)),
("improved", LogValue::Bool(metrics.improved)),
("best_loss", LogValue::f32(metrics.best_loss, 5)),
("best_checkpoint", LogValue::Str(&best_checkpoint_display)),
(
"stale_evals",
LogValue::Usize(metrics.evaluations_without_improvement),
),
];
if self.early_stopping_patience > 0 {
fields.push((
"early_stop_patience",
LogValue::Usize(self.early_stopping_patience),
));
fields.push(("stopped_early", LogValue::Bool(metrics.stopped_early)));
}
log::emit("eval", &fields);
if self.config.mixed_precision.enabled {
self.sync_working_weights_from_masters()?;
}
Ok(metrics)
}
pub fn save_checkpoint(&self, path: impl AsRef<Path>) -> Result<(), TrainerError> {
if self.config.mixed_precision.enabled {
self.restore_masters_into_parameters()?;
}
let result = save_checkpoint_with_best(
path,
&self.model,
&self.optimizer,
&self.tokenizer,
&self.schedule,
&self.config,
self.step,
self.data_loader.cursor(),
self.tokens_processed,
self.micro_batches_processed,
self.best_validation_loss,
self.evaluations_without_improvement,
self.loss_scaler_state(),
);
if self.config.mixed_precision.enabled {
self.sync_working_weights_from_masters()?;
}
result
}
pub fn loss_scale(&self) -> f32 {
if self.config.mixed_precision.enabled {
self.loss_scaler.scale()
} else {
1.0
}
}
pub fn loss_scaler_state(&self) -> Option<LossScalerState> {
self.config
.mixed_precision
.enabled
.then(|| self.loss_scaler.state())
}
fn sync_working_weights_from_masters(&self) -> Result<(), TrainerError> {
let Some(masters) = &self.master_weights else {
return Ok(());
};
let format = self.config.mixed_precision.format;
for (parameter, master) in self.model.parameters().iter().zip(masters.iter()) {
parameter.set_data(cast_storage(master, format));
}
self.reupload_parameters_if_gpu()?;
Ok(())
}
fn restore_masters_into_parameters(&self) -> Result<(), TrainerError> {
let Some(masters) = &self.master_weights else {
return Ok(());
};
for (parameter, master) in self.model.parameters().iter().zip(masters.iter()) {
parameter.set_data_keep_grad(master.clone());
}
self.reupload_parameters_if_gpu()?;
Ok(())
}
fn reupload_parameters_if_gpu(&self) -> Result<(), TrainerError> {
match &self.backend {
TrainingBackend::Cpu => Ok(()),
TrainingBackend::Gpu(gpu) => {
#[cfg(feature = "gpu")]
{
self.model.prepare_gpu(gpu)?;
Ok(())
}
#[cfg(not(feature = "gpu"))]
{
let _ = gpu;
Err(TrainerError::InvalidData(
"GPU training requires the `gpu` feature".into(),
))
}
}
}
}
fn capture_masters_from_parameters(&mut self) -> Result<(), TrainerError> {
let Some(masters) = &mut self.master_weights else {
return Ok(());
};
for (parameter, master) in self.model.parameters().iter().zip(masters.iter_mut()) {
*master = parameter.to_cpu()?.data();
}
Ok(())
}
pub fn from_checkpoint(
checkpoint: LoadedCheckpoint,
tokens: Vec<usize>,
backend: TrainingBackend,
) -> Result<Self, TrainerError> {
Self::from_checkpoint_with_memory_budget(checkpoint, tokens, backend, None)
}
pub fn from_checkpoint_with_memory_budget(
checkpoint: LoadedCheckpoint,
tokens: Vec<usize>,
backend: TrainingBackend,
memory_budget_bytes: Option<usize>,
) -> Result<Self, TrainerError> {
Self::from_checkpoint_with_dataset_and_memory_budget(
checkpoint,
TokenDataset::from_tokens(tokens),
backend,
memory_budget_bytes,
2,
)
}
pub fn from_checkpoint_with_dataset_and_memory_budget(
checkpoint: LoadedCheckpoint,
training_data: TokenDataset,
backend: TrainingBackend,
memory_budget_bytes: Option<usize>,
prefetch_batches: usize,
) -> Result<Self, TrainerError> {
validate_training_inputs(
&checkpoint.model,
&checkpoint.tokenizer,
&training_data,
&checkpoint.schedule,
&checkpoint.trainer_config,
)?;
validate_data_parallel(&checkpoint.trainer_config, &backend)?;
enforce_memory_budget(
&checkpoint.model.config,
&checkpoint.trainer_config,
&backend,
memory_budget_bytes,
)?;
let data_loader = PrefetchDataLoader::new(
training_data.clone(),
checkpoint.data_loader.sequence_length,
checkpoint.data_loader.batch_size,
checkpoint.data_loader.seed,
checkpoint.data_loader.cursor,
prefetch_batches,
);
let data_loader = data_loader?;
let data_parallel = create_data_parallel_pool(
&checkpoint.model.config,
&checkpoint.trainer_config,
&backend,
)?;
let micro_batches_processed = checkpoint.rng.dropout_stream;
let loss_scaler_state = checkpoint.loss_scaler.clone();
let (master_weights, loss_scaler) = initialize_mixed_precision(
&checkpoint.model,
&checkpoint.trainer_config,
loss_scaler_state.as_ref(),
)?;
Ok(Self {
model: checkpoint.model,
tokenizer: checkpoint.tokenizer,
optimizer: checkpoint.optimizer,
schedule: checkpoint.schedule,
config: checkpoint.trainer_config,
backend,
data_loader,
data_parallel,
step: checkpoint.training_step,
micro_batches_processed,
tokens_processed: checkpoint.tokens_processed,
validation_data: None,
evaluation_every: 0,
best_checkpoint_path: None,
best_validation_loss: checkpoint.best_validation_loss,
last_evaluation_step: None,
early_stopping_patience: 0,
early_stopping_min_delta: 0.0,
evaluations_without_improvement: checkpoint.evaluations_without_improvement,
stopped_early: false,
master_weights,
loss_scaler,
metrics_jsonl: None,
})
}
pub fn step(&self) -> usize {
self.step
}
pub fn stopped_early(&self) -> bool {
self.stopped_early
}
pub fn evaluations_without_improvement(&self) -> usize {
self.evaluations_without_improvement
}
pub fn best_validation_loss(&self) -> Option<f32> {
self.best_validation_loss
}
pub fn micro_batches_processed(&self) -> usize {
self.micro_batches_processed
}
pub fn cursor(&self) -> usize {
self.data_loader.cursor()
}
pub fn tokens_processed(&self) -> usize {
self.tokens_processed
}
pub fn last_evaluation_step(&self) -> Option<usize> {
self.last_evaluation_step
}
pub fn extend_schedule_total_steps(
&mut self,
additional_steps: usize,
) -> Option<(usize, usize)> {
let previous = self.schedule.total_steps;
let required = self.step.saturating_add(additional_steps);
if required > previous {
self.schedule.total_steps = required;
Some((previous, required))
} else {
None
}
}
pub fn schedule_total_steps(&self) -> usize {
self.schedule.total_steps
}
fn next_batch(&mut self) -> Result<(IndexTensor, IndexTensor), TrainerError> {
let _span = profile::span("data.load_batch");
let batch = self.data_loader.next_batch()?;
Ok((batch.input_ids, batch.target_ids))
}
}
#[cfg(feature = "training")]
fn initialize_mixed_precision(
model: &CausalLM,
config: &TrainerConfig,
scaler_state: Option<&LossScalerState>,
) -> Result<(Option<Vec<Vec<f32>>>, DynamicLossScaler), TrainerError> {
config.mixed_precision.validate()?;
let loss_scaler = match scaler_state {
Some(state) if config.mixed_precision.enabled => {
DynamicLossScaler::from_state(&config.mixed_precision, state)
}
_ => DynamicLossScaler::from_config(&config.mixed_precision),
};
if !config.mixed_precision.enabled {
return Ok((None, loss_scaler));
}
let masters = model
.parameters()
.iter()
.map(|parameter| Ok(parameter.to_cpu()?.data()))
.collect::<Result<Vec<_>, TrainerError>>()?;
let format = config.mixed_precision.format;
for (parameter, master) in model.parameters().iter().zip(masters.iter()) {
parameter.set_data(cast_storage(master, format));
}
Ok((Some(masters), loss_scaler))
}
#[cfg(feature = "training")]
fn trainable_parameters(model: &CausalLM) -> Vec<Tensor> {
model
.named_parameters()
.into_iter()
.filter(|(_, parameter)| parameter.requires_grad())
.map(|(_, parameter)| parameter)
.collect()
}
#[cfg(feature = "training")]
pub fn clip_gradient_norm(parameters: &[Tensor], maximum_norm: f32) -> f32 {
if maximum_norm <= 0.0 {
return 0.0;
}
let squared_norm = parameters
.iter()
.filter_map(Tensor::grad)
.flat_map(|gradient| gradient.into_iter())
.map(|value| (value as f64) * (value as f64))
.sum::<f64>();
let norm = squared_norm.sqrt() as f32;
if norm.is_finite() && norm > maximum_norm {
let scale = maximum_norm / (norm + 1e-6);
for parameter in parameters {
parameter.scale_grad(scale);
}
}
norm
}
#[cfg(all(feature = "training", feature = "gpu"))]
fn gpu_gradient_norm_and_scale(
parameters: &[Tensor],
maximum_norm: f32,
gpu: &GpuContext,
) -> Result<(f32, f32), TrainerError> {
if maximum_norm <= 0.0 {
return Ok((0.0, 1.0));
}
let mut squared_norm = 0.0f64;
for parameter in parameters {
let Some(gradient) = parameter.gpu_grad() else {
continue;
};
squared_norm += gpu.squared_norm(&gradient)?.to_vec()?[0] as f64;
}
let norm = squared_norm.sqrt() as f32;
let scale = if norm.is_finite() && norm > maximum_norm {
maximum_norm / (norm + 1e-6)
} else {
1.0
};
Ok((norm, scale))
}
#[cfg(all(feature = "training", not(feature = "gpu")))]
fn gpu_gradient_norm_and_scale(
_parameters: &[Tensor],
_maximum_norm: f32,
_gpu: &GpuContext,
) -> Result<(f32, f32), TrainerError> {
Err(TrainerError::Gpu(GpuError::Initialization(
"GPU support is not compiled in; rebuild with `--features gpu`".into(),
)))
}
#[cfg(feature = "training")]
pub fn plan_training_memory(
model_config: &CausalLmConfig,
trainer_config: &TrainerConfig,
backend: &TrainingBackend,
memory_budget_bytes: Option<usize>,
) -> (MemoryEstimate, DeviceBudget) {
let mut estimate = estimate_training_memory(model_config, trainer_config);
let budget = match backend {
TrainingBackend::Cpu => DeviceBudget::cpu(memory_budget_bytes),
TrainingBackend::Gpu(gpu) => {
apply_gpu_parameter_residency(
&mut estimate,
model_config,
trainer_config.low_memory.layer_offload,
);
let _ = gpu.apply_memory_budget(memory_budget_bytes);
let mut budget = gpu.memory_budget();
if let Some(total_bytes) = memory_budget_bytes {
budget.total_bytes = total_bytes;
}
budget
}
};
(estimate, budget)
}
#[cfg(feature = "training")]
fn enforce_memory_budget(
model_config: &CausalLmConfig,
trainer_config: &TrainerConfig,
backend: &TrainingBackend,
memory_budget_bytes: Option<usize>,
) -> Result<(), TrainerError> {
let (estimate, budget) =
plan_training_memory(model_config, trainer_config, backend, memory_budget_bytes);
check_memory_budget(&estimate, &budget)?;
Ok(())
}
#[cfg(feature = "training")]
fn create_data_parallel_pool(
model_config: &CausalLmConfig,
config: &TrainerConfig,
backend: &TrainingBackend,
) -> Result<Option<DataParallelPool>, TrainerError> {
if config.data_parallel_workers <= 1 {
return Ok(None);
}
match backend {
TrainingBackend::Cpu => Ok(Some(DataParallelPool::Cpu(CpuDataParallelPool::new(
model_config.clone(),
config.data_parallel_workers,
)?))),
TrainingBackend::Gpu(_) => {
#[cfg(feature = "gpu")]
{
Ok(Some(DataParallelPool::Gpu(GpuDataParallelPool::new(
model_config.clone(),
config.data_parallel_workers,
)?)))
}
#[cfg(not(feature = "gpu"))]
{
Err(TrainerError::InvalidData(
"GPU data-parallel requires the `gpu` feature".into(),
))
}
}
}
}
#[cfg(feature = "training")]
fn validate_data_parallel(
config: &TrainerConfig,
backend: &TrainingBackend,
) -> Result<(), TrainerError> {
if config.data_parallel_workers == 0 {
return Err(TrainerError::InvalidData(
"data-parallel workers must be positive".into(),
));
}
if config.data_parallel_workers == 1 {
return Ok(());
}
if matches!(backend, TrainingBackend::Gpu(_)) {
#[cfg(not(feature = "gpu"))]
{
return Err(TrainerError::InvalidData(
"GPU data-parallel requires the `gpu` feature".into(),
));
}
#[cfg(feature = "gpu")]
if config.low_memory.layer_offload {
return Err(TrainerError::InvalidData(
"GPU data-parallel does not support layer_offload; disable layer offload or use a single GPU worker"
.into(),
));
}
}
if config.batch_size < config.data_parallel_workers {
return Err(TrainerError::InvalidData(format!(
"batch size {} must be >= data-parallel workers {}",
config.batch_size, config.data_parallel_workers
)));
}
if config.batch_size % config.data_parallel_workers != 0 {
return Err(TrainerError::InvalidData(format!(
"batch size {} must be divisible by data-parallel workers {}",
config.batch_size, config.data_parallel_workers
)));
}
let _ = backend;
Ok(())
}
#[cfg(feature = "training")]
fn validate_training_inputs(
model: &CausalLM,
tokenizer: &BpeTokenizer,
training_data: &TokenDataset,
schedule: &LearningRateSchedule,
config: &TrainerConfig,
) -> Result<(), TrainerError> {
if model.config.vocab_size != tokenizer.vocab_size() {
return Err(TrainerError::InvalidData(
"tokenizer and model vocabulary sizes differ".into(),
));
}
if config.batch_size == 0 || config.sequence_length == 0 {
return Err(TrainerError::InvalidData(
"batch size and sequence length must be positive".into(),
));
}
if config.gradient_accumulation_steps == 0 {
return Err(TrainerError::InvalidData(
"gradient accumulation steps must be positive".into(),
));
}
if config.data_parallel_workers == 0 {
return Err(TrainerError::InvalidData(
"data-parallel workers must be positive".into(),
));
}
config.mixed_precision.validate()?;
if config.sequence_length > model.config.max_sequence_length {
return Err(TrainerError::InvalidData(
"trainer sequence length exceeds model context".into(),
));
}
if training_data.token_count() <= config.sequence_length {
return Err(TrainerError::InvalidData(
"corpus is too short for one training sequence".into(),
));
}
training_data.validate_token_ids(model.config.vocab_size)?;
if schedule.total_steps == 0
|| schedule.base_learning_rate <= 0.0
|| schedule.minimum_learning_rate < 0.0
{
return Err(TrainerError::InvalidData(
"learning-rate schedule is invalid".into(),
));
}
Ok(())
}
#[cfg(all(test, feature = "training"))]
mod tests {
use super::{
load_checkpoint, training_metrics_jsonl_line, LearningRateSchedule, LowMemoryProfile,
PackedSequenceSampler, Trainer, TrainerConfig, TrainerError, TrainingBackend,
TrainingMetrics,
};
use crate::amp::MixedPrecisionConfig;
use crate::model::{CausalLM, CausalLmConfig};
use crate::optimizer::AdamWConfig;
use crate::tokenizer::{BpeTokenizer, FIRST_MERGE_ID};
use std::time::{SystemTime, UNIX_EPOCH};
#[test]
fn packed_sampler_shuffles_non_overlapping_sequences_deterministically() {
let mut first = PackedSequenceSampler::new(33, 4, 17, 0);
let mut second = PackedSequenceSampler::new(33, 4, 17, 0);
let first_starts = (0..8).map(|_| first.next_start()).collect::<Vec<_>>();
let second_starts = (0..8).map(|_| second.next_start()).collect::<Vec<_>>();
let mut sorted_starts = first_starts.clone();
sorted_starts.sort_unstable();
assert_eq!(first_starts, second_starts);
assert_ne!(first_starts, sorted_starts);
assert_eq!(sorted_starts, vec![0, 4, 8, 12, 16, 20, 24, 28]);
}
#[test]
fn packed_sampler_resumes_at_exact_sample() {
let mut uninterrupted = PackedSequenceSampler::new(41, 4, 23, 0);
for _ in 0..13 {
uninterrupted.next_start();
}
let expected = uninterrupted.next_start();
let mut resumed = PackedSequenceSampler::new(41, 4, 23, 13);
assert_eq!(resumed.next_start(), expected);
}
#[test]
fn trainer_refuses_obvious_memory_budget_overflow() {
let text = "abababababababababababababababababababab";
let tokenizer = BpeTokenizer::train_from_bytes(text.as_bytes(), FIRST_MERGE_ID).unwrap();
let tokens = tokenizer.encode(text);
let model = CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 4,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 3,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
let result = Trainer::new_with_memory_budget(
model,
tokenizer,
tokens,
AdamWConfig::default(),
LearningRateSchedule {
base_learning_rate: 0.02,
minimum_learning_rate: 0.02,
warmup_steps: 0,
total_steps: 10,
},
TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
},
TrainingBackend::Cpu,
Some(1024),
);
assert!(matches!(result, Err(TrainerError::MemoryBudget(_))));
}
#[test]
fn tiny_corpus_loss_decreases_over_ten_steps() {
let text = "abababababababababababababababababababab";
let tokenizer = BpeTokenizer::train_from_bytes(text.as_bytes(), FIRST_MERGE_ID).unwrap();
let model = CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 4,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 3,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
let mut trainer = Trainer::from_text(
model,
tokenizer,
text,
AdamWConfig {
weight_decay: 0.0,
..Default::default()
},
LearningRateSchedule {
base_learning_rate: 0.02,
minimum_learning_rate: 0.02,
warmup_steps: 0,
total_steps: 10,
},
TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 0,
gradient_accumulation_steps: 1,
mixed_precision: MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
},
TrainingBackend::Cpu,
)
.unwrap();
let metrics = trainer.train_steps(10).unwrap();
assert!(metrics.last().unwrap().loss < metrics.first().unwrap().loss);
}
#[test]
fn training_metrics_jsonl_line_has_plot_fields() {
let metrics = TrainingMetrics {
step: 3,
micro_batches: 1,
micro_batches_processed: 3,
loss: 1.25,
perplexity: 3.490_342_9,
learning_rate: 3e-4,
gradient_norm: 0.5,
tokens_per_second: 100.0,
loss_scale: 1024.0,
overflow: false,
optimizer_stepped: true,
};
let line = training_metrics_jsonl_line(&metrics, Some(4096));
let value: serde_json::Value = serde_json::from_str(&line).unwrap();
assert_eq!(value["step"], 3);
assert_eq!(value["loss"], 1.25);
assert!((value["ppl"].as_f64().unwrap() - metrics.perplexity as f64).abs() < 1e-5);
assert!((value["lr"].as_f64().unwrap() - 3e-4).abs() < 1e-9);
assert_eq!(value["grad_norm"], 0.5);
assert_eq!(value["loss_scale"], 1024.0);
assert_eq!(value["rss"], 4096);
let without_rss = training_metrics_jsonl_line(&metrics, None);
let value: serde_json::Value = serde_json::from_str(&without_rss).unwrap();
assert!(value["rss"].is_null());
}
#[test]
fn metrics_jsonl_export_writes_one_line_per_step() {
let text = "abababababababababababababababababababab";
let tokenizer = BpeTokenizer::train_from_bytes(text.as_bytes(), FIRST_MERGE_ID).unwrap();
let model = CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 4,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 3,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
let mut trainer = Trainer::from_text(
model,
tokenizer,
text,
AdamWConfig {
weight_decay: 0.0,
..Default::default()
},
LearningRateSchedule {
base_learning_rate: 0.02,
minimum_learning_rate: 0.02,
warmup_steps: 0,
total_steps: 2,
},
TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 0,
gradient_accumulation_steps: 1,
mixed_precision: MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
},
TrainingBackend::Cpu,
)
.unwrap();
let path = std::env::temp_dir().join(format!(
"oxidelm-metrics-{}-{}.jsonl",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
trainer.configure_metrics_jsonl(&path).unwrap();
let metrics = trainer.train_steps(2).unwrap();
drop(trainer);
let text = std::fs::read_to_string(&path).unwrap();
let _ = std::fs::remove_file(&path);
let lines: Vec<_> = text.lines().filter(|line| !line.is_empty()).collect();
assert_eq!(lines.len(), metrics.len());
for (line, current) in lines.iter().zip(metrics.iter()) {
let value: serde_json::Value = serde_json::from_str(line).unwrap();
assert_eq!(value["step"], current.step);
assert!((value["loss"].as_f64().unwrap() - current.loss as f64).abs() < 1e-5);
assert!((value["ppl"].as_f64().unwrap() - current.perplexity as f64).abs() < 1e-5);
assert!((value["lr"].as_f64().unwrap() - current.learning_rate as f64).abs() < 1e-9);
assert!(
(value["grad_norm"].as_f64().unwrap() - current.gradient_norm as f64).abs() < 1e-5
);
assert_eq!(value["loss_scale"], 1.0);
assert!(value.get("rss").is_some());
}
}
#[test]
fn data_parallel_matches_single_worker_step() {
let text = "abababababababababababababababababababababababababababababababab";
let tokenizer = BpeTokenizer::train_from_bytes(text.as_bytes(), FIRST_MERGE_ID).unwrap();
let config = CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 4,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 11,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
};
let tokens = tokenizer.encode(text);
let schedule = LearningRateSchedule {
base_learning_rate: 0.01,
minimum_learning_rate: 0.01,
warmup_steps: 0,
total_steps: 4,
};
let optimizer = AdamWConfig {
weight_decay: 0.0,
..Default::default()
};
let single_config = TrainerConfig {
batch_size: 4,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 0,
gradient_accumulation_steps: 1,
mixed_precision: MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
let mut parallel_config = single_config.clone();
parallel_config.data_parallel_workers = 2;
let mut single = Trainer::new(
CausalLM::new(config.clone()).unwrap(),
tokenizer.clone(),
tokens.clone(),
optimizer.clone(),
schedule.clone(),
single_config,
TrainingBackend::Cpu,
)
.unwrap();
let mut parallel = Trainer::new(
CausalLM::new(config).unwrap(),
tokenizer,
tokens,
optimizer,
schedule,
parallel_config,
TrainingBackend::Cpu,
)
.unwrap();
let single_metrics = single.train_steps(3).unwrap();
let parallel_metrics = parallel.train_steps(3).unwrap();
assert_eq!(single_metrics.len(), parallel_metrics.len());
for (left, right) in single_metrics.iter().zip(parallel_metrics.iter()) {
assert_eq!(left.step, right.step);
assert!(
(left.loss - right.loss).abs() < 1e-4,
"loss {} vs {}",
left.loss,
right.loss
);
assert!(
(left.gradient_norm - right.gradient_norm).abs() < 1e-3,
"grad {} vs {}",
left.gradient_norm,
right.gradient_norm
);
}
let single_params = parameter_snapshot(&single);
let parallel_params = parameter_snapshot(¶llel);
for ((left_name, left), (right_name, right)) in
single_params.iter().zip(parallel_params.iter())
{
assert_eq!(left_name, right_name);
for (a, b) in left.iter().zip(right.iter()) {
assert!((a - b).abs() < 1e-4, "{left_name}: {a} vs {b}");
}
}
}
#[test]
fn gradient_accumulation_advances_micro_batches_once_per_optimizer_step() {
let text = "abababababababababababababababababababababababab";
let tokenizer = BpeTokenizer::train_from_bytes(text.as_bytes(), FIRST_MERGE_ID).unwrap();
let model = CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 4,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 3,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
let mut trainer = Trainer::from_text(
model,
tokenizer,
text,
AdamWConfig {
weight_decay: 0.0,
..Default::default()
},
LearningRateSchedule {
base_learning_rate: 0.02,
minimum_learning_rate: 0.02,
warmup_steps: 0,
total_steps: 10,
},
TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 0,
gradient_accumulation_steps: 3,
mixed_precision: MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
},
TrainingBackend::Cpu,
)
.unwrap();
let before_cursor = trainer.cursor();
let metrics = trainer.train_steps(2).unwrap();
assert_eq!(metrics.len(), 2);
assert_eq!(metrics[0].step, 1);
assert_eq!(metrics[0].micro_batches, 3);
assert_eq!(metrics[0].micro_batches_processed, 3);
assert_eq!(metrics[1].step, 2);
assert_eq!(metrics[1].micro_batches, 3);
assert_eq!(metrics[1].micro_batches_processed, 6);
assert_eq!(trainer.step(), 2);
assert_eq!(trainer.optimizer.state().step, 2);
assert_eq!(trainer.micro_batches_processed(), 6);
assert_eq!(trainer.cursor(), before_cursor + 6);
assert_eq!(trainer.tokens_processed(), 2 * 3 * 1 * 4);
}
#[test]
fn from_checkpoint_restores_optimizer_schedule_cursor_and_step() {
let text = "abababababababababababababababababababab";
let tokenizer = BpeTokenizer::train_from_bytes(text.as_bytes(), FIRST_MERGE_ID).unwrap();
let tokens = tokenizer.encode(text);
let model = CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 4,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 3,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
let mut trainer = Trainer::new(
model,
tokenizer,
tokens.clone(),
AdamWConfig {
weight_decay: 0.0,
..Default::default()
},
LearningRateSchedule {
base_learning_rate: 0.02,
minimum_learning_rate: 0.002,
warmup_steps: 2,
total_steps: 10,
},
TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 0,
gradient_accumulation_steps: 1,
mixed_precision: MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
},
TrainingBackend::Cpu,
)
.unwrap();
trainer.train_steps(3).unwrap();
let step = trainer.step();
let cursor = trainer.cursor();
let tokens_processed = trainer.tokens_processed();
let optimizer_step = trainer.optimizer.state().step;
let learning_rate = trainer.schedule.learning_rate(step);
let path = std::env::temp_dir().join(format!(
"oxidelm-resume-{}.bin",
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
trainer.save_checkpoint(&path).unwrap();
let checkpoint = load_checkpoint(&path).unwrap();
let mut resumed =
Trainer::from_checkpoint(checkpoint, tokens, TrainingBackend::Cpu).unwrap();
let _ = std::fs::remove_file(&path);
assert_eq!(resumed.step(), step);
assert_eq!(resumed.cursor(), cursor);
assert_eq!(resumed.tokens_processed(), tokens_processed);
assert_eq!(resumed.optimizer.state().step, optimizer_step);
assert_eq!(resumed.schedule.warmup_steps, 2);
assert_eq!(resumed.schedule.total_steps, 10);
assert_eq!(resumed.schedule.learning_rate(step), learning_rate);
assert_eq!(resumed.model.config.seed, 3);
let (expected_inputs, expected_targets) = trainer.next_batch().unwrap();
let (resumed_inputs, resumed_targets) = resumed.next_batch().unwrap();
assert_eq!(resumed_inputs.data(), expected_inputs.data());
assert_eq!(resumed_targets.data(), expected_targets.data());
}
fn tiny_resume_fixture(
seed: u64,
attention_dropout: f32,
residual_dropout: f32,
embedding_dropout: f32,
gradient_accumulation_steps: usize,
) -> (Trainer, Vec<usize>) {
let text = "abababababababababababababababababababababababab";
let tokenizer = BpeTokenizer::train_from_bytes(text.as_bytes(), FIRST_MERGE_ID).unwrap();
let tokens = tokenizer.encode(text);
let model = CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 4,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout,
residual_dropout,
embedding_dropout,
use_bias: false,
})
.unwrap();
let trainer = Trainer::new(
model,
tokenizer,
tokens.clone(),
AdamWConfig {
weight_decay: 0.0,
..Default::default()
},
LearningRateSchedule {
base_learning_rate: 0.02,
minimum_learning_rate: 0.002,
warmup_steps: 2,
total_steps: 20,
},
TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 0,
gradient_accumulation_steps,
mixed_precision: MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
},
TrainingBackend::Cpu,
)
.unwrap();
(trainer, tokens)
}
fn parameter_snapshot(trainer: &Trainer) -> Vec<(String, Vec<f32>)> {
trainer
.model
.named_parameters()
.into_iter()
.map(|(name, parameter)| (name, parameter.data()))
.collect()
}
fn assert_resume_matches_continuous(
seed: u64,
attention_dropout: f32,
residual_dropout: f32,
embedding_dropout: f32,
gradient_accumulation_steps: usize,
warm_steps: usize,
continue_steps: usize,
label: &str,
) {
let (mut continuous, _) = tiny_resume_fixture(
seed,
attention_dropout,
residual_dropout,
embedding_dropout,
gradient_accumulation_steps,
);
continuous.train_steps(warm_steps).unwrap();
let continuous_tail = continuous.train_steps(continue_steps).unwrap();
let continuous_params = parameter_snapshot(&continuous);
let (mut interrupted, tokens) = tiny_resume_fixture(
seed,
attention_dropout,
residual_dropout,
embedding_dropout,
gradient_accumulation_steps,
);
interrupted.train_steps(warm_steps).unwrap();
let path = std::env::temp_dir().join(format!(
"oxidelm-resume-eq-{label}-{}.bin",
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
interrupted.save_checkpoint(&path).unwrap();
let checkpoint = load_checkpoint(&path).unwrap();
let mut resumed =
Trainer::from_checkpoint(checkpoint, tokens, TrainingBackend::Cpu).unwrap();
let _ = std::fs::remove_file(&path);
assert_eq!(resumed.step(), warm_steps, "{label}: step after reload");
assert_eq!(
resumed.micro_batches_processed(),
interrupted.micro_batches_processed(),
"{label}: micro-batch counter after reload"
);
assert_eq!(resumed.cursor(), interrupted.cursor(), "{label}: cursor");
let resumed_tail = resumed.train_steps(continue_steps).unwrap();
assert_eq!(resumed_tail.len(), continuous_tail.len());
for (continuous_metrics, resumed_metrics) in
continuous_tail.iter().zip(resumed_tail.iter())
{
assert_eq!(
resumed_metrics.step, continuous_metrics.step,
"{label}: step"
);
assert_eq!(
resumed_metrics.loss.to_bits(),
continuous_metrics.loss.to_bits(),
"{label}: loss at step {}",
continuous_metrics.step
);
assert_eq!(
resumed_metrics.learning_rate.to_bits(),
continuous_metrics.learning_rate.to_bits(),
"{label}: learning rate at step {}",
continuous_metrics.step
);
assert_eq!(
resumed_metrics.gradient_norm.to_bits(),
continuous_metrics.gradient_norm.to_bits(),
"{label}: grad norm at step {}",
continuous_metrics.step
);
}
let resumed_params = parameter_snapshot(&resumed);
assert_eq!(resumed_params.len(), continuous_params.len());
for ((continuous_name, continuous_values), (resumed_name, resumed_values)) in
continuous_params.iter().zip(resumed_params.iter())
{
assert_eq!(resumed_name, continuous_name);
assert_eq!(
resumed_values, continuous_values,
"{label}: parameter {continuous_name}"
);
}
}
#[test]
fn resume_matches_continuous_training_losses_and_parameters() {
assert_resume_matches_continuous(11, 0.0, 0.0, 0.0, 1, 3, 4, "plain");
assert_resume_matches_continuous(13, 0.1, 0.05, 0.05, 2, 3, 3, "dropout-accum");
}
#[test]
fn resume_restores_amp_scaler_checkpoint_interval_and_offload() {
use crate::amp::LossScalerState;
use crate::optimizer::MomentDtype;
use std::time::{SystemTime, UNIX_EPOCH};
let text = "abababababababababababababababababababababababab";
let tokenizer = BpeTokenizer::train_from_bytes(text.as_bytes(), FIRST_MERGE_ID).unwrap();
let tokens = tokenizer.encode(text);
let model = CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 2,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 4,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 21,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
let mut amp = MixedPrecisionConfig::enabled_fp16();
amp.initial_loss_scale = 1_024.0;
amp.growth_interval = 1;
amp.growth_factor = 2.0;
amp.min_loss_scale = 1.0;
amp.max_loss_scale = 65_536.0;
let mut trainer = Trainer::new(
model,
tokenizer,
tokens.clone(),
AdamWConfig {
weight_decay: 0.0,
moment_dtype: MomentDtype::Fp16,
offload_moments: true,
..Default::default()
},
LearningRateSchedule {
base_learning_rate: 0.02,
minimum_learning_rate: 0.002,
warmup_steps: 0,
total_steps: 20,
},
TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 0,
gradient_accumulation_steps: 1,
mixed_precision: amp,
activation_checkpointing: true,
data_parallel_workers: 1,
low_memory: LowMemoryProfile {
activation_checkpoint_interval: 2,
layer_offload: true,
trim_caches_each_microbatch: true,
..Default::default()
},
},
TrainingBackend::Cpu,
)
.unwrap();
for _ in 0..3 {
let _ = trainer.train_step().unwrap();
}
let scale_before = trainer.loss_scale();
let scaler_before = trainer.loss_scaler_state().expect("AMP scaler state");
assert!(scale_before > 1_024.0, "scaler should grow across steps");
let path = std::env::temp_dir().join(format!(
"oxidelm-resume-lowram-{}.bin",
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
trainer.save_checkpoint(&path).unwrap();
let checkpoint = load_checkpoint(&path).unwrap();
let _ = std::fs::remove_file(&path);
assert_eq!(
checkpoint.loss_scaler.as_ref(),
Some(&LossScalerState {
scale: scaler_before.scale,
consecutive_successful_steps: scaler_before.consecutive_successful_steps,
})
);
assert_eq!(
checkpoint
.trainer_config
.low_memory
.activation_checkpoint_interval,
2
);
assert!(checkpoint.trainer_config.low_memory.layer_offload);
assert!(checkpoint.trainer_config.low_memory.trim_caches_each_microbatch);
assert!(checkpoint.optimizer.config().offload_moments);
assert_eq!(checkpoint.optimizer.config().moment_dtype, MomentDtype::Fp16);
let resumed =
Trainer::from_checkpoint(checkpoint, tokens, TrainingBackend::Cpu).unwrap();
assert_eq!(resumed.loss_scale(), scale_before);
assert_eq!(
resumed.loss_scaler_state().as_ref(),
Some(&scaler_before)
);
assert_eq!(
resumed.config.low_memory.activation_checkpoint_interval,
2
);
assert!(resumed.config.low_memory.layer_offload);
assert!(resumed.config.low_memory.trim_caches_each_microbatch);
assert!(resumed.optimizer.config().offload_moments);
assert!(resumed.config.activation_checkpointing);
assert!(resumed.config.mixed_precision.enabled);
}
#[test]
fn mixed_precision_recovers_from_loss_scale_overflow() {
let text = "abababababababababababababababababababababababab";
let tokenizer = BpeTokenizer::train_from_bytes(text.as_bytes(), FIRST_MERGE_ID).unwrap();
let model = CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 4,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 3,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
let mut amp = MixedPrecisionConfig::enabled_fp16();
amp.initial_loss_scale = 1.0e12;
amp.growth_interval = 1_000;
amp.min_loss_scale = 1.0;
amp.max_loss_scale = 1.0e12;
amp.backoff_factor = 0.5;
let mut trainer = Trainer::from_text(
model,
tokenizer,
text,
AdamWConfig {
weight_decay: 0.0,
..Default::default()
},
LearningRateSchedule {
base_learning_rate: 0.02,
minimum_learning_rate: 0.02,
warmup_steps: 0,
total_steps: 10,
},
TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 0,
gradient_accumulation_steps: 1,
mixed_precision: amp,
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
},
TrainingBackend::Cpu,
)
.unwrap();
let masters_before = trainer
.master_weights
.as_ref()
.unwrap()
.iter()
.map(|values| values.clone())
.collect::<Vec<_>>();
let scale_before = trainer.loss_scale();
let metrics = trainer.train_step().unwrap();
assert!(metrics.overflow);
assert!(!metrics.optimizer_stepped);
assert_eq!(trainer.optimizer.state().step, 0);
assert!(trainer.loss_scale() < scale_before);
assert_eq!(
trainer.master_weights.as_ref().unwrap(),
&masters_before,
"overflow must leave FP32 masters unchanged"
);
let mut recovered = false;
for _ in 0..80 {
let metrics = trainer.train_step().unwrap();
if metrics.optimizer_stepped {
recovered = true;
break;
}
}
assert!(recovered, "loss scaler should recover after backoff");
assert_eq!(trainer.optimizer.state().step, 1);
assert!(trainer.loss_scale().is_finite() && trainer.loss_scale() >= 1.0);
}
#[test]
fn early_stopping_uses_validation_loss_and_keeps_best_checkpoint() {
let text = "abababababababababababababababababababababababab";
let tokenizer = BpeTokenizer::train_from_bytes(text.as_bytes(), FIRST_MERGE_ID).unwrap();
let tokens = tokenizer.encode(text);
let validation = tokens.clone();
let model = CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 4,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 3,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
let trainer = Trainer::new(
model,
tokenizer,
tokens.clone(),
AdamWConfig {
weight_decay: 0.0,
..Default::default()
},
LearningRateSchedule {
base_learning_rate: 0.02,
minimum_learning_rate: 0.02,
warmup_steps: 0,
total_steps: 20,
},
TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 0,
gradient_accumulation_steps: 1,
mixed_precision: MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
},
TrainingBackend::Cpu,
)
.unwrap();
let best_path = std::env::temp_dir().join(format!(
"oxidelm-best-{}.bin",
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let seed_path = std::env::temp_dir().join(format!(
"oxidelm-early-seed-{}.bin",
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
trainer.save_checkpoint(&seed_path).unwrap();
let mut checkpoint = load_checkpoint(&seed_path).unwrap();
checkpoint.best_validation_loss = Some(0.0);
let mut trainer =
Trainer::from_checkpoint(checkpoint, tokens, TrainingBackend::Cpu).unwrap();
let _ = std::fs::remove_file(&seed_path);
trainer
.configure_evaluation(validation, 1, &best_path)
.unwrap();
trainer.configure_early_stopping(2, 0.0).unwrap();
let metrics = trainer.train_steps(10).unwrap();
assert_eq!(metrics.len(), 2);
assert!(trainer.stopped_early());
assert_eq!(trainer.evaluations_without_improvement(), 2);
assert_eq!(trainer.best_validation_loss(), Some(0.0));
assert!(!best_path.exists());
let _ = std::fs::remove_file(&best_path);
}
#[test]
fn best_checkpoint_updates_only_on_validation_improvement() {
let text = "abababababababababababababababababababababababab";
let tokenizer = BpeTokenizer::train_from_bytes(text.as_bytes(), FIRST_MERGE_ID).unwrap();
let tokens = tokenizer.encode(text);
let validation = tokens.clone();
let model = CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 4,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 3,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
let mut trainer = Trainer::new(
model,
tokenizer,
tokens.clone(),
AdamWConfig {
weight_decay: 0.0,
..Default::default()
},
LearningRateSchedule {
base_learning_rate: 0.02,
minimum_learning_rate: 0.02,
warmup_steps: 0,
total_steps: 20,
},
TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 0,
gradient_accumulation_steps: 1,
mixed_precision: MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
},
TrainingBackend::Cpu,
)
.unwrap();
let best_path = std::env::temp_dir().join(format!(
"oxidelm-best-retain-{}.bin",
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
trainer
.configure_evaluation(validation.clone(), 1, &best_path)
.unwrap();
trainer.configure_early_stopping(0, 0.0).unwrap();
let first = trainer.evaluate_and_save_best().unwrap();
assert!(first.improved);
assert!(best_path.exists());
let first_mtime = std::fs::metadata(&best_path).unwrap().modified().unwrap();
let first_best = first.best_loss;
let seed_path = std::env::temp_dir().join(format!(
"oxidelm-best-seed-{}.bin",
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
trainer.save_checkpoint(&seed_path).unwrap();
let mut checkpoint = load_checkpoint(&seed_path).unwrap();
checkpoint.best_validation_loss = Some(first_best - 1.0);
let mut trainer =
Trainer::from_checkpoint(checkpoint, tokens, TrainingBackend::Cpu).unwrap();
let _ = std::fs::remove_file(&seed_path);
trainer
.configure_evaluation(validation, 1, &best_path)
.unwrap();
let second = trainer.evaluate_and_save_best().unwrap();
assert!(!second.improved);
assert_eq!(second.evaluations_without_improvement, 1);
let second_mtime = std::fs::metadata(&best_path).unwrap().modified().unwrap();
assert_eq!(first_mtime, second_mtime);
assert_eq!(trainer.best_validation_loss(), Some(first_best - 1.0));
let _ = std::fs::remove_file(&best_path);
}
}