use crate::amp::{AmpFormat, MixedPrecisionConfig};
use crate::memory::{
apply_gpu_parameter_residency, check_memory_budget, estimate_training_memory_detailed,
host_memory_is_tiny, DeviceBudget, MemoryBudgetError,
MemoryEstimate,
};
use crate::model::{
ArchitecturePreset, ArchitectureSpec, CausalLmConfig, ModelConfigError, DEFAULT_INITIALIZER_STD,
};
use crate::optimizer::{AdamWConfig, MomentDtype, OptimizerError, OptimizerKind};
use crate::peft::LoraConfig;
use crate::qat::QatConfig;
use crate::rope::RopeScaling;
use crate::tokenizer::{PreTokenization, FIRST_MERGE_ID};
use crate::trainer::{
LearningRateSchedule, LowMemoryProfile, SelectiveCheckpoint, TrainerConfig, TrainingBackend,
};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::error::Error;
use std::fmt;
use std::fs::File;
use std::io::{BufReader, Write};
use std::path::{Path, PathBuf};
pub fn default_hidden_size() -> usize {
64
}
pub fn default_num_layers() -> usize {
2
}
pub fn default_num_heads() -> usize {
4
}
pub fn default_intermediate_size() -> usize {
128
}
pub fn default_sequence_length() -> usize {
32
}
pub fn default_rms_norm_epsilon() -> f32 {
1e-5
}
pub fn default_rope_theta() -> f32 {
10_000.0
}
pub fn default_seed() -> u64 {
42
}
pub fn default_initializer_std() -> f32 {
DEFAULT_INITIALIZER_STD
}
pub fn default_tie_word_embeddings() -> bool {
true
}
pub fn default_dropout() -> f32 {
0.0
}
pub fn default_vocab_size() -> usize {
FIRST_MERGE_ID + 256
}
pub fn default_min_frequency() -> usize {
2
}
pub fn default_batch_size() -> usize {
1
}
pub fn default_gradient_accumulation_steps() -> usize {
1
}
pub fn default_activation_checkpointing() -> bool {
false
}
pub fn default_data_parallel_workers() -> usize {
1
}
pub fn default_max_gradient_norm() -> f32 {
1.0
}
pub fn default_log_every() -> usize {
1
}
pub fn default_steps() -> usize {
100
}
pub fn default_validation_split() -> f64 {
0.1
}
pub fn default_early_stopping_patience() -> usize {
0
}
pub fn default_early_stopping_min_delta() -> f32 {
0.0
}
pub fn default_shard_tokens() -> usize {
1_048_576
}
pub fn default_tokenize_chunk_bytes() -> usize {
1_048_576
}
pub fn default_prefetch_batches() -> usize {
2
}
pub fn default_base_learning_rate() -> f32 {
3e-4
}
pub fn default_minimum_learning_rate() -> f32 {
3e-5
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct TrainConfigFile {
#[serde(default)]
pub model: ModelSection,
#[serde(default)]
pub tokenizer: TokenizerSection,
#[serde(default)]
pub trainer: TrainerSection,
#[serde(default)]
pub optimizer: OptimizerSection,
#[serde(default)]
pub schedule: ScheduleSection,
#[serde(default)]
pub data: DataSection,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ModelSection {
pub architecture: Option<ArchitecturePreset>,
pub hidden_size: Option<usize>,
pub num_layers: Option<usize>,
pub num_heads: Option<usize>,
pub num_key_value_heads: Option<usize>,
pub intermediate_size: Option<usize>,
pub max_sequence_length: Option<usize>,
pub rms_norm_epsilon: Option<f32>,
pub rope_theta: Option<f32>,
pub rope_scaling: Option<RopeScaling>,
pub seed: Option<u64>,
pub initializer_std: Option<f32>,
pub tie_word_embeddings: Option<bool>,
pub attention_dropout: Option<f32>,
pub residual_dropout: Option<f32>,
pub embedding_dropout: Option<f32>,
pub use_bias: Option<bool>,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct TokenizerSection {
pub vocab_size: Option<usize>,
pub min_frequency: Option<usize>,
pub pre_tokenization: Option<PreTokenization>,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct TrainerSection {
pub batch_size: Option<usize>,
pub sequence_length: Option<usize>,
pub max_gradient_norm: Option<f32>,
pub log_every: Option<usize>,
pub gradient_accumulation_steps: Option<usize>,
pub activation_checkpointing: Option<bool>,
pub data_parallel_workers: Option<usize>,
pub mixed_precision: Option<bool>,
pub amp_format: Option<AmpFormat>,
pub initial_loss_scale: Option<f32>,
pub loss_scale_growth_factor: Option<f32>,
pub loss_scale_backoff_factor: Option<f32>,
pub loss_scale_growth_interval: Option<usize>,
pub activation_checkpoint_interval: Option<usize>,
pub selective_checkpoint: Option<SelectiveCheckpoint>,
pub attention_chunk_size: Option<usize>,
pub layer_offload: Option<bool>,
pub trim_caches_each_microbatch: Option<bool>,
pub lora: Option<LoraConfig>,
pub freeze: Option<Vec<String>>,
pub qat: Option<QatConfig>,
pub rebuild_token_cache: Option<bool>,
pub metrics_jsonl: Option<PathBuf>,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct OptimizerSection {
pub beta1: Option<f32>,
pub beta2: Option<f32>,
pub epsilon: Option<f32>,
pub weight_decay: Option<f32>,
pub no_decay_rms_norm: Option<bool>,
pub no_decay_embeddings: Option<Vec<String>>,
pub rms_norm_lr_scale: Option<f32>,
pub embedding_lr_scale: Option<f32>,
pub kind: Option<OptimizerKind>,
pub moment_dtype: Option<MomentDtype>,
pub offload_moments: Option<bool>,
pub momentum: Option<f32>,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ScheduleSection {
pub base_learning_rate: Option<f32>,
pub minimum_learning_rate: Option<f32>,
pub warmup_steps: Option<usize>,
pub total_steps: Option<usize>,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct DataSection {
pub validation_split: Option<f64>,
pub eval_every: Option<usize>,
pub early_stopping_patience: Option<usize>,
pub early_stopping_min_delta: Option<f32>,
pub shard_tokens: Option<usize>,
pub tokenize_chunk_bytes: Option<usize>,
pub prefetch_batches: Option<usize>,
}
#[derive(Clone, Debug, Default)]
pub struct TrainConfigOverrides {
pub architecture: Option<ArchitecturePreset>,
pub hidden_size: Option<usize>,
pub num_layers: Option<usize>,
pub num_heads: Option<usize>,
pub num_key_value_heads: Option<usize>,
pub intermediate_size: Option<usize>,
pub max_sequence_length: Option<usize>,
pub sequence_length: Option<usize>,
pub rms_norm_epsilon: Option<f32>,
pub rope_theta: Option<f32>,
pub rope_scaling: Option<RopeScaling>,
pub seed: Option<u64>,
pub initializer_std: Option<f32>,
pub tie_word_embeddings: Option<bool>,
pub attention_dropout: Option<f32>,
pub residual_dropout: Option<f32>,
pub embedding_dropout: Option<f32>,
pub use_bias: Option<bool>,
pub vocab_size: Option<usize>,
pub min_frequency: Option<usize>,
pub pre_tokenization: Option<PreTokenization>,
pub batch_size: Option<usize>,
pub max_gradient_norm: Option<f32>,
pub log_every: Option<usize>,
pub gradient_accumulation_steps: Option<usize>,
pub activation_checkpointing: Option<bool>,
pub data_parallel_workers: Option<usize>,
pub mixed_precision: Option<bool>,
pub amp_format: Option<AmpFormat>,
pub initial_loss_scale: Option<f32>,
pub loss_scale_growth_factor: Option<f32>,
pub loss_scale_backoff_factor: Option<f32>,
pub loss_scale_growth_interval: Option<usize>,
pub beta1: Option<f32>,
pub beta2: Option<f32>,
pub epsilon: Option<f32>,
pub weight_decay: Option<f32>,
pub no_decay_rms_norm: Option<bool>,
pub no_decay_embeddings: Option<Vec<String>>,
pub rms_norm_lr_scale: Option<f32>,
pub embedding_lr_scale: Option<f32>,
pub optimizer_kind: Option<OptimizerKind>,
pub moment_dtype: Option<MomentDtype>,
pub offload_moments: Option<bool>,
pub momentum: Option<f32>,
pub base_learning_rate: Option<f32>,
pub minimum_learning_rate: Option<f32>,
pub warmup_steps: Option<usize>,
pub steps: Option<usize>,
pub validation_split: Option<f64>,
pub eval_every: Option<usize>,
pub early_stopping_patience: Option<usize>,
pub early_stopping_min_delta: Option<f32>,
pub shard_tokens: Option<usize>,
pub tokenize_chunk_bytes: Option<usize>,
pub prefetch_batches: Option<usize>,
pub activation_checkpoint_interval: Option<usize>,
pub selective_checkpoint: Option<SelectiveCheckpoint>,
pub attention_chunk_size: Option<usize>,
pub layer_offload: Option<bool>,
pub trim_caches_each_microbatch: Option<bool>,
pub lora: Option<LoraConfig>,
pub freeze: Option<Vec<String>>,
pub qat: Option<QatConfig>,
pub rebuild_token_cache: Option<bool>,
pub metrics_jsonl: Option<PathBuf>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ResolvedTrainConfig {
pub architecture: Option<ArchitecturePreset>,
pub hidden_size: usize,
pub num_layers: usize,
pub num_heads: usize,
pub num_key_value_heads: usize,
pub intermediate_size: usize,
pub max_sequence_length: usize,
pub sequence_length: usize,
pub rms_norm_epsilon: f32,
pub rope_theta: f32,
pub rope_scaling: Option<RopeScaling>,
pub seed: u64,
pub initializer_std: f32,
pub tie_word_embeddings: bool,
pub attention_dropout: f32,
pub residual_dropout: f32,
pub embedding_dropout: f32,
pub use_bias: bool,
pub vocab_size: usize,
pub min_frequency: usize,
pub pre_tokenization: PreTokenization,
pub batch_size: usize,
pub max_gradient_norm: f32,
pub log_every: usize,
pub gradient_accumulation_steps: usize,
pub activation_checkpointing: bool,
pub data_parallel_workers: usize,
pub mixed_precision: MixedPrecisionConfig,
pub optimizer: AdamWConfig,
pub base_learning_rate: f32,
pub minimum_learning_rate: f32,
pub warmup_steps: usize,
pub steps: usize,
pub validation_split: f64,
pub eval_every: usize,
pub early_stopping_patience: usize,
pub early_stopping_min_delta: f32,
pub shard_tokens: usize,
pub tokenize_chunk_bytes: usize,
pub prefetch_batches: usize,
pub low_memory: LowMemoryProfile,
pub metrics_jsonl: Option<PathBuf>,
}
#[derive(Debug)]
pub enum ConfigError {
Io(std::io::Error),
Json(serde_json::Error),
Invalid(String),
Model(ModelConfigError),
Optimizer(OptimizerError),
MemoryBudget(MemoryBudgetError),
}
impl fmt::Display for ConfigError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Io(error) => write!(formatter, "config I/O error: {error}"),
Self::Json(error) => write!(formatter, "config JSON error: {error}"),
Self::Invalid(message) => {
write!(formatter, "invalid training configuration: {message}")
}
Self::Model(error) => error.fmt(formatter),
Self::Optimizer(error) => error.fmt(formatter),
Self::MemoryBudget(error) => error.fmt(formatter),
}
}
}
impl Error for ConfigError {}
impl From<std::io::Error> for ConfigError {
fn from(error: std::io::Error) -> Self {
Self::Io(error)
}
}
impl From<serde_json::Error> for ConfigError {
fn from(error: serde_json::Error) -> Self {
Self::Json(error)
}
}
impl From<ModelConfigError> for ConfigError {
fn from(error: ModelConfigError) -> Self {
Self::Model(error)
}
}
impl From<OptimizerError> for ConfigError {
fn from(error: OptimizerError) -> Self {
Self::Optimizer(error)
}
}
impl From<MemoryBudgetError> for ConfigError {
fn from(error: MemoryBudgetError) -> Self {
Self::MemoryBudget(error)
}
}
pub fn load_train_config_file(path: impl AsRef<Path>) -> Result<TrainConfigFile, ConfigError> {
let file = File::open(path)?;
let reader = BufReader::new(file);
Ok(serde_json::from_reader(reader)?)
}
pub fn write_train_config_file(
path: impl AsRef<Path>,
file: &TrainConfigFile,
) -> Result<(), ConfigError> {
let mut value = strip_nulls(serde_json::to_value(file)?);
clean_json_numbers(&mut value);
let mut out = File::create(path)?;
serde_json::to_writer_pretty(&mut out, &value)?;
writeln!(out)?;
Ok(())
}
#[derive(Clone, Debug)]
pub struct InitTrainConfigResult {
pub file: TrainConfigFile,
pub config: ResolvedTrainConfig,
pub estimate: MemoryEstimate,
pub budget_bytes: usize,
pub applied: Vec<String>,
}
pub fn init_train_config(
architecture: Option<ArchitecturePreset>,
budget_bytes: usize,
use_gpu: bool,
) -> Result<InitTrainConfigResult, ConfigError> {
let architecture = architecture.unwrap_or(ArchitecturePreset::LlamaTiny);
let overrides = TrainConfigOverrides {
architecture: Some(architecture),
sequence_length: Some(architecture.spec().max_sequence_length),
..TrainConfigOverrides::default()
};
let resolved = resolve_train_config(None, &overrides)?;
let fitted = fit_training_memory_for_device(resolved, budget_bytes, use_gpu)?;
Ok(InitTrainConfigResult {
file: fitted.config.to_config_file(),
config: fitted.config.clone(),
estimate: fitted.estimate,
budget_bytes: fitted.budget_bytes,
applied: fitted.applied,
})
}
pub fn train_config_json_schema() -> Value {
json!({
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/MickyBalladelli/OxideLM/training/schema.json",
"title": "OxideLM TrainConfigFile",
"description": "Starter / recipe training configuration loaded by `oxidelm --train --config`.",
"type": "object",
"additionalProperties": false,
"properties": {
"model": {
"type": "object",
"additionalProperties": false,
"properties": {
"architecture": {
"type": "string",
"enum": [
"llama-tiny",
"llama-micro",
"llama-small",
"gpt-tiny",
"gpt-micro",
"gpt-small"
]
},
"hidden_size": { "type": "integer", "minimum": 1 },
"num_layers": { "type": "integer", "minimum": 1 },
"num_heads": { "type": "integer", "minimum": 1 },
"num_key_value_heads": { "type": "integer", "minimum": 1 },
"intermediate_size": { "type": "integer", "minimum": 1 },
"max_sequence_length": { "type": "integer", "minimum": 1 },
"rms_norm_epsilon": { "type": "number", "exclusiveMinimum": 0 },
"rope_theta": { "type": "number", "exclusiveMinimum": 0 },
"rope_scaling": { "type": "object" },
"seed": { "type": "integer", "minimum": 0 },
"initializer_std": { "type": "number", "exclusiveMinimum": 0 },
"tie_word_embeddings": { "type": "boolean" },
"attention_dropout": { "type": "number", "minimum": 0, "maximum": 1 },
"residual_dropout": { "type": "number", "minimum": 0, "maximum": 1 },
"embedding_dropout": { "type": "number", "minimum": 0, "maximum": 1 },
"use_bias": { "type": "boolean" }
}
},
"tokenizer": {
"type": "object",
"additionalProperties": false,
"properties": {
"vocab_size": { "type": "integer", "minimum": 260 },
"min_frequency": { "type": "integer", "minimum": 1 },
"pre_tokenization": {
"oneOf": [
{ "type": "string", "const": "raw" },
{ "type": "object" }
]
}
}
},
"trainer": {
"type": "object",
"additionalProperties": false,
"properties": {
"batch_size": { "type": "integer", "minimum": 1 },
"sequence_length": { "type": "integer", "minimum": 1 },
"max_gradient_norm": { "type": "number", "exclusiveMinimum": 0 },
"log_every": { "type": "integer", "minimum": 1 },
"gradient_accumulation_steps": { "type": "integer", "minimum": 1 },
"activation_checkpointing": { "type": "boolean" },
"data_parallel_workers": { "type": "integer", "minimum": 1 },
"mixed_precision": { "type": "boolean" },
"amp_format": { "type": "string", "enum": ["fp16", "bf16"] },
"initial_loss_scale": { "type": "number" },
"loss_scale_growth_factor": { "type": "number" },
"loss_scale_backoff_factor": { "type": "number" },
"loss_scale_growth_interval": { "type": "integer", "minimum": 1 },
"activation_checkpoint_interval": { "type": "integer", "minimum": 1 },
"selective_checkpoint": {
"type": "string",
"enum": ["full", "attention", "ffn"]
},
"attention_chunk_size": { "type": "integer", "minimum": 0 },
"layer_offload": { "type": "boolean" },
"trim_caches_each_microbatch": { "type": "boolean" },
"lora": { "type": "object" },
"freeze": {
"type": "array",
"items": { "type": "string" }
},
"qat": true,
"rebuild_token_cache": { "type": "boolean" },
"metrics_jsonl": { "type": "string" }
}
},
"optimizer": {
"type": "object",
"additionalProperties": false,
"properties": {
"kind": {
"type": "string",
"enum": ["adamw", "sgd", "lion", "adafactor"]
},
"moment_dtype": { "type": "string", "enum": ["fp32", "fp16"] },
"offload_moments": { "type": "boolean" },
"beta1": { "type": "number" },
"beta2": { "type": "number" },
"epsilon": { "type": "number" },
"weight_decay": { "type": "number" },
"momentum": { "type": "number" },
"no_decay_rms_norm": { "type": "boolean" },
"no_decay_embeddings": {
"type": "array",
"items": { "type": "string" }
},
"rms_norm_lr_scale": { "type": "number" },
"embedding_lr_scale": { "type": "number" }
}
},
"schedule": {
"type": "object",
"additionalProperties": false,
"properties": {
"base_learning_rate": { "type": "number", "exclusiveMinimum": 0 },
"minimum_learning_rate": { "type": "number", "minimum": 0 },
"warmup_steps": { "type": "integer", "minimum": 0 },
"total_steps": { "type": "integer", "minimum": 1 }
}
},
"data": {
"type": "object",
"additionalProperties": false,
"properties": {
"validation_split": { "type": "number", "exclusiveMinimum": 0, "exclusiveMaximum": 1 },
"eval_every": { "type": "integer", "minimum": 1 },
"early_stopping_patience": { "type": "integer", "minimum": 0 },
"early_stopping_min_delta": { "type": "number", "minimum": 0 },
"shard_tokens": { "type": "integer", "minimum": 1 },
"tokenize_chunk_bytes": { "type": "integer", "minimum": 1 },
"prefetch_batches": { "type": "integer", "minimum": 0 }
}
}
}
})
}
fn strip_nulls(value: Value) -> Value {
match value {
Value::Object(map) => Value::Object(
map.into_iter()
.filter_map(|(key, child)| {
if child.is_null() {
None
} else {
Some((key, strip_nulls(child)))
}
})
.collect(),
),
Value::Array(items) => Value::Array(items.into_iter().map(strip_nulls).collect()),
other => other,
}
}
fn clean_json_numbers(value: &mut Value) {
match value {
Value::Number(number) => {
if number.is_i64() || number.is_u64() {
return;
}
if let Some(float) = number.as_f64() {
let cleaned = tidy_float(float);
if cleaned.fract() == 0.0 && cleaned.abs() <= i64::MAX as f64 {
*number = serde_json::Number::from(cleaned as i64);
} else if let Some(parsed) = serde_json::Number::from_f64(cleaned) {
*number = parsed;
}
}
}
Value::Object(map) => {
for child in map.values_mut() {
clean_json_numbers(child);
}
}
Value::Array(items) => {
for child in items.iter_mut() {
clean_json_numbers(child);
}
}
_ => {}
}
}
fn tidy_float(float: f64) -> f64 {
if !float.is_finite() || float == 0.0 {
return float;
}
let exp = float.abs().log10().floor();
let scale = 10f64.powi((5.0 - exp) as i32);
(float * scale).round() / scale
}
fn pick<T>(override_value: Option<T>, file_value: Option<T>, default: T) -> T {
override_value.or(file_value).unwrap_or(default)
}
pub fn resolve_train_config(
file: Option<&TrainConfigFile>,
overrides: &TrainConfigOverrides,
) -> Result<ResolvedTrainConfig, ConfigError> {
let file = file.cloned().unwrap_or_default();
let architecture = overrides.architecture.or(file.model.architecture);
let preset: Option<ArchitectureSpec> = architecture.map(ArchitecturePreset::spec);
let steps = pick(overrides.steps, file.schedule.total_steps, default_steps());
let sequence_length = pick(
overrides.sequence_length,
file.trainer.sequence_length,
default_sequence_length(),
);
let max_sequence_length = overrides
.max_sequence_length
.or(file.model.max_sequence_length)
.or(preset.map(|spec| spec.max_sequence_length))
.unwrap_or(sequence_length);
let warmup_steps = overrides
.warmup_steps
.or(file.schedule.warmup_steps)
.unwrap_or_else(|| (steps / 20).max(1));
let eval_every = overrides
.eval_every
.or(file.data.eval_every)
.unwrap_or_else(|| (steps.saturating_add(9) / 10).max(1));
let adam_defaults = AdamWConfig::default();
let num_heads = pick(
overrides.num_heads,
file.model.num_heads,
preset
.map(|spec| spec.num_heads)
.unwrap_or_else(default_num_heads),
);
let num_key_value_heads = overrides
.num_key_value_heads
.or(file.model.num_key_value_heads)
.or(preset.map(|spec| spec.num_key_value_heads))
.unwrap_or(num_heads);
let mut rope_scaling = overrides.rope_scaling.clone().or(file.model.rope_scaling);
if let Some(scaling) = &mut rope_scaling {
scaling.infer_original_context(max_sequence_length);
}
let config = ResolvedTrainConfig {
architecture,
hidden_size: pick(
overrides.hidden_size,
file.model.hidden_size,
preset
.map(|spec| spec.hidden_size)
.unwrap_or_else(default_hidden_size),
),
num_layers: pick(
overrides.num_layers,
file.model.num_layers,
preset
.map(|spec| spec.num_layers)
.unwrap_or_else(default_num_layers),
),
num_heads,
num_key_value_heads,
intermediate_size: pick(
overrides.intermediate_size,
file.model.intermediate_size,
preset
.map(|spec| spec.intermediate_size)
.unwrap_or_else(default_intermediate_size),
),
max_sequence_length,
sequence_length,
rms_norm_epsilon: pick(
overrides.rms_norm_epsilon,
file.model.rms_norm_epsilon,
default_rms_norm_epsilon(),
),
rope_theta: pick(
overrides.rope_theta,
file.model.rope_theta,
default_rope_theta(),
),
rope_scaling,
seed: pick(overrides.seed, file.model.seed, default_seed()),
initializer_std: pick(
overrides.initializer_std,
file.model.initializer_std,
default_initializer_std(),
),
tie_word_embeddings: pick(
overrides.tie_word_embeddings,
file.model.tie_word_embeddings,
preset
.map(|spec| spec.tie_word_embeddings)
.unwrap_or_else(default_tie_word_embeddings),
),
attention_dropout: pick(
overrides.attention_dropout,
file.model.attention_dropout,
default_dropout(),
),
residual_dropout: pick(
overrides.residual_dropout,
file.model.residual_dropout,
default_dropout(),
),
embedding_dropout: pick(
overrides.embedding_dropout,
file.model.embedding_dropout,
default_dropout(),
),
use_bias: pick(
overrides.use_bias,
file.model.use_bias,
preset.map(|spec| spec.use_bias).unwrap_or(false),
),
vocab_size: pick(
overrides.vocab_size,
file.tokenizer.vocab_size,
default_vocab_size(),
),
min_frequency: pick(
overrides.min_frequency,
file.tokenizer.min_frequency,
default_min_frequency(),
),
pre_tokenization: pick(
overrides.pre_tokenization.clone(),
file.tokenizer.pre_tokenization.clone(),
PreTokenization::Raw,
),
batch_size: pick(
overrides.batch_size,
file.trainer.batch_size,
default_batch_size(),
),
max_gradient_norm: pick(
overrides.max_gradient_norm,
file.trainer.max_gradient_norm,
default_max_gradient_norm(),
),
log_every: pick(
overrides.log_every,
file.trainer.log_every,
default_log_every(),
),
gradient_accumulation_steps: pick(
overrides.gradient_accumulation_steps,
file.trainer.gradient_accumulation_steps,
default_gradient_accumulation_steps(),
),
activation_checkpointing: pick(
overrides.activation_checkpointing,
file.trainer.activation_checkpointing,
default_activation_checkpointing(),
),
data_parallel_workers: pick(
overrides.data_parallel_workers,
file.trainer.data_parallel_workers,
default_data_parallel_workers(),
),
mixed_precision: {
let defaults = MixedPrecisionConfig::default();
MixedPrecisionConfig {
enabled: pick(
overrides.mixed_precision,
file.trainer.mixed_precision,
false,
),
format: pick(
overrides.amp_format,
file.trainer.amp_format,
defaults.format,
),
initial_loss_scale: pick(
overrides.initial_loss_scale,
file.trainer.initial_loss_scale,
defaults.initial_loss_scale,
),
growth_factor: pick(
overrides.loss_scale_growth_factor,
file.trainer.loss_scale_growth_factor,
defaults.growth_factor,
),
backoff_factor: pick(
overrides.loss_scale_backoff_factor,
file.trainer.loss_scale_backoff_factor,
defaults.backoff_factor,
),
growth_interval: pick(
overrides.loss_scale_growth_interval,
file.trainer.loss_scale_growth_interval,
defaults.growth_interval,
),
min_loss_scale: defaults.min_loss_scale,
max_loss_scale: defaults.max_loss_scale,
}
},
optimizer: AdamWConfig {
beta1: pick(overrides.beta1, file.optimizer.beta1, adam_defaults.beta1),
beta2: pick(overrides.beta2, file.optimizer.beta2, adam_defaults.beta2),
epsilon: pick(
overrides.epsilon,
file.optimizer.epsilon,
adam_defaults.epsilon,
),
weight_decay: pick(
overrides.weight_decay,
file.optimizer.weight_decay,
adam_defaults.weight_decay,
),
no_decay_rms_norm: pick(
overrides.no_decay_rms_norm,
file.optimizer.no_decay_rms_norm,
adam_defaults.no_decay_rms_norm,
),
no_decay_embeddings: pick(
overrides.no_decay_embeddings.clone(),
file.optimizer.no_decay_embeddings.clone(),
adam_defaults.no_decay_embeddings,
),
rms_norm_lr_scale: pick(
overrides.rms_norm_lr_scale,
file.optimizer.rms_norm_lr_scale,
adam_defaults.rms_norm_lr_scale,
),
embedding_lr_scale: pick(
overrides.embedding_lr_scale,
file.optimizer.embedding_lr_scale,
adam_defaults.embedding_lr_scale,
),
kind: pick(overrides.optimizer_kind, file.optimizer.kind, adam_defaults.kind),
moment_dtype: pick(
overrides.moment_dtype,
file.optimizer.moment_dtype,
adam_defaults.moment_dtype,
),
offload_moments: pick(
overrides.offload_moments,
file.optimizer.offload_moments,
adam_defaults.offload_moments,
),
momentum: pick(
overrides.momentum,
file.optimizer.momentum,
adam_defaults.momentum,
),
},
base_learning_rate: pick(
overrides.base_learning_rate,
file.schedule.base_learning_rate,
default_base_learning_rate(),
),
minimum_learning_rate: pick(
overrides.minimum_learning_rate,
file.schedule.minimum_learning_rate,
default_minimum_learning_rate(),
),
warmup_steps,
steps,
validation_split: pick(
overrides.validation_split,
file.data.validation_split,
default_validation_split(),
),
eval_every,
early_stopping_patience: pick(
overrides.early_stopping_patience,
file.data.early_stopping_patience,
default_early_stopping_patience(),
),
early_stopping_min_delta: pick(
overrides.early_stopping_min_delta,
file.data.early_stopping_min_delta,
default_early_stopping_min_delta(),
),
shard_tokens: pick(
overrides.shard_tokens,
file.data.shard_tokens,
default_shard_tokens(),
),
tokenize_chunk_bytes: pick(
overrides.tokenize_chunk_bytes,
file.data.tokenize_chunk_bytes,
default_tokenize_chunk_bytes(),
),
prefetch_batches: pick(
overrides.prefetch_batches,
file.data.prefetch_batches,
default_prefetch_batches(),
),
low_memory: {
let defaults = LowMemoryProfile::default();
LowMemoryProfile {
activation_checkpoint_interval: pick(
overrides.activation_checkpoint_interval,
file.trainer.activation_checkpoint_interval,
defaults.activation_checkpoint_interval,
),
selective_checkpoint: pick(
overrides.selective_checkpoint,
file.trainer.selective_checkpoint,
defaults.selective_checkpoint,
),
attention_chunk_size: pick(
overrides.attention_chunk_size,
file.trainer.attention_chunk_size,
defaults.attention_chunk_size,
),
layer_offload: pick(
overrides.layer_offload,
file.trainer.layer_offload,
defaults.layer_offload,
),
trim_caches_each_microbatch: pick(
overrides.trim_caches_each_microbatch,
file.trainer.trim_caches_each_microbatch,
defaults.trim_caches_each_microbatch,
),
lora: overrides.lora.clone().or(file.trainer.lora.clone()),
freeze: pick(
overrides.freeze.clone(),
file.trainer.freeze.clone(),
defaults.freeze,
),
qat: overrides.qat.clone().or(file.trainer.qat.clone()),
rebuild_token_cache: pick(
overrides.rebuild_token_cache,
file.trainer.rebuild_token_cache,
defaults.rebuild_token_cache,
),
}
},
metrics_jsonl: overrides
.metrics_jsonl
.clone()
.or_else(|| file.trainer.metrics_jsonl.clone()),
};
config.validate_static()?;
Ok(config)
}
impl ResolvedTrainConfig {
pub fn trainer_config(&self) -> TrainerConfig {
TrainerConfig {
batch_size: self.batch_size,
sequence_length: self.sequence_length,
max_gradient_norm: self.max_gradient_norm,
log_every: self.log_every,
gradient_accumulation_steps: self.gradient_accumulation_steps,
mixed_precision: self.mixed_precision.clone(),
activation_checkpointing: self.activation_checkpointing,
data_parallel_workers: self.data_parallel_workers,
low_memory: self.low_memory.clone(),
}
}
pub fn schedule(&self) -> LearningRateSchedule {
LearningRateSchedule {
base_learning_rate: self.base_learning_rate,
minimum_learning_rate: self.minimum_learning_rate,
warmup_steps: self.warmup_steps,
total_steps: self.steps,
}
}
pub fn causal_lm_config(&self, vocab_size: usize, eos_token_id: usize) -> CausalLmConfig {
CausalLmConfig {
vocab_size,
hidden_size: self.hidden_size,
num_layers: self.num_layers,
num_heads: self.num_heads,
num_key_value_heads: self.num_key_value_heads,
intermediate_size: self.intermediate_size,
max_sequence_length: self.max_sequence_length,
rms_norm_epsilon: self.rms_norm_epsilon,
rope_theta: self.rope_theta,
rope_scaling: self.rope_scaling.clone(),
eos_token_id,
seed: self.seed,
initializer_std: self.initializer_std,
tie_word_embeddings: self.tie_word_embeddings,
attention_dropout: self.attention_dropout,
residual_dropout: self.residual_dropout,
embedding_dropout: self.embedding_dropout,
use_bias: self.use_bias,
}
}
pub fn to_config_file(&self) -> TrainConfigFile {
let amp_defaults = MixedPrecisionConfig::default();
let low_defaults = LowMemoryProfile::default();
TrainConfigFile {
model: ModelSection {
architecture: self.architecture,
hidden_size: Some(self.hidden_size),
num_layers: Some(self.num_layers),
num_heads: Some(self.num_heads),
num_key_value_heads: Some(self.num_key_value_heads),
intermediate_size: Some(self.intermediate_size),
max_sequence_length: Some(self.max_sequence_length),
rms_norm_epsilon: Some(self.rms_norm_epsilon),
rope_theta: Some(self.rope_theta),
rope_scaling: self.rope_scaling.clone(),
seed: Some(self.seed),
initializer_std: Some(self.initializer_std),
tie_word_embeddings: Some(self.tie_word_embeddings),
attention_dropout: Some(self.attention_dropout),
residual_dropout: Some(self.residual_dropout),
embedding_dropout: Some(self.embedding_dropout),
use_bias: Some(self.use_bias),
},
tokenizer: TokenizerSection {
vocab_size: Some(self.vocab_size),
min_frequency: Some(self.min_frequency),
pre_tokenization: if self.pre_tokenization == PreTokenization::Raw {
None
} else {
Some(self.pre_tokenization.clone())
},
},
trainer: TrainerSection {
batch_size: Some(self.batch_size),
sequence_length: Some(self.sequence_length),
max_gradient_norm: Some(self.max_gradient_norm),
log_every: Some(self.log_every),
gradient_accumulation_steps: Some(self.gradient_accumulation_steps),
activation_checkpointing: Some(self.activation_checkpointing),
data_parallel_workers: if self.data_parallel_workers
== default_data_parallel_workers()
{
None
} else {
Some(self.data_parallel_workers)
},
mixed_precision: Some(self.mixed_precision.enabled),
amp_format: if self.mixed_precision.enabled {
Some(self.mixed_precision.format)
} else {
None
},
initial_loss_scale: if self.mixed_precision.enabled
&& self.mixed_precision.initial_loss_scale != amp_defaults.initial_loss_scale
{
Some(self.mixed_precision.initial_loss_scale)
} else {
None
},
loss_scale_growth_factor: if self.mixed_precision.enabled
&& self.mixed_precision.growth_factor != amp_defaults.growth_factor
{
Some(self.mixed_precision.growth_factor)
} else {
None
},
loss_scale_backoff_factor: if self.mixed_precision.enabled
&& self.mixed_precision.backoff_factor != amp_defaults.backoff_factor
{
Some(self.mixed_precision.backoff_factor)
} else {
None
},
loss_scale_growth_interval: if self.mixed_precision.enabled
&& self.mixed_precision.growth_interval != amp_defaults.growth_interval
{
Some(self.mixed_precision.growth_interval)
} else {
None
},
activation_checkpoint_interval: if self.activation_checkpointing
|| self.low_memory.activation_checkpoint_interval
!= low_defaults.activation_checkpoint_interval
{
Some(self.low_memory.activation_checkpoint_interval)
} else {
None
},
selective_checkpoint: if self.activation_checkpointing
&& self.low_memory.selective_checkpoint != low_defaults.selective_checkpoint
{
Some(self.low_memory.selective_checkpoint)
} else {
None
},
attention_chunk_size: if self.low_memory.attention_chunk_size > 0 {
Some(self.low_memory.attention_chunk_size)
} else {
None
},
layer_offload: if self.low_memory.layer_offload {
Some(true)
} else {
None
},
trim_caches_each_microbatch: if self.low_memory.trim_caches_each_microbatch {
Some(true)
} else {
None
},
lora: self.low_memory.lora.clone(),
freeze: if self.low_memory.freeze.is_empty() {
None
} else {
Some(self.low_memory.freeze.clone())
},
qat: self.low_memory.qat.clone(),
rebuild_token_cache: if self.low_memory.rebuild_token_cache {
Some(true)
} else {
None
},
metrics_jsonl: self.metrics_jsonl.clone(),
},
optimizer: OptimizerSection {
beta1: Some(self.optimizer.beta1),
beta2: Some(self.optimizer.beta2),
epsilon: Some(self.optimizer.epsilon),
weight_decay: Some(self.optimizer.weight_decay),
no_decay_rms_norm: None,
no_decay_embeddings: None,
rms_norm_lr_scale: None,
embedding_lr_scale: None,
kind: Some(self.optimizer.kind),
moment_dtype: Some(self.optimizer.moment_dtype),
offload_moments: Some(self.optimizer.offload_moments),
momentum: if self.optimizer.kind == OptimizerKind::Sgd {
Some(self.optimizer.momentum)
} else {
None
},
},
schedule: ScheduleSection {
base_learning_rate: Some(self.base_learning_rate),
minimum_learning_rate: Some(self.minimum_learning_rate),
warmup_steps: Some(self.warmup_steps),
total_steps: Some(self.steps),
},
data: DataSection {
validation_split: Some(self.validation_split),
eval_every: Some(self.eval_every),
early_stopping_patience: Some(self.early_stopping_patience),
early_stopping_min_delta: Some(self.early_stopping_min_delta),
shard_tokens: Some(self.shard_tokens),
tokenize_chunk_bytes: Some(self.tokenize_chunk_bytes),
prefetch_batches: Some(self.prefetch_batches),
},
}
}
pub fn validate_static(&self) -> Result<(), ConfigError> {
if self.steps == 0 {
return Err(ConfigError::Invalid("steps must be positive".into()));
}
if self.eval_every == 0 {
return Err(ConfigError::Invalid(
"evaluation interval must be positive".into(),
));
}
if !self.early_stopping_min_delta.is_finite() || self.early_stopping_min_delta < 0.0 {
return Err(ConfigError::Invalid(
"early stopping min delta must be finite and non-negative".into(),
));
}
if self.shard_tokens == 0 || self.tokenize_chunk_bytes == 0 {
return Err(ConfigError::Invalid(
"shard size and tokenization chunk size must be positive".into(),
));
}
if self.low_memory.activation_checkpoint_interval == 0 {
return Err(ConfigError::Invalid(
"activation_checkpoint_interval must be positive".into(),
));
}
if let Some(lora) = &self.low_memory.lora {
lora.validate()
.map_err(|error| ConfigError::Invalid(error.to_string()))?;
}
if let Some(qat) = &self.low_memory.qat {
qat.validate()
.map_err(|error| ConfigError::Invalid(error.to_string()))?;
}
if !self.validation_split.is_finite()
|| self.validation_split <= 0.0
|| self.validation_split >= 1.0
{
return Err(ConfigError::Invalid(
"validation split must be greater than 0 and less than 1".into(),
));
}
if self.batch_size == 0 || self.sequence_length == 0 {
return Err(ConfigError::Invalid(
"batch size and sequence length must be positive".into(),
));
}
if self.gradient_accumulation_steps == 0 {
return Err(ConfigError::Invalid(
"gradient accumulation steps must be positive".into(),
));
}
if self.data_parallel_workers == 0 {
return Err(ConfigError::Invalid(
"data-parallel workers must be positive".into(),
));
}
if self.data_parallel_workers > 1 {
if self.batch_size < self.data_parallel_workers {
return Err(ConfigError::Invalid(format!(
"batch size {} must be >= data-parallel workers {}",
self.batch_size, self.data_parallel_workers
)));
}
if self.batch_size % self.data_parallel_workers != 0 {
return Err(ConfigError::Invalid(format!(
"batch size {} must be divisible by data-parallel workers {}",
self.batch_size, self.data_parallel_workers
)));
}
}
if let Err(error) = self.mixed_precision.validate() {
return Err(ConfigError::Invalid(error.to_string()));
}
if self.max_sequence_length < self.sequence_length {
return Err(ConfigError::Invalid(
"max sequence length must be at least the trainer sequence length".into(),
));
}
if self.log_every == 0 {
return Err(ConfigError::Invalid("log_every must be positive".into()));
}
if !(self.max_gradient_norm.is_finite() && self.max_gradient_norm > 0.0) {
return Err(ConfigError::Invalid(
"max gradient norm must be positive and finite".into(),
));
}
if self.vocab_size < FIRST_MERGE_ID {
return Err(ConfigError::Invalid(format!(
"vocabulary size {} is too small; minimum is {FIRST_MERGE_ID}",
self.vocab_size
)));
}
if self.min_frequency == 0 {
return Err(ConfigError::Invalid(
"tokenizer min_frequency must be positive".into(),
));
}
if let Err(error) = self.pre_tokenization.validate() {
return Err(ConfigError::Invalid(error.to_string()));
}
if self.base_learning_rate <= 0.0
|| !self.base_learning_rate.is_finite()
|| self.minimum_learning_rate < 0.0
|| !self.minimum_learning_rate.is_finite()
{
return Err(ConfigError::Invalid(
"learning-rate schedule rates are invalid".into(),
));
}
if self.minimum_learning_rate > self.base_learning_rate {
return Err(ConfigError::Invalid(
"minimum learning rate must not exceed base learning rate".into(),
));
}
let model = self.causal_lm_config(self.vocab_size, 0);
model.validate()?;
self.optimizer.validate()?;
Ok(())
}
pub fn validate_corpus(
&self,
training_tokens: usize,
validation_tokens: usize,
) -> Result<(), ConfigError> {
if training_tokens <= self.sequence_length {
return Err(ConfigError::Invalid(format!(
"training corpus has {training_tokens} tokens; need more than sequence length {}",
self.sequence_length
)));
}
if validation_tokens <= self.sequence_length {
return Err(ConfigError::Invalid(format!(
"validation corpus has {validation_tokens} tokens; need more than sequence length {}",
self.sequence_length
)));
}
Ok(())
}
}
const FIT_MIN_SEQUENCE_LENGTH: usize = 16;
const FIT_MIN_HIDDEN_SIZE: usize = 32;
const FIT_MIN_NUM_LAYERS: usize = 1;
const FIT_MAX_GRAD_ACCUM: usize = 64;
#[derive(Clone, Debug)]
pub struct FitMemoryResult {
pub config: ResolvedTrainConfig,
pub estimate: MemoryEstimate,
pub budget_bytes: usize,
pub applied: Vec<String>,
}
pub fn apply_gpu_tiny_host_safeguards(config: &mut ResolvedTrainConfig) -> Vec<String> {
let mut applied = Vec::new();
if config.batch_size > 1 {
let previous_batch = config.batch_size;
let effective = previous_batch.saturating_mul(config.gradient_accumulation_steps);
config.batch_size = 1;
config.gradient_accumulation_steps = effective
.max(config.gradient_accumulation_steps)
.min(FIT_MAX_GRAD_ACCUM);
applied.push(format!(
"batch_size=1 gradient_accumulation_steps={} (gpu+tiny-host microbatch)",
config.gradient_accumulation_steps
));
}
if !config.low_memory.layer_offload {
config.low_memory.layer_offload = true;
applied.push(
"layer_offload=true (--layer-offload / trainer.layer_offload)".into(),
);
}
if !config.low_memory.trim_caches_each_microbatch {
config.low_memory.trim_caches_each_microbatch = true;
applied.push(
"trim_caches_each_microbatch=true (--trim-caches / trainer.trim_caches_each_microbatch)"
.into(),
);
}
if !config.optimizer.offload_moments {
config.optimizer.offload_moments = true;
applied.push(
"offload_moments=true (--offload-moments / optimizer.offload_moments)".into(),
);
}
applied
}
pub fn fit_training_memory(
config: ResolvedTrainConfig,
budget_bytes: usize,
) -> Result<FitMemoryResult, ConfigError> {
fit_training_memory_for_device(config, budget_bytes, false)
}
pub fn fit_training_memory_for_device(
mut config: ResolvedTrainConfig,
budget_bytes: usize,
use_gpu: bool,
) -> Result<FitMemoryResult, ConfigError> {
if budget_bytes == 0 {
return Err(ConfigError::Invalid(
"fit-memory budget must be positive".into(),
));
}
let budget = DeviceBudget::cpu(Some(budget_bytes));
let mut applied = Vec::new();
if let Some(estimate) = try_memory_plan(&config, &budget, use_gpu)? {
return Ok(FitMemoryResult {
config,
estimate,
budget_bytes,
applied,
});
}
if use_gpu && host_memory_is_tiny() {
let safeguards = apply_gpu_tiny_host_safeguards(&mut config);
if !safeguards.is_empty() {
applied.extend(safeguards);
if let Some(estimate) = try_memory_plan(&config, &budget, use_gpu)? {
return Ok(FitMemoryResult {
config,
estimate,
budget_bytes,
applied,
});
}
}
}
if config.gradient_accumulation_steps < 8 {
config.gradient_accumulation_steps = 8;
applied.push(
"gradient_accumulation_steps=8 (--gradient-accumulation-steps / trainer.gradient_accumulation_steps)"
.into(),
);
if let Some(estimate) = try_memory_plan(&config, &budget, use_gpu)? {
return Ok(FitMemoryResult {
config,
estimate,
budget_bytes,
applied,
});
}
}
if !config.activation_checkpointing {
config.activation_checkpointing = true;
applied.push(
"activation_checkpointing=true (--activation-checkpointing / trainer.activation_checkpointing)"
.into(),
);
if let Some(estimate) = try_memory_plan(&config, &budget, use_gpu)? {
return Ok(FitMemoryResult {
config,
estimate,
budget_bytes,
applied,
});
}
}
while config.batch_size > 1 {
let previous_batch = config.batch_size;
let next_batch = (previous_batch / 2).max(1);
let effective = previous_batch.saturating_mul(config.gradient_accumulation_steps);
config.batch_size = next_batch;
config.gradient_accumulation_steps = (effective / next_batch)
.max(config.gradient_accumulation_steps)
.min(FIT_MAX_GRAD_ACCUM);
applied.push(format!(
"batch_size={} gradient_accumulation_steps={} (--batch-size / --gradient-accumulation-steps)",
config.batch_size, config.gradient_accumulation_steps
));
if let Some(estimate) = try_memory_plan(&config, &budget, use_gpu)? {
return Ok(FitMemoryResult {
config,
estimate,
budget_bytes,
applied,
});
}
}
while config.sequence_length > FIT_MIN_SEQUENCE_LENGTH {
let next = (config.sequence_length / 2).max(FIT_MIN_SEQUENCE_LENGTH);
if next == config.sequence_length {
break;
}
config.sequence_length = next;
if config.max_sequence_length < config.sequence_length {
config.max_sequence_length = config.sequence_length;
} else if config.max_sequence_length > config.sequence_length {
config.max_sequence_length = config.sequence_length;
}
applied.push(format!(
"sequence_length={} (--sequence-length / trainer.sequence_length)",
config.sequence_length
));
if let Some(estimate) = try_memory_plan(&config, &budget, use_gpu)? {
return Ok(FitMemoryResult {
config,
estimate,
budget_bytes,
applied,
});
}
}
if !config.mixed_precision.enabled {
config.mixed_precision.enabled = true;
config.mixed_precision.format = AmpFormat::Bf16;
applied.push(
"mixed_precision=true amp_format=bf16 (--mixed-precision --amp-format bf16 / trainer.mixed_precision)"
.into(),
);
if let Some(estimate) = try_memory_plan(&config, &budget, use_gpu)? {
return Ok(FitMemoryResult {
config,
estimate,
budget_bytes,
applied,
});
}
}
while config.hidden_size > FIT_MIN_HIDDEN_SIZE || config.num_layers > FIT_MIN_NUM_LAYERS {
let mut changed = false;
if config.num_layers > FIT_MIN_NUM_LAYERS {
config.num_layers = (config.num_layers - 1).max(FIT_MIN_NUM_LAYERS);
changed = true;
applied.push(format!(
"num_layers={} (--num-layers / model.num_layers)",
config.num_layers
));
} else if config.hidden_size > FIT_MIN_HIDDEN_SIZE {
let next_hidden = (config.hidden_size / 2).max(FIT_MIN_HIDDEN_SIZE);
let mut heads = config.num_heads.min(next_hidden).max(1);
while next_hidden % heads != 0 && heads > 1 {
heads -= 1;
}
if next_hidden % heads != 0 {
break;
}
let mut kv = config.num_key_value_heads.min(heads).max(1);
while heads % kv != 0 && kv > 1 {
kv -= 1;
}
config.hidden_size = next_hidden;
config.num_heads = heads;
config.num_key_value_heads = kv;
config.intermediate_size = next_hidden.saturating_mul(2).max(FIT_MIN_HIDDEN_SIZE);
changed = true;
applied.push(format!(
"hidden_size={} num_heads={} num_key_value_heads={} intermediate_size={}",
config.hidden_size,
config.num_heads,
config.num_key_value_heads,
config.intermediate_size
));
}
if !changed {
break;
}
config.validate_static()?;
if let Some(estimate) = try_memory_plan(&config, &budget, use_gpu)? {
return Ok(FitMemoryResult {
config,
estimate,
budget_bytes,
applied,
});
}
}
if config.prefetch_batches > 0 {
config.prefetch_batches = 0;
applied.push(
"prefetch_batches=0 (--prefetch-batches / data.prefetch_batches)".into(),
);
if let Some(estimate) = try_memory_plan(&config, &budget, use_gpu)? {
return Ok(FitMemoryResult {
config,
estimate,
budget_bytes,
applied,
});
}
}
if use_gpu {
let safeguards = apply_gpu_tiny_host_safeguards(&mut config);
if !safeguards.is_empty() {
applied.extend(safeguards);
if let Some(estimate) = try_memory_plan(&config, &budget, use_gpu)? {
return Ok(FitMemoryResult {
config,
estimate,
budget_bytes,
applied,
});
}
}
}
let model = config.causal_lm_config(config.vocab_size, 0);
let trainer = config.trainer_config();
let mut estimate = estimate_training_memory_detailed(
&model,
&trainer,
&config.optimizer,
config.prefetch_batches,
);
if use_gpu {
apply_gpu_parameter_residency(
&mut estimate,
&model,
config.low_memory.layer_offload,
);
}
Err(MemoryBudgetError::ExceedsDeviceBudget {
label: budget.label,
estimated_bytes: estimate.total_bytes,
budget_bytes,
estimate,
}
.into())
}
fn try_memory_plan(
config: &ResolvedTrainConfig,
budget: &DeviceBudget,
use_gpu: bool,
) -> Result<Option<MemoryEstimate>, ConfigError> {
config.validate_static()?;
let model = config.causal_lm_config(config.vocab_size, 0);
let trainer = config.trainer_config();
let mut estimate = estimate_training_memory_detailed(
&model,
&trainer,
&config.optimizer,
config.prefetch_batches,
);
if use_gpu {
apply_gpu_parameter_residency(
&mut estimate,
&model,
config.low_memory.layer_offload,
);
}
match check_memory_budget(&estimate, budget) {
Ok(()) => Ok(Some(estimate)),
Err(MemoryBudgetError::ExceedsDeviceBudget { .. }) => Ok(None),
Err(MemoryBudgetError::ExceedsMaxBuffer { .. }) => Ok(None),
}
}
pub fn validate_training_plan(
model_config: &CausalLmConfig,
trainer_config: &TrainerConfig,
schedule: &LearningRateSchedule,
optimizer: &AdamWConfig,
backend: &TrainingBackend,
memory_budget_bytes: Option<usize>,
) -> Result<(MemoryEstimate, DeviceBudget), ConfigError> {
model_config.validate()?;
optimizer.validate()?;
if trainer_config.batch_size == 0 || trainer_config.sequence_length == 0 {
return Err(ConfigError::Invalid(
"batch size and sequence length must be positive".into(),
));
}
if trainer_config.gradient_accumulation_steps == 0 {
return Err(ConfigError::Invalid(
"gradient accumulation steps must be positive".into(),
));
}
if trainer_config.data_parallel_workers == 0 {
return Err(ConfigError::Invalid(
"data-parallel workers must be positive".into(),
));
}
if trainer_config.data_parallel_workers > 1 {
if matches!(backend, TrainingBackend::Gpu(_))
&& trainer_config.low_memory.layer_offload
{
return Err(ConfigError::Invalid(
"GPU data-parallel does not support layer_offload; disable layer offload or use a single GPU worker"
.into(),
));
}
if trainer_config.batch_size % trainer_config.data_parallel_workers != 0 {
return Err(ConfigError::Invalid(format!(
"batch size {} must be divisible by data-parallel workers {}",
trainer_config.batch_size, trainer_config.data_parallel_workers
)));
}
}
if trainer_config.sequence_length > model_config.max_sequence_length {
return Err(ConfigError::Invalid(
"trainer sequence length exceeds model context".into(),
));
}
if schedule.total_steps == 0
|| schedule.base_learning_rate <= 0.0
|| schedule.minimum_learning_rate < 0.0
{
return Err(ConfigError::Invalid(
"learning-rate schedule is invalid".into(),
));
}
let mut estimate = estimate_training_memory_detailed(
model_config,
trainer_config,
optimizer,
0,
);
let budget = match backend {
TrainingBackend::Cpu => DeviceBudget::cpu(memory_budget_bytes),
TrainingBackend::Gpu(gpu) => {
apply_gpu_parameter_residency(
&mut estimate,
model_config,
trainer_config.low_memory.layer_offload,
);
gpu.apply_memory_budget(memory_budget_bytes).map_err(|error| {
ConfigError::Invalid(format!("failed to apply GPU memory budget: {error}"))
})?;
let mut budget = gpu.memory_budget();
if let Some(total_bytes) = memory_budget_bytes {
budget.total_bytes = total_bytes;
}
budget
}
};
check_memory_budget(&estimate, &budget)?;
Ok((estimate, budget))
}
fn resume_flag_mismatch(
flag: &str,
checkpoint: impl fmt::Display,
requested: impl fmt::Display,
) -> ConfigError {
ConfigError::Invalid(format!(
"resume low-RAM flag mismatch: {flag}={requested} but checkpoint has {flag}={checkpoint}; omit the flag (and matching JSON key) to keep the checkpoint value"
))
}
fn check_resume_flag<T: PartialEq + fmt::Debug>(
flag: &str,
override_value: Option<T>,
file_value: Option<T>,
checkpoint: T,
) -> Result<(), ConfigError> {
let Some(requested) = override_value.or(file_value) else {
return Ok(());
};
if requested != checkpoint {
return Err(resume_flag_mismatch(
flag,
format!("{checkpoint:?}").to_ascii_lowercase(),
format!("{requested:?}").to_ascii_lowercase(),
));
}
Ok(())
}
pub fn validate_resume_low_ram_flags(
trainer_config: &TrainerConfig,
optimizer_config: &AdamWConfig,
file: Option<&TrainConfigFile>,
overrides: &TrainConfigOverrides,
) -> Result<(), ConfigError> {
let file_trainer = file.map(|config| &config.trainer);
let file_optimizer = file.map(|config| &config.optimizer);
check_resume_flag(
"activation_checkpointing",
overrides.activation_checkpointing,
file_trainer.and_then(|section| section.activation_checkpointing),
trainer_config.activation_checkpointing,
)?;
check_resume_flag(
"activation_checkpoint_interval",
overrides.activation_checkpoint_interval,
file_trainer.and_then(|section| section.activation_checkpoint_interval),
trainer_config.low_memory.activation_checkpoint_interval,
)?;
check_resume_flag(
"selective_checkpoint",
overrides.selective_checkpoint,
file_trainer.and_then(|section| section.selective_checkpoint),
trainer_config.low_memory.selective_checkpoint,
)?;
check_resume_flag(
"attention_chunk_size",
overrides.attention_chunk_size,
file_trainer.and_then(|section| section.attention_chunk_size),
trainer_config.low_memory.attention_chunk_size,
)?;
check_resume_flag(
"layer_offload",
overrides.layer_offload,
file_trainer.and_then(|section| section.layer_offload),
trainer_config.low_memory.layer_offload,
)?;
check_resume_flag(
"trim_caches_each_microbatch",
overrides.trim_caches_each_microbatch,
file_trainer.and_then(|section| section.trim_caches_each_microbatch),
trainer_config.low_memory.trim_caches_each_microbatch,
)?;
check_resume_flag(
"offload_moments",
overrides.offload_moments,
file_optimizer.and_then(|section| section.offload_moments),
optimizer_config.offload_moments,
)?;
check_resume_flag(
"moment_dtype",
overrides.moment_dtype,
file_optimizer.and_then(|section| section.moment_dtype),
optimizer_config.moment_dtype,
)?;
check_resume_flag(
"mixed_precision",
overrides.mixed_precision,
file_trainer.and_then(|section| section.mixed_precision),
trainer_config.mixed_precision.enabled,
)?;
check_resume_flag(
"amp_format",
overrides.amp_format,
file_trainer.and_then(|section| section.amp_format),
trainer_config.mixed_precision.format,
)?;
check_resume_flag(
"initial_loss_scale",
overrides.initial_loss_scale,
file_trainer.and_then(|section| section.initial_loss_scale),
trainer_config.mixed_precision.initial_loss_scale,
)?;
Ok(())
}
pub fn format_steps_clarification(
cli_steps: Option<usize>,
file_total_steps: Option<usize>,
resolved_steps: usize,
resume: bool,
eval_only: bool,
checkpoint_step: Option<usize>,
checkpoint_total_steps: Option<usize>,
) -> Vec<String> {
let mut messages = Vec::new();
if eval_only {
if let (Some(step), Some(total)) = (checkpoint_step, checkpoint_total_steps) {
messages.push(format!(
"--eval-only: checkpoint step={step} schedule.total_steps={total}; no optimizer steps will run"
));
} else {
messages.push(
"--eval-only: no optimizer steps will run; validation evaluation only".into(),
);
}
return messages;
}
match (cli_steps, file_total_steps) {
(Some(cli), Some(file)) if cli != file => {
if resume {
messages.push(format!(
"--steps/--max-steps={cli} overrides config schedule.total_steps={file} for this session only (resume runs {cli} additional optimizer steps; LR horizon follows the checkpoint and may extend)"
));
} else {
messages.push(format!(
"--steps/--max-steps={cli} overrides config schedule.total_steps={file} (session length and LR schedule horizon are both {cli})"
));
}
}
(Some(cli), None) => {
if resume {
messages.push(format!(
"--steps/--max-steps={cli}: resume will run {cli} additional optimizer steps"
));
} else {
messages.push(format!(
"--steps/--max-steps={cli}: session length and LR schedule horizon are {cli}"
));
}
}
(None, Some(file)) => {
if resume {
messages.push(format!(
"no --steps/--max-steps; using config schedule.total_steps={file} as {file} additional resume steps (alias: --max-steps)"
));
} else {
messages.push(format!(
"using config schedule.total_steps={file} as session length (CLI aliases: --steps / --max-steps)"
));
}
}
(None, None) => {
if resume {
messages.push(format!(
"no --steps/--max-steps; defaulting to {resolved_steps} additional resume steps (pass --steps N or --max-steps N)"
));
}
}
(Some(_), Some(_)) => {}
}
if resume {
if let (Some(step), Some(total)) = (checkpoint_step, checkpoint_total_steps) {
if step >= total {
messages.push(format!(
"checkpoint already at step {step} with schedule.total_steps={total}; without an explicit --steps/--max-steps this run still trains {resolved_steps} more steps (or use --eval-only)"
));
} else if step.saturating_add(resolved_steps) > total {
messages.push(format!(
"checkpoint step {step} + session {resolved_steps} exceeds schedule.total_steps={total}; LR horizon will extend to {}",
step + resolved_steps
));
}
}
}
messages
}
#[cfg(test)]
mod tests {
use super::{
format_steps_clarification, load_train_config_file, resolve_train_config,
validate_resume_low_ram_flags, validate_training_plan, ConfigError, TrainConfigFile,
TrainConfigOverrides,
};
use crate::amp::{AmpFormat, MixedPrecisionConfig};
use crate::optimizer::AdamWConfig;
use crate::rope::RopeScalingType;
use crate::trainer::{
LearningRateSchedule, LowMemoryProfile, TrainerConfig, TrainingBackend,
};
use std::io::Write;
use std::time::{SystemTime, UNIX_EPOCH};
#[test]
fn defaults_match_miniature_training() {
let config = resolve_train_config(None, &TrainConfigOverrides::default()).unwrap();
assert_eq!(config.hidden_size, 64);
assert_eq!(config.num_layers, 2);
assert_eq!(config.num_heads, 4);
assert_eq!(config.num_key_value_heads, 4);
assert_eq!(config.intermediate_size, 128);
assert_eq!(config.sequence_length, 32);
assert_eq!(config.max_sequence_length, 32);
assert_eq!(config.batch_size, 1);
assert_eq!(config.gradient_accumulation_steps, 1);
assert!(!config.activation_checkpointing);
assert_eq!(config.data_parallel_workers, 1);
assert_eq!(config.steps, 100);
assert_eq!(config.warmup_steps, 5);
assert_eq!(config.eval_every, 10);
assert_eq!(config.early_stopping_patience, 0);
assert_eq!(config.early_stopping_min_delta, 0.0);
assert_eq!(config.initializer_std, 0.02);
assert!(config.tie_word_embeddings);
assert!(!config.use_bias);
assert!(config.architecture.is_none());
assert_eq!(config.attention_dropout, 0.0);
assert_eq!(config.residual_dropout, 0.0);
assert_eq!(config.embedding_dropout, 0.0);
assert!(config.rope_scaling.is_none());
}
#[test]
fn architecture_preset_fills_dimensions_and_bias() {
use crate::model::ArchitecturePreset;
let overrides = TrainConfigOverrides {
architecture: Some(ArchitecturePreset::GptTiny),
..TrainConfigOverrides::default()
};
let config = resolve_train_config(None, &overrides).unwrap();
assert_eq!(config.architecture, Some(ArchitecturePreset::GptTiny));
assert_eq!(config.hidden_size, 128);
assert_eq!(config.num_layers, 4);
assert_eq!(config.num_heads, 4);
assert_eq!(config.num_key_value_heads, 4);
assert_eq!(config.intermediate_size, 512);
assert_eq!(config.max_sequence_length, 256);
assert!(config.use_bias);
assert!(config.tie_word_embeddings);
}
#[test]
fn explicit_dims_override_architecture_preset() {
use crate::model::ArchitecturePreset;
let file: TrainConfigFile = serde_json::from_str(
r#"{ "model": { "architecture": "llama-tiny", "hidden_size": 256 } }"#,
)
.unwrap();
let config = resolve_train_config(Some(&file), &TrainConfigOverrides::default()).unwrap();
assert_eq!(config.architecture, Some(ArchitecturePreset::LlamaTiny));
assert_eq!(config.hidden_size, 256);
assert_eq!(config.num_layers, 4);
assert_eq!(config.num_key_value_heads, 2);
assert!(!config.use_bias);
}
#[test]
fn resolves_yarn_scaling_from_json() {
let file: TrainConfigFile = serde_json::from_str(
r#"{
"model": {
"max_sequence_length": 8192,
"rope_scaling": {
"rope_type": "yarn",
"factor": 4.0,
"original_max_position_embeddings": 2048
}
},
"trainer": { "sequence_length": 32 }
}"#,
)
.unwrap();
let config = resolve_train_config(Some(&file), &TrainConfigOverrides::default()).unwrap();
let scaling = config.rope_scaling.unwrap();
assert_eq!(scaling.kind, RopeScalingType::Yarn);
assert_eq!(scaling.factor, 4.0);
assert_eq!(scaling.original_max_position_embeddings, 2_048);
}
#[test]
fn file_then_cli_override_order() {
let file: TrainConfigFile = serde_json::from_str(
r#"{
"model": { "hidden_size": 128, "num_layers": 4, "num_heads": 8, "num_key_value_heads": 2, "initializer_std": 0.04, "attention_dropout": 0.1 },
"trainer": { "batch_size": 2, "sequence_length": 64 },
"schedule": { "total_steps": 200, "warmup_steps": 20 }
}"#,
)
.unwrap();
let overrides = TrainConfigOverrides {
hidden_size: Some(256),
initializer_std: Some(0.03),
attention_dropout: Some(0.2),
steps: Some(50),
..TrainConfigOverrides::default()
};
let config = resolve_train_config(Some(&file), &overrides).unwrap();
assert_eq!(config.hidden_size, 256);
assert_eq!(config.num_layers, 4);
assert_eq!(config.num_heads, 8);
assert_eq!(config.num_key_value_heads, 2);
assert_eq!(config.attention_dropout, 0.2);
assert_eq!(config.batch_size, 2);
assert_eq!(config.sequence_length, 64);
assert_eq!(config.max_sequence_length, 64);
assert_eq!(config.initializer_std, 0.03);
assert_eq!(config.steps, 50);
assert_eq!(config.warmup_steps, 20);
}
#[test]
fn steps_clarification_flags_total_steps_mismatch_and_resume_horizon() {
let mismatch =
format_steps_clarification(Some(50), Some(200), 50, false, false, None, None);
assert!(mismatch.iter().any(|message| message.contains("overrides")));
assert!(mismatch.iter().any(|message| message.contains("200")));
let resume_done =
format_steps_clarification(None, Some(100), 100, true, false, Some(100), Some(100));
assert!(resume_done
.iter()
.any(|message| message.contains("already at step")));
assert!(resume_done.iter().any(|message| message.contains("--eval-only")));
let resume_extend =
format_steps_clarification(Some(50), Some(100), 50, true, false, Some(80), Some(100));
assert!(resume_extend
.iter()
.any(|message| message.contains("will extend")));
let eval_only =
format_steps_clarification(None, Some(100), 100, true, true, Some(100), Some(100));
assert_eq!(eval_only.len(), 1);
assert!(eval_only[0].contains("--eval-only"));
assert!(!eval_only[0].contains("still trains"));
}
#[test]
fn rejects_heads_that_do_not_divide_hidden_size() {
let overrides = TrainConfigOverrides {
hidden_size: Some(60),
num_heads: Some(4),
..TrainConfigOverrides::default()
};
let error = resolve_train_config(None, &overrides).unwrap_err();
assert!(matches!(error, ConfigError::Model(_)));
}
#[test]
fn rejects_key_value_heads_that_do_not_divide_query_heads() {
let overrides = TrainConfigOverrides {
num_heads: Some(4),
num_key_value_heads: Some(3),
..TrainConfigOverrides::default()
};
let error = resolve_train_config(None, &overrides).unwrap_err();
assert!(matches!(error, ConfigError::Model(_)));
}
#[test]
fn loads_json_config_file() {
let stamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let path = std::env::temp_dir().join(format!("oxidelm-train-config-{stamp}.json"));
let mut file = std::fs::File::create(&path).unwrap();
write!(
file,
r#"{{"model":{{"hidden_size":32,"num_heads":4,"num_layers":1,"intermediate_size":64}},"trainer":{{"sequence_length":16}}}}"#
)
.unwrap();
let loaded = load_train_config_file(&path).unwrap();
std::fs::remove_file(&path).unwrap();
let config = resolve_train_config(Some(&loaded), &TrainConfigOverrides::default()).unwrap();
assert_eq!(config.hidden_size, 32);
assert_eq!(config.sequence_length, 16);
assert_eq!(config.max_sequence_length, 16);
}
#[test]
fn metrics_jsonl_override_wins_over_config_file() {
let file: TrainConfigFile = serde_json::from_str(
r#"{"trainer":{"metrics_jsonl":"from-file.jsonl"}}"#,
)
.unwrap();
let resolved = resolve_train_config(Some(&file), &TrainConfigOverrides::default()).unwrap();
assert_eq!(
resolved.metrics_jsonl.as_deref(),
Some(std::path::Path::new("from-file.jsonl"))
);
let overrides = TrainConfigOverrides {
metrics_jsonl: Some(std::path::PathBuf::from("from-cli.jsonl")),
..TrainConfigOverrides::default()
};
let resolved = resolve_train_config(Some(&file), &overrides).unwrap();
assert_eq!(
resolved.metrics_jsonl.as_deref(),
Some(std::path::Path::new("from-cli.jsonl"))
);
}
#[test]
fn validate_training_plan_refuses_tiny_memory_budget() {
let resolved = resolve_train_config(None, &TrainConfigOverrides::default()).unwrap();
let model = resolved.causal_lm_config(resolved.vocab_size, 0);
let trainer = resolved.trainer_config();
let schedule = LearningRateSchedule {
base_learning_rate: resolved.base_learning_rate,
minimum_learning_rate: resolved.minimum_learning_rate,
warmup_steps: resolved.warmup_steps,
total_steps: resolved.steps,
};
let error = validate_training_plan(
&model,
&trainer,
&schedule,
&AdamWConfig::default(),
&TrainingBackend::Cpu,
Some(1),
)
.unwrap_err();
match error {
ConfigError::MemoryBudget(error) => {
let message = error.to_string();
assert!(message.contains("remediation:"));
assert!(message.contains("--fit-memory"));
}
other => panic!("unexpected error: {other}"),
}
}
#[test]
fn fit_training_memory_enables_checkpointing_to_meet_budget() {
use super::fit_training_memory;
let mut overrides = TrainConfigOverrides::default();
overrides.hidden_size = Some(128);
overrides.num_layers = Some(4);
overrides.num_heads = Some(4);
overrides.num_key_value_heads = Some(2);
overrides.intermediate_size = Some(384);
overrides.sequence_length = Some(64);
overrides.max_sequence_length = Some(64);
overrides.batch_size = Some(2);
overrides.activation_checkpointing = Some(false);
let resolved = resolve_train_config(None, &overrides).unwrap();
let model = resolved.causal_lm_config(resolved.vocab_size, 0);
let trainer = resolved.trainer_config();
let baseline = crate::memory::estimate_training_memory(&model, &trainer);
let tight = baseline.total_bytes.saturating_mul(90) / 100;
let fitted = fit_training_memory(resolved, tight.max(1)).unwrap();
assert!(fitted.config.activation_checkpointing || fitted.config.batch_size == 1);
assert!(fitted.estimate.total_bytes <= tight.saturating_mul(95) / 100 + tight);
}
#[test]
fn gpu_tiny_host_safeguards_prefer_offload_and_microbatch() {
use super::{apply_gpu_tiny_host_safeguards, fit_training_memory_for_device};
let mut overrides = TrainConfigOverrides::default();
overrides.batch_size = Some(4);
overrides.gradient_accumulation_steps = Some(2);
overrides.layer_offload = Some(false);
overrides.offload_moments = Some(false);
let mut resolved = resolve_train_config(None, &overrides).unwrap();
let applied = apply_gpu_tiny_host_safeguards(&mut resolved);
assert!(applied.iter().any(|line| line.contains("batch_size=1")));
assert_eq!(resolved.batch_size, 1);
assert_eq!(resolved.gradient_accumulation_steps, 8);
assert!(resolved.low_memory.layer_offload);
assert!(resolved.low_memory.trim_caches_each_microbatch);
assert!(resolved.optimizer.offload_moments);
let mut overrides = TrainConfigOverrides::default();
overrides.hidden_size = Some(32);
overrides.num_layers = Some(1);
overrides.num_heads = Some(4);
overrides.num_key_value_heads = Some(2);
overrides.intermediate_size = Some(64);
overrides.sequence_length = Some(16);
overrides.max_sequence_length = Some(16);
overrides.batch_size = Some(1);
overrides.activation_checkpointing = Some(true);
overrides.prefetch_batches = Some(0);
overrides.mixed_precision = Some(true);
overrides.amp_format = Some(AmpFormat::Bf16);
overrides.layer_offload = Some(false);
overrides.offload_moments = Some(true);
let resolved = resolve_train_config(None, &overrides).unwrap();
let model = resolved.causal_lm_config(resolved.vocab_size, 0);
let trainer = resolved.trainer_config();
let mut mirrored = crate::memory::estimate_training_memory_detailed(
&model,
&trainer,
&resolved.optimizer,
resolved.prefetch_batches,
);
crate::memory::apply_gpu_parameter_residency(&mut mirrored, &model, false);
let mut offloaded = crate::memory::estimate_training_memory_detailed(
&model,
&trainer,
&resolved.optimizer,
resolved.prefetch_batches,
);
crate::memory::apply_gpu_parameter_residency(&mut offloaded, &model, true);
assert!(mirrored.total_bytes > offloaded.total_bytes);
let min_pass = offloaded
.total_bytes
.saturating_mul(100)
.saturating_add(94)
/ 95;
let max_fail_mirror = mirrored.total_bytes.saturating_mul(100) / 95;
assert!(min_pass < max_fail_mirror);
let tight = (min_pass + max_fail_mirror) / 2;
let fitted = fit_training_memory_for_device(resolved, tight.max(1), true).unwrap();
assert!(fitted.config.low_memory.layer_offload);
assert_eq!(fitted.config.batch_size, 1);
assert!(fitted.estimate.total_bytes <= tight.saturating_mul(95) / 100);
}
fn low_ram_checkpoint_configs() -> (TrainerConfig, AdamWConfig) {
(
TrainerConfig {
batch_size: 1,
sequence_length: 32,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 8,
mixed_precision: MixedPrecisionConfig::enabled_fp16(),
activation_checkpointing: true,
data_parallel_workers: 1,
low_memory: LowMemoryProfile {
activation_checkpoint_interval: 2,
layer_offload: true,
trim_caches_each_microbatch: true,
..Default::default()
},
},
AdamWConfig {
offload_moments: true,
..Default::default()
},
)
}
#[test]
fn resume_accepts_matching_low_ram_flags_and_ignores_unset() {
let (trainer, optimizer) = low_ram_checkpoint_configs();
validate_resume_low_ram_flags(
&trainer,
&optimizer,
None,
&TrainConfigOverrides::default(),
)
.unwrap();
let overrides = TrainConfigOverrides {
activation_checkpoint_interval: Some(2),
layer_offload: Some(true),
offload_moments: Some(true),
mixed_precision: Some(true),
amp_format: Some(AmpFormat::Fp16),
..TrainConfigOverrides::default()
};
validate_resume_low_ram_flags(&trainer, &optimizer, None, &overrides).unwrap();
}
#[test]
fn resume_rejects_mismatched_low_ram_flags() {
let (trainer, optimizer) = low_ram_checkpoint_configs();
let overrides = TrainConfigOverrides {
activation_checkpoint_interval: Some(1),
..TrainConfigOverrides::default()
};
let error =
validate_resume_low_ram_flags(&trainer, &optimizer, None, &overrides).unwrap_err();
let message = error.to_string();
assert!(message.contains("resume low-RAM flag mismatch"));
assert!(message.contains("activation_checkpoint_interval"));
let file: TrainConfigFile = serde_json::from_str(
r#"{ "optimizer": { "offload_moments": false } }"#,
)
.unwrap();
let error = validate_resume_low_ram_flags(
&trainer,
&optimizer,
Some(&file),
&TrainConfigOverrides::default(),
)
.unwrap_err();
assert!(error.to_string().contains("offload_moments"));
let overrides = TrainConfigOverrides {
amp_format: Some(AmpFormat::Bf16),
..TrainConfigOverrides::default()
};
let error =
validate_resume_low_ram_flags(&trainer, &optimizer, None, &overrides).unwrap_err();
assert!(error.to_string().contains("amp_format"));
}
#[test]
fn init_train_config_writes_preset_dims_under_large_budget() {
use crate::model::ArchitecturePreset;
use super::{init_train_config, resolve_train_config, write_train_config_file};
use std::path::PathBuf;
let init = init_train_config(Some(ArchitecturePreset::LlamaTiny), 32 * 1024 * 1024 * 1024, false)
.unwrap();
assert_eq!(init.config.architecture, Some(ArchitecturePreset::LlamaTiny));
assert_eq!(init.config.hidden_size, 128);
assert_eq!(init.config.sequence_length, 256);
assert_eq!(init.file.model.architecture, Some(ArchitecturePreset::LlamaTiny));
assert_eq!(init.file.model.hidden_size, Some(128));
assert_eq!(init.file.trainer.sequence_length, Some(256));
assert!(init.estimate.total_bytes <= init.budget_bytes);
let path = std::env::temp_dir().join(format!(
"oxidelm-init-train-{}.json",
std::process::id()
));
write_train_config_file(&path, &init.file).unwrap();
let loaded = super::load_train_config_file(&path).unwrap();
let resolved = resolve_train_config(Some(&loaded), &TrainConfigOverrides::default()).unwrap();
assert_eq!(resolved.hidden_size, init.config.hidden_size);
assert_eq!(resolved.num_layers, init.config.num_layers);
assert_eq!(resolved.sequence_length, init.config.sequence_length);
let _ = std::fs::remove_file(PathBuf::from(&path));
}
#[test]
fn init_train_config_fits_tight_budget() {
use crate::model::ArchitecturePreset;
use super::init_train_config;
let loose = init_train_config(Some(ArchitecturePreset::GptTiny), usize::MAX / 4, false)
.unwrap();
let tight_budget = (loose.estimate.total_bytes / 4).max(1);
let fitted =
init_train_config(Some(ArchitecturePreset::GptTiny), tight_budget, false).unwrap();
assert!(fitted.estimate.total_bytes <= fitted.budget_bytes);
assert!(!fitted.applied.is_empty() || fitted.config.activation_checkpointing);
}
#[test]
fn train_config_json_schema_has_sections() {
let schema = super::train_config_json_schema();
assert_eq!(schema["title"], "OxideLM TrainConfigFile");
assert!(schema["properties"]["model"].is_object());
assert!(schema["properties"]["trainer"].is_object());
assert!(schema["properties"]["optimizer"].is_object());
assert_eq!(
schema["properties"]["model"]["properties"]["architecture"]["enum"][0],
"llama-tiny"
);
}
}