use memmap2::MmapOptions;
#[cfg(feature = "training")]
use oxidelm::amp::AmpFormat;
use oxidelm::anomaly::{self, parse_detect_anomaly};
#[cfg(feature = "training")]
use oxidelm::config::{
apply_gpu_tiny_host_safeguards, fit_training_memory_for_device, format_steps_clarification,
init_train_config, load_train_config_file, resolve_train_config, train_config_json_schema,
validate_resume_low_ram_flags, validate_training_plan, write_train_config_file,
TrainConfigOverrides,
};
#[cfg(feature = "training")]
use oxidelm::layers::WeightQuantization;
#[cfg(feature = "training")]
use oxidelm::optimizer::{MomentDtype, OptimizerKind};
#[cfg(feature = "training")]
use oxidelm::trainer::SelectiveCheckpoint;
#[cfg(feature = "training")]
use oxidelm::corpus::resolve_corpus;
#[cfg(feature = "training")]
use oxidelm::data_loader::{TokenDataset, TokenShardWriter};
use oxidelm::analyze::{
analyze_diff, analyze_path, format_analyze_diff_report, format_analyze_report,
write_analyze_diff_json, write_analyze_json, AnalyzeFormat, AnalyzeOptions, FailOn,
};
use oxidelm::generate::{Generator, SamplingConfig};
use oxidelm::gpu::{ExecutionDevice, GpuCacheLimits, GpuContext};
use oxidelm::inspect::{format_report, inspect_checkpoint};
use oxidelm::log::{self, parse_log_mode, LogMode, LogValue};
use oxidelm::memory::{
check_memory_budget, estimate_inference_memory, format_memory_report, DeviceBudget,
};
#[cfg(feature = "training")]
use oxidelm::memory::{cap_gpu_budget_for_tiny_host, default_cpu_budget_bytes, host_memory_is_tiny};
#[cfg(feature = "training")]
use oxidelm::model::CausalLM;
use oxidelm::profile::{self, parse_profile_mode, ProfileMode};
use oxidelm::quality::{
builtin_tiny_tasks, evaluate_quality, format_report as format_quality_report, load_tasks,
EvalTask, QualityReport,
};
use oxidelm::quantize::{
format_quantize_report, parse_pattern_list, quantize_checkpoint, QuantizeMethod,
QuantizeOptions,
};
use oxidelm::state_dict::{
export_checkpoint_weights, export_quantized_safetensors, import_checkpoint_weights,
import_quantized_safetensors, WeightFormat,
};
use oxidelm::gguf::{
export_checkpoint_gguf, import_gguf_checkpoint, GgmlType, GgufExportOptions, GgufImportOptions,
};
use oxidelm::tokenizer::BpeTokenizer;
#[cfg(feature = "training")]
use oxidelm::tokenizer::PreTokenization;
use oxidelm::tokenizer_quality::{
evaluate_bytes, format_report as format_tokenizer_quality_report,
};
use oxidelm::trainer::load_checkpoint;
#[cfg(feature = "training")]
use oxidelm::trainer::{plan_training_memory, Trainer, TrainingBackend};
use std::error::Error;
use std::fs::File;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
fn main() -> Result<(), Box<dyn Error>> {
profile::init_from_env();
anomaly::init_from_env();
let arguments: Vec<String> = std::env::args().collect();
let (mode, profile_mode, detect_anomaly, rest) = take_global_flags(&arguments[1..])?;
if let Some(mode) = mode {
log::set_log_mode(mode);
}
if let Some(mode) = profile_mode {
profile::set_profile_mode(mode);
}
if let Some(enabled) = detect_anomaly {
anomaly::set_detect_anomaly(enabled);
}
let result = match rest.first().map(String::as_str) {
#[cfg(feature = "training")]
Some("--train") => train(&rest[1..]),
#[cfg(not(feature = "training"))]
Some("--train") => Err(
"the 'training' feature is disabled; rebuild with `--features training` to use --train"
.into(),
),
Some("--generate") => generate(&rest[1..]),
Some("--inspect") => inspect(&rest[1..]),
Some("--analyze") => analyze(&rest[1..]),
Some("--analyze-diff") => analyze_diff_cmd(&rest[1..]),
Some("--quantize") => quantize_cmd(&rest[1..]),
Some("--eval") => eval_quality(&rest[1..]),
Some("--tokenizer-report") => tokenizer_report(&rest[1..]),
Some("--import-tokenizer") => import_tokenizer(&rest[1..]),
Some("--export-weights") => export_weights(&rest[1..]),
Some("--import-weights") => import_weights(&rest[1..]),
Some("--export-quantized") => export_quantized(&rest[1..]),
Some("--import-quantized") => import_quantized(&rest[1..]),
Some("--export-gguf") => export_gguf(&rest[1..]),
Some("--import-gguf") => import_gguf(&rest[1..]),
Some("--export-adapters") => export_adapters(&rest[1..]),
Some("--import-adapters") => import_adapters(&rest[1..]),
Some("--export-qat") => export_qat_cmd(&rest[1..]),
_ => {
print_usage();
Ok(())
}
};
profile::flush();
result
}
fn take_global_flags(
arguments: &[String],
) -> Result<
(
Option<LogMode>,
Option<ProfileMode>,
Option<bool>,
Vec<String>,
),
Box<dyn Error>,
> {
let mut mode = None;
let mut profile_mode = None;
let mut detect_anomaly = None;
let mut rest = Vec::with_capacity(arguments.len());
let mut index = 0;
while index < arguments.len() {
match arguments[index].as_str() {
"--log" => {
let value = arguments.get(index + 1).ok_or("missing value for --log")?;
mode = Some(parse_log_mode(value)?);
index += 2;
}
"--profile" => {
let next = arguments.get(index + 1).map(String::as_str);
if next.is_some_and(|value| {
!value.starts_with("--") && parse_profile_mode(value).is_ok()
}) {
profile_mode = Some(parse_profile_mode(next.unwrap())?);
index += 2;
} else {
profile_mode = Some(ProfileMode::Summary);
index += 1;
}
}
"--detect-anomaly" => {
let next = arguments.get(index + 1).map(String::as_str);
if next.is_some_and(|value| {
!value.starts_with("--") && parse_detect_anomaly(value).is_ok()
}) {
detect_anomaly = Some(parse_detect_anomaly(next.unwrap())?);
index += 2;
} else {
detect_anomaly = Some(true);
index += 1;
}
}
_ => {
rest.push(arguments[index].clone());
index += 1;
}
}
}
Ok((mode, profile_mode, detect_anomaly, rest))
}
#[cfg(feature = "training")]
fn train(arguments: &[String]) -> Result<(), Box<dyn Error>> {
if arguments.first().map(String::as_str) == Some("init") {
return train_init(&arguments[1..]);
}
let (config_file, overrides) = parse_train_config_args(arguments)?;
let positionals = train_positionals(arguments);
let dry_run = arguments
.iter()
.any(|argument| argument == "--dry-run" || argument == "--plan-only");
let eval_only = arguments.iter().any(|argument| argument == "--eval-only");
let (fit_memory, fit_budget) = parse_fit_memory(arguments)?;
let memory_budget_bytes = parse_memory_budget(arguments)?;
let input_path = positionals.first().copied();
let checkpoint_path = positionals.get(1).copied().unwrap_or("model.bin");
let resume = arguments.iter().any(|argument| argument == "--resume");
let use_gpu = arguments.iter().any(|argument| argument == "--gpu");
if use_gpu {
require_gpu_feature()?;
}
if eval_only && dry_run {
return Err("--eval-only cannot be combined with --dry-run/--plan-only".into());
}
let file = match config_file {
Some(path) => Some(load_train_config_file(path).map_err(cli_error)?),
None => None,
};
let mut resolved = resolve_train_config(file.as_ref(), &overrides).map_err(cli_error)?;
if !resume {
emit_steps_clarification(
overrides.steps,
file.as_ref().and_then(|config| config.schedule.total_steps),
resolved.steps,
false,
eval_only,
None,
None,
);
}
if eval_only {
if let Some(steps) = overrides.steps {
log::emit_message(
"steps",
&format!("--eval-only ignores --steps/--max-steps={steps}"),
);
}
}
// --gpu + tiny host/unified RAM: prefer CPU offload + microbatch 1 over refusing.
if use_gpu && host_memory_is_tiny() {
let safeguards = apply_gpu_tiny_host_safeguards(&mut resolved);
if !safeguards.is_empty() {
log::emit_message(
"gpu_low_host_ram",
"tiny host RAM detected; enabling CPU offload + microbatch 1",
);
for change in &safeguards {
log::emit_message("gpu_low_host_ram", change);
}
}
}
let effective_memory_budget = if fit_memory {
let budget = fit_budget
.or(memory_budget_bytes)
.unwrap_or_else(default_cpu_budget_bytes);
let budget = if use_gpu {
cap_gpu_budget_for_tiny_host(Some(budget)).unwrap_or(budget)
} else {
budget
};
let fitted =
fit_training_memory_for_device(resolved, budget, use_gpu).map_err(cli_error)?;
if fitted.applied.is_empty() {
log::emit_message(
"fit_memory",
&format!(
"config already fits budget {}",
oxidelm::memory::format_bytes(budget)
),
);
} else {
for change in &fitted.applied {
log::emit_message("fit_memory", change);
}
}
resolved = fitted.config;
Some(budget)
} else if use_gpu {
cap_gpu_budget_for_tiny_host(memory_budget_bytes)
} else {
memory_budget_bytes
};
let best_checkpoint_path = option_value(arguments, "--best-checkpoint")
.map(PathBuf::from)
.unwrap_or_else(|| default_best_checkpoint_path(checkpoint_path));
if best_checkpoint_path == Path::new(checkpoint_path) {
return Err("best checkpoint path must differ from final checkpoint path".into());
}
if dry_run {
return train_dry_run(&resolved, use_gpu, effective_memory_budget, config_file);
}
let input_path = input_path.ok_or("missing training text file or directory")?;
let token_cache = option_value(arguments, "--token-cache")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(format!("{checkpoint_path}.tokens")));
if resolved.low_memory.rebuild_token_cache
|| arguments
.iter()
.any(|argument| argument == "--rebuild-token-cache")
{
if token_cache.exists() {
std::fs::remove_dir_all(&token_cache)?;
let path_display = token_cache.display().to_string();
log::emit(
"token_cache",
&[
("action", LogValue::Str("rebuild")),
("path", LogValue::Str(&path_display)),
],
);
}
}
let corpus_cache = token_cache.join("source");
let resolved_corpus = resolve_corpus(input_path, &corpus_cache)?;
let input_display = resolved_corpus.input.display().to_string();
let corpus_path_display = resolved_corpus.path.display().to_string();
log::emit(
"corpus",
&[
("input", LogValue::Str(&input_display)),
("files", LogValue::Usize(resolved_corpus.file_count())),
(
"materialized",
LogValue::Bool(resolved_corpus.materialized),
),
("path", LogValue::Str(&corpus_path_display)),
(
"bytes",
LogValue::Usize(resolved_corpus.total_bytes() as usize),
),
],
);
let corpus_file = File::open(&resolved_corpus.path)?;
if corpus_file.metadata()?.len() < 4 {
return Err("corpus must contain at least four bytes for training and validation".into());
}
// SAFETY: The mapping is read-only and the source file remains open for the
// lifetime of the mapping.
let corpus = unsafe { MmapOptions::new().map(&corpus_file)? };
let validation_bytes = ((corpus.len() as f64 * resolved.validation_split).round() as usize)
.clamp(2, corpus.len() - 2);
let split_at = corpus.len() - validation_bytes;
let (training_corpus, validation_corpus) = corpus.split_at(split_at);
let validation_split_actual = validation_corpus.len() as f64 / corpus.len() as f64;
let training_cache = token_cache.join("train");
let validation_cache = token_cache.join("validation");
let execution = resolve_execution_device(use_gpu);
if use_gpu {
emit_backend_selection(&execution);
}
let backend = match execution {
ExecutionDevice::Gpu(gpu) => TrainingBackend::Gpu(gpu),
ExecutionDevice::Cpu { .. } => TrainingBackend::Cpu,
};
let mut trainer = if resume {
let checkpoint = load_checkpoint(checkpoint_path)?;
validate_resume_low_ram_flags(
&checkpoint.trainer_config,
checkpoint.optimizer.config(),
file.as_ref(),
&overrides,
)
.map_err(cli_error)?;
let loss_scale = checkpoint
.loss_scaler
.as_ref()
.map(|state| state.scale)
.unwrap_or(1.0);
log::emit(
"resume",
&[
("checkpoint", LogValue::Str(checkpoint_path)),
("step", LogValue::Usize(checkpoint.training_step)),
("cursor", LogValue::Usize(checkpoint.cursor)),
(
"tokens_processed",
LogValue::Usize(checkpoint.tokens_processed),
),
(
"optimizer_step",
LogValue::Usize(checkpoint.optimizer.state().step),
),
(
"seed",
LogValue::Usize(checkpoint.model.config.seed as usize),
),
("loss_scale", LogValue::f32(loss_scale, 1)),
(
"activation_checkpoint_interval",
LogValue::Usize(
checkpoint
.trainer_config
.low_memory
.activation_checkpoint_interval,
),
),
(
"layer_offload",
LogValue::Bool(checkpoint.trainer_config.low_memory.layer_offload),
),
(
"offload_moments",
LogValue::Bool(checkpoint.optimizer.config().offload_moments),
),
],
);
log::emit_message(
"tokenizer",
&format!("restoring tokenizer from checkpoint {checkpoint_path}"),
);
let training_data = tokenize_to_shards(
&checkpoint.tokenizer,
training_corpus,
&training_cache,
resolved.shard_tokens,
resolved.tokenize_chunk_bytes,
)?;
let validation_data = tokenize_to_shards(
&checkpoint.tokenizer,
validation_corpus,
&validation_cache,
resolved.shard_tokens,
resolved.tokenize_chunk_bytes,
)?;
resolved.validate_corpus(training_data.token_count(), validation_data.token_count())?;
log::emit(
"dataset",
&[
("train_tokens", LogValue::Usize(training_data.token_count())),
(
"validation_tokens",
LogValue::Usize(validation_data.token_count()),
),
(
"validation_split",
LogValue::f64(validation_split_actual, 3),
),
(
"prefetch_batches",
LogValue::Usize(resolved.prefetch_batches),
),
],
);
let (memory_estimate, memory_budget) = plan_training_memory(
&checkpoint.model.config,
&checkpoint.trainer_config,
&backend,
effective_memory_budget,
);
log::emit_message(
"memory",
&format_memory_report(&memory_estimate, Some(&memory_budget)),
);
if use_gpu {
emit_gpu_cache_limits(effective_memory_budget);
}
let checkpoint_step = checkpoint.training_step;
let checkpoint_total_steps = checkpoint.schedule.total_steps;
let mut trainer = Trainer::from_checkpoint_with_dataset_and_memory_budget(
checkpoint,
training_data,
backend,
effective_memory_budget,
resolved.prefetch_batches,
)
.map_err(cli_error)?;
emit_steps_clarification(
overrides.steps,
file.as_ref().and_then(|config| config.schedule.total_steps),
resolved.steps,
true,
eval_only,
Some(checkpoint_step),
Some(checkpoint_total_steps),
);
if !eval_only {
if let Some((previous, next)) =
trainer.extend_schedule_total_steps(resolved.steps)
{
log::emit_message(
"steps",
&format!(
"extended schedule.total_steps {previous} -> {next} for {} additional resume steps",
resolved.steps
),
);
}
}
trainer.configure_evaluation_dataset(
validation_data,
resolved.eval_every,
&best_checkpoint_path,
)?;
trainer.configure_early_stopping(
resolved.early_stopping_patience,
resolved.early_stopping_min_delta,
)?;
trainer
} else {
if let Some(path) = config_file {
log::emit("config", &[("file", LogValue::Str(path))]);
}
let architecture = resolved
.architecture
.map(|preset| preset.as_str())
.unwrap_or("custom");
log::emit(
"config",
&[
("architecture", LogValue::Str(architecture)),
("hidden_size", LogValue::Usize(resolved.hidden_size)),
("num_layers", LogValue::Usize(resolved.num_layers)),
("num_heads", LogValue::Usize(resolved.num_heads)),
(
"num_key_value_heads",
LogValue::Usize(resolved.num_key_value_heads),
),
(
"intermediate_size",
LogValue::Usize(resolved.intermediate_size),
),
("sequence_length", LogValue::Usize(resolved.sequence_length)),
("batch_size", LogValue::Usize(resolved.batch_size)),
(
"gradient_accumulation_steps",
LogValue::Usize(resolved.gradient_accumulation_steps),
),
(
"effective_batch_size",
LogValue::Usize(
resolved.batch_size * resolved.gradient_accumulation_steps,
),
),
(
"activation_checkpointing",
LogValue::Bool(resolved.activation_checkpointing),
),
(
"mixed_precision",
LogValue::Bool(resolved.mixed_precision.enabled),
),
(
"amp_format",
LogValue::Str(resolved.mixed_precision.format.as_str()),
),
(
"initial_loss_scale",
LogValue::f32(resolved.mixed_precision.initial_loss_scale, 1),
),
("vocab_size", LogValue::Usize(resolved.vocab_size)),
("steps", LogValue::Usize(resolved.steps)),
("seed", LogValue::Usize(resolved.seed as usize)),
("use_bias", LogValue::Bool(resolved.use_bias)),
(
"attention_dropout",
LogValue::f32(resolved.attention_dropout, 3),
),
(
"residual_dropout",
LogValue::f32(resolved.residual_dropout, 3),
),
(
"embedding_dropout",
LogValue::f32(resolved.embedding_dropout, 3),
),
],
);
log::emit_message(
"tokenizer",
&format!("training tokenizer from held-out training partition of {input_display}"),
);
let tokenizer = BpeTokenizer::train_from_bytes_with_options(
training_corpus,
resolved.vocab_size,
resolved.min_frequency,
resolved.pre_tokenization.clone(),
)?;
emit_tokenizer_quality(&tokenizer, validation_corpus, 3);
let training_data = tokenize_to_shards(
&tokenizer,
training_corpus,
&training_cache,
resolved.shard_tokens,
resolved.tokenize_chunk_bytes,
)?;
let validation_data = tokenize_to_shards(
&tokenizer,
validation_corpus,
&validation_cache,
resolved.shard_tokens,
resolved.tokenize_chunk_bytes,
)?;
resolved.validate_corpus(training_data.token_count(), validation_data.token_count())?;
log::emit(
"dataset",
&[
("train_tokens", LogValue::Usize(training_data.token_count())),
(
"validation_tokens",
LogValue::Usize(validation_data.token_count()),
),
(
"validation_split",
LogValue::f64(validation_split_actual, 3),
),
(
"prefetch_batches",
LogValue::Usize(resolved.prefetch_batches),
),
],
);
let model_config =
resolved.causal_lm_config(tokenizer.vocab_size(), tokenizer.special_tokens.eos);
let trainer_config = resolved.trainer_config();
let schedule = resolved.schedule();
let (memory_estimate, memory_budget) = validate_training_plan(
&model_config,
&trainer_config,
&schedule,
&resolved.optimizer,
&backend,
effective_memory_budget,
)
.map_err(cli_error)?;
log::emit_message(
"memory",
&format_memory_report(&memory_estimate, Some(&memory_budget)),
);
if use_gpu {
emit_gpu_cache_limits(effective_memory_budget);
}
let model = CausalLM::new(model_config)?;
let mut trainer = Trainer::new_with_dataset_and_memory_budget(
model,
tokenizer,
training_data,
resolved.optimizer.clone(),
schedule,
trainer_config,
backend,
effective_memory_budget,
resolved.prefetch_batches,
)
.map_err(cli_error)?;
trainer.configure_evaluation_dataset(
validation_data,
resolved.eval_every,
&best_checkpoint_path,
)?;
trainer.configure_early_stopping(
resolved.early_stopping_patience,
resolved.early_stopping_min_delta,
)?;
trainer
};
if eval_only {
trainer.evaluate_and_save_best()?;
let best_checkpoint_display = best_checkpoint_path.display().to_string();
log::emit(
"checkpoint",
&[
("final_checkpoint", LogValue::Str(checkpoint_path)),
("best_checkpoint", LogValue::Str(&best_checkpoint_display)),
("eval_only", LogValue::Bool(true)),
],
);
return Ok(());
}
if let Some(path) = &resolved.metrics_jsonl {
trainer.configure_metrics_jsonl(path)?;
let path_display = path.display().to_string();
log::emit(
"metrics",
&[("jsonl", LogValue::Str(&path_display))],
);
}
trainer.train_steps(resolved.steps)?;
if trainer.stopped_early() {
log::emit(
"early_stop",
&[
("step", LogValue::Usize(trainer.step())),
(
"stale_evals",
LogValue::Usize(trainer.evaluations_without_improvement()),
),
(
"patience",
LogValue::Usize(resolved.early_stopping_patience),
),
],
);
} else if trainer.last_evaluation_step() != Some(trainer.step()) {
trainer.evaluate_and_save_best()?;
}
trainer.save_checkpoint(checkpoint_path)?;
let best_checkpoint_display = best_checkpoint_path.display().to_string();
log::emit(
"checkpoint",
&[
("final_checkpoint", LogValue::Str(checkpoint_path)),
("best_checkpoint", LogValue::Str(&best_checkpoint_display)),
],
);
Ok(())
}
#[cfg(feature = "training")]
fn train_dry_run(
resolved: &oxidelm::config::ResolvedTrainConfig,
use_gpu: bool,
memory_budget_bytes: Option<usize>,
config_file: Option<&str>,
) -> Result<(), Box<dyn Error>> {
if let Some(path) = config_file {
log::emit("config", &[("file", LogValue::Str(path))]);
}
let architecture = resolved
.architecture
.map(|preset| preset.as_str())
.unwrap_or("custom");
log::emit(
"config",
&[
("architecture", LogValue::Str(architecture)),
("hidden_size", LogValue::Usize(resolved.hidden_size)),
("num_layers", LogValue::Usize(resolved.num_layers)),
("num_heads", LogValue::Usize(resolved.num_heads)),
(
"num_key_value_heads",
LogValue::Usize(resolved.num_key_value_heads),
),
(
"intermediate_size",
LogValue::Usize(resolved.intermediate_size),
),
("sequence_length", LogValue::Usize(resolved.sequence_length)),
("batch_size", LogValue::Usize(resolved.batch_size)),
(
"gradient_accumulation_steps",
LogValue::Usize(resolved.gradient_accumulation_steps),
),
(
"effective_batch_size",
LogValue::Usize(resolved.batch_size * resolved.gradient_accumulation_steps),
),
(
"activation_checkpointing",
LogValue::Bool(resolved.activation_checkpointing),
),
(
"mixed_precision",
LogValue::Bool(resolved.mixed_precision.enabled),
),
(
"amp_format",
LogValue::Str(resolved.mixed_precision.format.as_str()),
),
("vocab_size", LogValue::Usize(resolved.vocab_size)),
("steps", LogValue::Usize(resolved.steps)),
(
"prefetch_batches",
LogValue::Usize(resolved.prefetch_batches),
),
],
);
let execution = resolve_execution_device(use_gpu);
if use_gpu {
emit_backend_selection(&execution);
}
let backend = match execution {
ExecutionDevice::Gpu(gpu) => TrainingBackend::Gpu(gpu),
ExecutionDevice::Cpu { .. } => TrainingBackend::Cpu,
};
let model_config = resolved.causal_lm_config(resolved.vocab_size, 0);
let trainer_config = resolved.trainer_config();
let schedule = resolved.schedule();
let (memory_estimate, memory_budget) = validate_training_plan(
&model_config,
&trainer_config,
&schedule,
&resolved.optimizer,
&backend,
memory_budget_bytes,
)
.map_err(cli_error)?;
log::emit_message(
"memory",
&format_memory_report(&memory_estimate, Some(&memory_budget)),
);
if use_gpu {
emit_gpu_cache_limits(memory_budget_bytes);
}
log::emit_message(
"plan",
"dry-run complete; no corpus tokenization or weight allocation",
);
Ok(())
}
#[cfg(feature = "training")]
fn train_positionals(arguments: &[String]) -> Vec<&str> {
let mut positionals = Vec::new();
let mut index = 0;
while index < arguments.len() {
let argument = arguments[index].as_str();
if !argument.starts_with("--") {
positionals.push(argument);
index += 1;
continue;
}
match argument {
"--resume"
| "--gpu"
| "--dry-run"
| "--plan-only"
| "--eval-only"
| "--tie-embeddings"
| "--no-tie-embeddings"
| "--use-bias"
| "--no-bias"
| "--activation-checkpointing"
| "--grad-checkpoint"
| "--no-activation-checkpointing"
| "--no-grad-checkpoint"
| "--mixed-precision"
| "--amp"
| "--no-mixed-precision"
| "--no-amp"
| "--no-decay-rms-norm"
| "--decay-rms-norm"
| "--decay-embeddings" => index += 1,
"--fit-memory" => {
let next = arguments.get(index + 1).map(String::as_str);
if next.is_some_and(|value| !value.starts_with("--") && parse_byte_size(value).is_ok())
{
index += 2;
} else {
index += 1;
}
}
_ => index += 2,
}
}
positionals
}
#[cfg(feature = "training")]
fn train_init(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let schema_only = arguments.iter().any(|argument| argument == "--schema");
let force = arguments.iter().any(|argument| argument == "--force");
let use_gpu = arguments.iter().any(|argument| argument == "--gpu");
if use_gpu {
require_gpu_feature()?;
}
let mut architecture = None;
let mut output: Option<&str> = None;
let mut index = 0;
while index < arguments.len() {
let argument = arguments[index].as_str();
match argument {
"--schema" | "--force" | "--gpu" => index += 1,
"--architecture" | "--arch" | "--preset" => {
architecture = Some(require_value(arguments, index, argument)?.parse()?);
index += 2;
}
"--memory-budget" => {
require_value(arguments, index, argument)?;
index += 2;
}
flag if flag.starts_with("--") => {
return Err(format!("unknown train init flag: {flag}").into());
}
positional => {
if output.is_some() {
return Err("train init accepts at most one output path".into());
}
output = Some(positional);
index += 1;
}
}
}
if schema_only {
let path = output.unwrap_or("training/schema.json");
write_schema_file(path, force)?;
log::emit_message("train_init", &format!("wrote config schema to {path}"));
return Ok(());
}
let path = output.unwrap_or("train.json");
if Path::new(path).exists() && !force {
return Err(format!(
"refusing to overwrite existing {path}; pass --force or choose another path"
)
.into());
}
if let Some(parent) = Path::new(path).parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent)?;
}
}
let memory_budget_bytes = parse_memory_budget(arguments)?
.or_else(|| {
if use_gpu {
cap_gpu_budget_for_tiny_host(None)
} else {
None
}
})
.unwrap_or_else(default_cpu_budget_bytes);
let init = init_train_config(architecture, memory_budget_bytes, use_gpu).map_err(cli_error)?;
write_train_config_file(path, &init.file).map_err(cli_error)?;
let preset = architecture
.unwrap_or(oxidelm::model::ArchitecturePreset::LlamaTiny)
.as_str();
log::emit_message(
"train_init",
&format!(
"wrote starter config to {path} (preset={preset}, budget={})",
oxidelm::memory::format_bytes(init.budget_bytes)
),
);
if init.applied.is_empty() {
log::emit_message(
"train_init",
&format!(
"preset already fits budget {}",
oxidelm::memory::format_bytes(init.budget_bytes)
),
);
} else {
for change in &init.applied {
log::emit_message("fit_memory", change);
}
}
log::emit_message(
"memory_plan",
&oxidelm::memory::format_memory_report(
&init.estimate,
Some(&oxidelm::memory::DeviceBudget::cpu(Some(init.budget_bytes))),
),
);
Ok(())
}
#[cfg(feature = "training")]
fn write_schema_file(path: &str, force: bool) -> Result<(), Box<dyn Error>> {
if Path::new(path).exists() && !force {
return Err(format!(
"refusing to overwrite existing {path}; pass --force or choose another path"
)
.into());
}
if let Some(parent) = Path::new(path).parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent)?;
}
}
let mut out = File::create(path)?;
serde_json::to_writer_pretty(&mut out, &train_config_json_schema())?;
writeln!(out)?;
Ok(())
}
#[cfg(feature = "training")]
fn tokenize_to_shards(
tokenizer: &BpeTokenizer,
bytes: &[u8],
directory: impl AsRef<Path>,
shard_tokens: usize,
chunk_bytes: usize,
) -> Result<TokenDataset, Box<dyn Error>> {
let mut writer = TokenShardWriter::create(directory, shard_tokens)?;
for chunk in bytes.chunks(chunk_bytes) {
let tokens = tokenizer.encode_bytes(chunk);
writer.extend(&tokens)?;
}
Ok(writer.finish()?)
}
#[cfg(feature = "training")]
fn parse_train_config_args(
arguments: &[String],
) -> Result<(Option<&str>, TrainConfigOverrides), Box<dyn Error>> {
let mut overrides = TrainConfigOverrides::default();
let mut config_file = None;
let mut index = 0;
while index < arguments.len() {
let argument = arguments[index].as_str();
if !argument.starts_with("--") {
index += 1;
continue;
}
match argument {
"--resume" | "--gpu" | "--dry-run" | "--plan-only" | "--eval-only" => {
index += 1;
}
"--fit-memory" => {
let next = arguments.get(index + 1).map(String::as_str);
if next.is_some_and(|value| !value.starts_with("--") && parse_byte_size(value).is_ok())
{
index += 2;
} else {
index += 1;
}
}
"--use-bias" => {
overrides.use_bias = Some(true);
index += 1;
}
"--no-bias" => {
overrides.use_bias = Some(false);
index += 1;
}
"--config" => {
config_file = Some(require_value(arguments, index, argument)?);
index += 2;
}
"--architecture" | "--arch" | "--preset" => {
overrides.architecture = Some(require_value(arguments, index, argument)?.parse()?);
index += 2;
}
"--best-checkpoint" | "--memory-budget" | "--token-cache" => {
require_value(arguments, index, argument)?;
index += 2;
}
"--metrics-jsonl" => {
overrides.metrics_jsonl =
Some(PathBuf::from(require_value(arguments, index, argument)?));
index += 2;
}
"--steps" | "--max-steps" => {
if overrides.steps.is_some() {
return Err(
"--steps and --max-steps are aliases; pass only one session step count"
.into(),
);
}
overrides.steps = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--validation-split" => {
overrides.validation_split = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--eval-every" => {
overrides.eval_every = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--early-stopping-patience" => {
overrides.early_stopping_patience = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--early-stopping-min-delta" => {
overrides.early_stopping_min_delta =
Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--shard-tokens" => {
overrides.shard_tokens = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--tokenize-chunk-bytes" => {
overrides.tokenize_chunk_bytes = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--prefetch-batches" => {
overrides.prefetch_batches = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--hidden-size" => {
overrides.hidden_size = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--num-layers" | "--layers" => {
overrides.num_layers = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--num-heads" | "--heads" => {
overrides.num_heads = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--num-key-value-heads" | "--kv-heads" => {
overrides.num_key_value_heads = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--intermediate-size" => {
overrides.intermediate_size = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--sequence-length" | "--seq-len" => {
overrides.sequence_length = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--max-sequence-length" | "--context" => {
overrides.max_sequence_length = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--batch-size" => {
overrides.batch_size = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--gradient-accumulation-steps" | "--grad-accum" => {
overrides.gradient_accumulation_steps =
Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--data-parallel-workers" | "--dp-workers" => {
overrides.data_parallel_workers = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--activation-checkpointing" | "--grad-checkpoint" => {
overrides.activation_checkpointing = Some(true);
index += 1;
}
"--no-activation-checkpointing" | "--no-grad-checkpoint" => {
overrides.activation_checkpointing = Some(false);
index += 1;
}
"--activation-checkpoint-interval" => {
overrides.activation_checkpoint_interval =
Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--selective-checkpoint" => {
overrides.selective_checkpoint = Some(SelectiveCheckpoint::parse(require_value(
arguments, index, argument,
)?)?);
index += 2;
}
"--attention-chunk-size" => {
overrides.attention_chunk_size = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--layer-offload" => {
overrides.layer_offload = Some(true);
index += 1;
}
"--trim-caches" => {
overrides.trim_caches_each_microbatch = Some(true);
index += 1;
}
"--rebuild-token-cache" => {
overrides.rebuild_token_cache = Some(true);
index += 1;
}
"--optimizer" => {
let value = require_value(arguments, index, argument)?;
overrides.optimizer_kind = Some(match value.trim().to_ascii_lowercase().as_str() {
"adamw" => OptimizerKind::AdamW,
"sgd" => OptimizerKind::Sgd,
"lion" => OptimizerKind::Lion,
"adafactor" => OptimizerKind::Adafactor,
other => {
return Err(format!(
"unknown optimizer '{other}'; expected adamw, sgd, lion, or adafactor"
)
.into());
}
});
index += 2;
}
"--moment-dtype" => {
let value = require_value(arguments, index, argument)?;
overrides.moment_dtype = Some(match value.trim().to_ascii_lowercase().as_str() {
"fp32" | "f32" => MomentDtype::Fp32,
"fp16" | "f16" | "half" => MomentDtype::Fp16,
other => {
return Err(format!(
"unknown moment dtype '{other}'; expected fp32 or fp16"
)
.into());
}
});
index += 2;
}
"--offload-moments" => {
overrides.offload_moments = Some(true);
index += 1;
}
"--freeze" => {
let value = require_value(arguments, index, argument)?;
overrides.freeze = Some(
value
.split(',')
.map(|part| part.trim().to_string())
.filter(|part| !part.is_empty())
.collect(),
);
index += 2;
}
"--qat" => {
let format = WeightQuantization::parse(require_value(
arguments, index, argument,
)?)?;
let mut qat = overrides.qat.take().unwrap_or_default();
qat.format = format;
overrides.qat = Some(qat);
index += 2;
}
"--qat-group-size" => {
let size: usize = parse_value(arguments, index, argument)?;
let mut qat = overrides.qat.take().unwrap_or_default();
qat.group_size = Some(size);
overrides.qat = Some(qat);
index += 2;
}
"--qat-learnable-scales" => {
let mut qat = overrides.qat.take().unwrap_or_default();
qat.learnable_scales = true;
overrides.qat = Some(qat);
index += 1;
}
"--qlora-base" => {
let format = WeightQuantization::parse(require_value(
arguments, index, argument,
)?)?;
let lora = overrides.lora.get_or_insert_with(|| oxidelm::peft::LoraConfig {
rank: 8,
alpha: 16.0,
dropout: 0.0,
dora: false,
targets: vec!["q_proj".into(), "v_proj".into()],
base_quant: None,
base_group_size: None,
});
lora.base_quant = Some(format);
index += 2;
}
"--qlora-group-size" => {
let size: usize = parse_value(arguments, index, argument)?;
let lora = overrides.lora.get_or_insert_with(|| oxidelm::peft::LoraConfig {
rank: 8,
alpha: 16.0,
dropout: 0.0,
dora: false,
targets: vec!["q_proj".into(), "v_proj".into()],
base_quant: None,
base_group_size: None,
});
lora.base_group_size = Some(size);
index += 2;
}
"--mixed-precision" | "--amp" => {
overrides.mixed_precision = Some(true);
index += 1;
}
"--no-mixed-precision" | "--no-amp" => {
overrides.mixed_precision = Some(false);
index += 1;
}
"--amp-format" => {
overrides.amp_format = Some(AmpFormat::parse(require_value(
arguments, index, argument,
)?)?);
index += 2;
}
"--initial-loss-scale" | "--loss-scale" => {
overrides.initial_loss_scale = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--vocab-size" => {
overrides.vocab_size = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--min-frequency" => {
overrides.min_frequency = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--pre-tokenization" => {
overrides.pre_tokenization =
Some(parse_pre_tokenization(require_value(arguments, index, argument)?)?);
index += 2;
}
"--seed" => {
overrides.seed = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--initializer-std" => {
overrides.initializer_std = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--attention-dropout" => {
overrides.attention_dropout = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--residual-dropout" => {
overrides.residual_dropout = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--embedding-dropout" => {
overrides.embedding_dropout = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--rms-norm-epsilon" => {
overrides.rms_norm_epsilon = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--rope-theta" => {
overrides.rope_theta = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--tie-embeddings" => {
overrides.tie_word_embeddings = Some(true);
index += 1;
}
"--no-tie-embeddings" => {
overrides.tie_word_embeddings = Some(false);
index += 1;
}
"--learning-rate" | "--lr" => {
overrides.base_learning_rate = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--min-learning-rate" => {
overrides.minimum_learning_rate = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--warmup-steps" => {
overrides.warmup_steps = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--weight-decay" => {
overrides.weight_decay = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--no-decay-rms-norm" => {
overrides.no_decay_rms_norm = Some(true);
index += 1;
}
"--decay-rms-norm" => {
overrides.no_decay_rms_norm = Some(false);
index += 1;
}
"--no-decay-embeddings" => {
let value = require_value(arguments, index, argument)?;
overrides.no_decay_embeddings = Some(
value
.split(',')
.map(str::trim)
.filter(|name| !name.is_empty())
.map(str::to_string)
.collect(),
);
index += 2;
}
"--decay-embeddings" => {
overrides.no_decay_embeddings = Some(Vec::new());
index += 1;
}
"--rms-norm-lr-scale" => {
overrides.rms_norm_lr_scale = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--embedding-lr-scale" => {
overrides.embedding_lr_scale = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--beta1" => {
overrides.beta1 = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--beta2" => {
overrides.beta2 = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--adam-epsilon" => {
overrides.epsilon = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--max-grad-norm" => {
overrides.max_gradient_norm = Some(parse_value(arguments, index, argument)?);
index += 2;
}
"--log-every" => {
overrides.log_every = Some(parse_value(arguments, index, argument)?);
index += 2;
}
other => return Err(format!("unknown training option: {other}").into()),
}
}
Ok((config_file, overrides))
}
#[cfg(feature = "training")]
fn emit_steps_clarification(
cli_steps: Option<usize>,
file_total_steps: Option<usize>,
resolved_steps: usize,
resume: bool,
eval_only: bool,
checkpoint_step: Option<usize>,
checkpoint_total_steps: Option<usize>,
) {
for message in format_steps_clarification(
cli_steps,
file_total_steps,
resolved_steps,
resume,
eval_only,
checkpoint_step,
checkpoint_total_steps,
) {
log::emit_message("steps", &message);
}
}
#[cfg(feature = "training")]
fn require_value<'a>(
arguments: &'a [String],
index: usize,
name: &str,
) -> Result<&'a str, Box<dyn Error>> {
arguments
.get(index + 1)
.map(String::as_str)
.ok_or_else(|| format!("missing value for {name}").into())
}
#[cfg(feature = "training")]
fn parse_value<T>(arguments: &[String], index: usize, name: &str) -> Result<T, Box<dyn Error>>
where
T: std::str::FromStr,
T::Err: Error + 'static,
{
Ok(require_value(arguments, index, name)?.parse()?)
}
#[cfg(feature = "training")]
fn parse_pre_tokenization(value: &str) -> Result<PreTokenization, Box<dyn Error>> {
match value {
"raw" => Ok(PreTokenization::Raw),
"documents" => Ok(PreTokenization::Documents {
separator: "\n\n".into(),
}),
"whitespace" => Ok(PreTokenization::Whitespace {
space: true,
tab: true,
newline: true,
}),
other => Err(format!(
"unsupported pre-tokenization '{other}'; expected raw, documents, or whitespace"
)
.into()),
}
}
fn generate(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let checkpoint_path = arguments.first().ok_or("missing checkpoint path")?;
let use_gpu = arguments.iter().any(|argument| argument == "--gpu");
if use_gpu {
require_gpu_feature()?;
}
let memory_budget_bytes = parse_memory_budget(arguments)?;
let prompt = positional_prompt(arguments);
if prompt.is_empty() {
return Err("missing generation prompt".into());
}
let checkpoint = load_checkpoint(checkpoint_path)?;
let max_new_tokens = parse_option(arguments, "--tokens")?.unwrap_or(128usize);
let prompt_tokens = checkpoint.tokenizer.encode(&prompt);
let generation_length = prompt_tokens
.len()
.saturating_add(max_new_tokens)
.min(checkpoint.model.config.max_sequence_length)
.max(1);
let memory_estimate = estimate_inference_memory(&checkpoint.model.config, 1, generation_length);
let execution = resolve_execution_device(use_gpu);
if use_gpu {
emit_backend_selection(&execution);
}
let memory_budget = match &execution {
ExecutionDevice::Gpu(gpu) => {
apply_gpu_cli_memory_budget(gpu, memory_budget_bytes)?;
let mut budget = gpu.memory_budget();
if let Some(total_bytes) = memory_budget_bytes {
budget.total_bytes = total_bytes;
}
budget
}
ExecutionDevice::Cpu { .. } => DeviceBudget::cpu(memory_budget_bytes),
};
log::emit_message(
"memory",
&format_memory_report(&memory_estimate, Some(&memory_budget)),
);
check_memory_budget(&memory_estimate, &memory_budget).map_err(cli_error)?;
let generator = match execution.gpu() {
Some(gpu) => Generator::with_gpu(&checkpoint.model, &checkpoint.tokenizer, gpu),
None => Generator::new(&checkpoint.model, &checkpoint.tokenizer),
};
let sampling = SamplingConfig {
max_new_tokens,
temperature: parse_option(arguments, "--temperature")?.unwrap_or(0.8),
top_k: parse_option(arguments, "--top-k")?.or(Some(40)),
top_p: parse_option(arguments, "--top-p")?.unwrap_or(0.95),
repetition_penalty: parse_option(arguments, "--repetition-penalty")?.unwrap_or(1.1),
repetition_penalty_window: parse_option(arguments, "--repetition-penalty-window")?,
frequency_penalty: parse_option(arguments, "--frequency-penalty")?.unwrap_or(0.0),
presence_penalty: parse_option(arguments, "--presence-penalty")?.unwrap_or(0.0),
seed: parse_option(arguments, "--seed")?.unwrap_or(42),
return_logprobs: false,
top_logprobs: 0,
};
let mut stdout = io::stdout().lock();
write!(stdout, "{prompt}")?;
stdout.flush()?;
let mut stream_error = None;
generator.generate_stream(&prompt, &sampling, |_token_id, bytes| {
if stream_error.is_none() {
if let Err(error) = stdout.write_all(bytes).and_then(|_| stdout.flush()) {
stream_error = Some(error);
}
}
})?;
if let Some(error) = stream_error {
return Err(error.into());
}
writeln!(stdout)?;
Ok(())
}
fn inspect(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let checkpoint_path = arguments.first().ok_or("missing checkpoint path")?;
if arguments.iter().any(|argument| argument.starts_with("--")) {
return Err("inspect does not take options; usage: oxidelm --inspect model.bin".into());
}
let report = inspect_checkpoint(checkpoint_path)?;
print!("{}", format_report(&report));
Ok(())
}
fn analyze(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let mut positionals = Vec::new();
let mut format = AnalyzeFormat::Auto;
let mut json_out: Option<String> = None;
let mut fail_on = Vec::new();
let mut corpus: Option<String> = None;
let mut eval_corpus: Option<String> = None;
let mut reference: Option<String> = None;
let mut eval_sequence_length: Option<usize> = None;
let mut eval_batch_size: usize = 1;
let mut max_perplexity: Option<f64> = None;
let mut max_kl: Option<f64> = None;
let mut probe_device = false;
let mut index = 0;
while index < arguments.len() {
match arguments[index].as_str() {
"--format" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --format")?;
format = AnalyzeFormat::parse(value)?;
index += 2;
}
"--json" => {
let value = arguments.get(index + 1).ok_or("missing value for --json")?;
json_out = Some(value.clone());
index += 2;
}
"--fail-on" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --fail-on")?;
for part in value.split(|ch| ch == ',' || ch == '|') {
let part = part.trim();
if part.is_empty() {
continue;
}
fail_on.push(FailOn::parse(part)?);
}
index += 2;
}
"--corpus" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --corpus")?;
corpus = Some(value.clone());
index += 2;
}
"--eval-corpus" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --eval-corpus")?;
eval_corpus = Some(value.clone());
index += 2;
}
"--reference" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --reference")?;
reference = Some(value.clone());
index += 2;
}
"--eval-sequence-length" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --eval-sequence-length")?;
eval_sequence_length = Some(value.parse()?);
index += 2;
}
"--eval-batch-size" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --eval-batch-size")?;
eval_batch_size = value.parse()?;
index += 2;
}
"--max-perplexity" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --max-perplexity")?;
max_perplexity = Some(value.parse()?);
index += 2;
}
"--max-kl" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --max-kl")?;
max_kl = Some(value.parse()?);
index += 2;
}
"--probe-device" => {
probe_device = true;
index += 1;
}
flag if flag.starts_with("--") => {
return Err(format!(
"unknown analyze option '{flag}'; usage: oxidelm --analyze <path> [--format oxide|safetensors|hf|auto] [--json out.json] [--fail-on nan|inf|schema|forward|quality] [--corpus text.txt] [--eval-corpus text.txt] [--reference model.bin] [--eval-sequence-length N] [--eval-batch-size N] [--max-perplexity X] [--max-kl X] [--probe-device]"
)
.into());
}
value => {
positionals.push(value);
index += 1;
}
}
}
let path = positionals
.first()
.copied()
.ok_or("missing path for --analyze")?;
if positionals.len() > 1 {
return Err("too many positional arguments for --analyze".into());
}
if reference.is_some() && eval_corpus.is_none() {
return Err("--reference requires --eval-corpus".into());
}
if eval_batch_size == 0 {
return Err("--eval-batch-size must be positive".into());
}
let options = AnalyzeOptions {
format,
fail_on,
corpus: corpus.map(PathBuf::from),
eval_corpus: eval_corpus.map(PathBuf::from),
reference: reference.map(PathBuf::from),
eval_sequence_length,
eval_batch_size,
max_perplexity,
max_kl,
probe_device,
..AnalyzeOptions::default()
};
let report = analyze_path(path, &options)?;
if let Some(json_path) = json_out {
write_analyze_json(&report, json_path)?;
}
print!("{}", format_analyze_report(&report));
Ok(())
}
fn analyze_diff_cmd(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let mut positionals = Vec::new();
let mut format = AnalyzeFormat::Auto;
let mut json_out: Option<String> = None;
let mut index = 0;
while index < arguments.len() {
match arguments[index].as_str() {
"--format" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --format")?;
format = AnalyzeFormat::parse(value)?;
index += 2;
}
"--json" => {
let value = arguments.get(index + 1).ok_or("missing value for --json")?;
json_out = Some(value.clone());
index += 2;
}
flag if flag.starts_with("--") => {
return Err(format!(
"unknown analyze-diff option '{flag}'; usage: oxidelm --analyze-diff <a> <b> [--format oxide|safetensors|hf|auto] [--json out.json]"
)
.into());
}
value => {
positionals.push(value);
index += 1;
}
}
}
if positionals.len() != 2 {
return Err(
"usage: oxidelm --analyze-diff <a> <b> [--format oxide|safetensors|hf|auto] [--json out.json]"
.into(),
);
}
let options = AnalyzeOptions {
format,
..AnalyzeOptions::default()
};
let report = analyze_diff(positionals[0], positionals[1], &options)?;
if let Some(json_path) = json_out {
write_analyze_diff_json(&report, json_path)?;
}
print!("{}", format_analyze_diff_report(&report));
Ok(())
}
fn quantize_cmd(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let mut positionals = Vec::new();
let mut options = QuantizeOptions::default();
let mut method_set = false;
let mut index = 0;
while index < arguments.len() {
match arguments[index].as_str() {
"--method" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --method")?;
options.method = QuantizeMethod::parse(value)?;
method_set = true;
index += 2;
}
"--bits" => {
let value = arguments.get(index + 1).ok_or("missing value for --bits")?;
options.bits = value
.parse::<u32>()
.map_err(|_| format!("invalid --bits '{value}'"))?;
index += 2;
}
"--format" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --format")?;
options.format = Some(WeightQuantization::parse(value)?);
index += 2;
}
"--group-size" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --group-size")?;
let group = value
.parse::<usize>()
.map_err(|_| format!("invalid --group-size '{value}'"))?;
options.group_size = Some(group);
index += 2;
}
"--granularity" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --granularity")?;
let granularity = oxidelm::layers::QuantGranularity::parse(value)?;
options.group_size = match granularity {
oxidelm::layers::QuantGranularity::PerChannel => Some(0),
oxidelm::layers::QuantGranularity::PerTensor => Some(1),
oxidelm::layers::QuantGranularity::PerGroup { size } => Some(size),
};
index += 2;
}
"--symmetric" => {
options.symmetric = true;
index += 1;
}
"--asymmetric" => {
options.symmetric = false;
index += 1;
}
"--scale-method" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --scale-method")?;
options.scale_method = oxidelm::layers::ScaleMethod::parse(value)?;
index += 2;
}
"--percentile" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --percentile")?;
options.percentile = value
.parse::<f32>()
.map_err(|_| format!("invalid --percentile '{value}'"))?;
index += 2;
}
"--calibration" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --calibration")?;
options.calibration = Some(PathBuf::from(value));
index += 2;
}
"--calibration-sequences" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --calibration-sequences")?;
options.calibration_sequences = value
.parse::<usize>()
.map_err(|_| format!("invalid --calibration-sequences '{value}'"))?;
index += 2;
}
"--calibration-batch-size" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --calibration-batch-size")?;
options.calibration_batch_size = value
.parse::<usize>()
.map_err(|_| format!("invalid --calibration-batch-size '{value}'"))?;
index += 2;
}
"--calibration-seed" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --calibration-seed")?;
options.calibration_seed = value
.parse::<u64>()
.map_err(|_| format!("invalid --calibration-seed '{value}'"))?;
index += 2;
}
"--smoothquant-alpha" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --smoothquant-alpha")?;
options.smoothquant_alpha = Some(
value
.parse::<f32>()
.map_err(|_| format!("invalid --smoothquant-alpha '{value}'"))?,
);
index += 2;
}
"--exclude" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --exclude")?;
options.exclude = parse_pattern_list(value);
index += 2;
}
"--include" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --include")?;
options.include = Some(parse_pattern_list(value));
index += 2;
}
"--eval-corpus" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --eval-corpus")?;
options.eval_corpus = Some(PathBuf::from(value));
index += 2;
}
"--eval-sequence-length" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --eval-sequence-length")?;
options.eval_sequence_length = Some(
value
.parse::<usize>()
.map_err(|_| format!("invalid --eval-sequence-length '{value}'"))?,
);
index += 2;
}
"--eval-batch-size" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --eval-batch-size")?;
options.eval_batch_size = value
.parse::<usize>()
.map_err(|_| format!("invalid --eval-batch-size '{value}'"))?;
index += 2;
}
"--max-ppl-delta" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --max-ppl-delta")?;
options.max_ppl_delta = Some(
value
.parse::<f32>()
.map_err(|_| format!("invalid --max-ppl-delta '{value}'"))?,
);
index += 2;
}
"--max-kl" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --max-kl")?;
options.max_kl = Some(
value
.parse::<f32>()
.map_err(|_| format!("invalid --max-kl '{value}'"))?,
);
index += 2;
}
"--json" => {
let value = arguments.get(index + 1).ok_or("missing value for --json")?;
options.json_out = Some(PathBuf::from(value));
index += 2;
}
"--inference-mode" | "--mode" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --inference-mode")?;
options.inference_mode =
Some(oxidelm::act_quant::InferenceQuantMode::parse(value)?);
index += 2;
}
"--activation-format" | "--act-format" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --activation-format")?;
options.activation_format =
Some(oxidelm::act_quant::ActivationQuantization::parse(value)?);
index += 2;
}
"--activation-granularity" | "--act-granularity" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --activation-granularity")?;
options.activation_granularity =
oxidelm::act_quant::ActGranularity::parse(value)?;
index += 2;
}
"--activation-mode" | "--act-mode" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --activation-mode")?;
options.activation_mode = oxidelm::act_quant::ActQuantMode::parse(value)?;
index += 2;
}
"--kv-format" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --kv-format")?;
options.kv_format = Some(oxidelm::kv_quant::KvQuantization::parse(value)?);
index += 2;
}
"--kv-granularity" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --kv-granularity")?;
options.kv_granularity = oxidelm::kv_quant::KvGranularity::parse(value)?;
index += 2;
}
flag if flag.starts_with("--") => {
return Err(format!(
"unknown quantize option '{flag}'; usage: oxidelm --quantize <in> <out> --method rtn|gptq|awq|smoothquant|hqq|spqr|quip|aqlm [--bits 8|4|3|2 | --format nf4|fp8_e4m3|fp8_e5m2] [--inference-mode W8A8|W4A8|W4A16] [--activation-format int8] [--activation-mode static|dynamic] [--kv-format int8|nf4|fp8_e4m3|fp8_e5m2] [--calibration path] [--exclude embed,lm_head] [--json out.json]"
)
.into());
}
value => {
positionals.push(value);
index += 1;
}
}
}
if positionals.len() != 2 {
return Err(
"usage: oxidelm --quantize <in> <out> --method rtn|gptq|awq|smoothquant|hqq|spqr|quip|aqlm [--bits 8|4|3|2 | --format nf4|fp8_e4m3|fp8_e5m2] [--group-size 0|1|32|64|128] [--asymmetric] [--scale-method absmax|minmax|percentile|mse] [--calibration path] [--exclude embed,lm_head] [--include <glob>] [--eval-corpus text.txt] [--max-ppl-delta X] [--max-kl X] [--json out.json]"
.into(),
);
}
if !method_set {
return Err("missing required --method (rtn|gptq|awq|smoothquant|hqq|spqr|quip|aqlm)".into());
}
let report = quantize_checkpoint(&positionals[0], &positionals[1], &options)?;
print!("{}", format_quantize_report(&report));
Ok(())
}
fn eval_quality(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let mut positionals = Vec::new();
let mut tasks_arg: Option<String> = None;
let mut index = 0;
while index < arguments.len() {
match arguments[index].as_str() {
"--tasks" => {
let value = arguments
.get(index + 1)
.ok_or("missing value for --tasks")?;
tasks_arg = Some(value.clone());
index += 2;
}
"--sequence-length" | "--batch-size" | "--memory-budget" => {
if index + 1 >= arguments.len() {
return Err(format!("missing value for {}", arguments[index]).into());
}
index += 2;
}
"--gpu" => index += 1,
flag if flag.starts_with("--") => {
return Err(format!(
"unknown eval option '{flag}'; usage: oxidelm --eval model.bin [heldout.txt] [--tasks builtin|none|path.json] [--sequence-length N] [--batch-size N] [--gpu]"
)
.into());
}
value => {
positionals.push(value);
index += 1;
}
}
}
let checkpoint_path = positionals
.first()
.copied()
.ok_or("missing checkpoint path")?;
let held_out_path = positionals.get(1).copied();
if positionals.len() > 2 {
return Err("too many positional arguments for --eval".into());
}
let tasks: Option<Vec<EvalTask>> = match tasks_arg.as_deref().unwrap_or("builtin") {
"none" => None,
"builtin" => Some(builtin_tiny_tasks()),
path => Some(load_tasks(path)?),
};
if held_out_path.is_none() && tasks.is_none() {
return Err("provide held-out text and/or --tasks builtin|path.json (not none)".into());
}
let checkpoint = load_checkpoint(checkpoint_path)?;
let sequence_length = parse_option(arguments, "--sequence-length")?.unwrap_or(
checkpoint
.trainer_config
.sequence_length
.min(checkpoint.model.config.max_sequence_length),
);
let batch_size = parse_option(arguments, "--batch-size")?.unwrap_or(
checkpoint.trainer_config.batch_size.max(1),
);
if sequence_length == 0 {
return Err("--sequence-length must be positive".into());
}
if batch_size == 0 {
return Err("--batch-size must be positive".into());
}
let held_out_text = if let Some(path) = held_out_path {
let text_file = File::open(path)?;
// SAFETY: The mapping is read-only and the source file remains open for the
// lifetime of the mapping.
let mapped = unsafe { MmapOptions::new().map(&text_file)? };
Some(String::from_utf8_lossy(&mapped).into_owned())
} else {
None
};
let use_gpu = arguments.iter().any(|argument| argument == "--gpu");
if use_gpu {
require_gpu_feature()?;
}
let memory_budget_bytes = parse_memory_budget(arguments)?;
let memory_estimate = estimate_inference_memory(
&checkpoint.model.config,
batch_size,
sequence_length,
);
let execution = resolve_execution_device(use_gpu);
if use_gpu {
emit_backend_selection(&execution);
}
let memory_budget = match &execution {
ExecutionDevice::Gpu(gpu) => {
apply_gpu_cli_memory_budget(gpu, memory_budget_bytes)?;
let mut budget = gpu.memory_budget();
if let Some(total_bytes) = memory_budget_bytes {
budget.total_bytes = total_bytes;
}
budget
}
ExecutionDevice::Cpu { .. } => DeviceBudget::cpu(memory_budget_bytes),
};
log::emit_message(
"memory",
&format_memory_report(&memory_estimate, Some(&memory_budget)),
);
check_memory_budget(&memory_estimate, &memory_budget).map_err(cli_error)?;
if let Some(gpu) = execution.gpu() {
checkpoint.model.prepare_gpu(gpu)?;
}
let report = evaluate_quality(
&checkpoint.model,
&checkpoint.tokenizer,
held_out_text.as_deref(),
sequence_length,
batch_size,
tasks.as_deref(),
execution.gpu(),
)?;
match log::log_mode() {
LogMode::Quiet => {}
LogMode::Json => emit_quality_report(&report),
LogMode::Human => print!("{}", format_quality_report(&report)),
}
Ok(())
}
fn emit_quality_report(report: &QualityReport) {
if let Some(ppl) = &report.perplexity {
log::emit(
"quality_perplexity",
&[
("loss", LogValue::f32(ppl.loss, 5)),
("perplexity", LogValue::f32(ppl.perplexity, 3)),
("tokens", LogValue::Usize(ppl.tokens_processed)),
("sequences", LogValue::Usize(ppl.sequences)),
("sequence_length", LogValue::Usize(ppl.sequence_length)),
("wall_s", LogValue::f64(ppl.wall_time_seconds, 3)),
],
);
}
if let Some(next) = &report.next_token {
log::emit(
"quality_next_token",
&[
("correct", LogValue::Usize(next.correct)),
("total", LogValue::Usize(next.total)),
("accuracy", LogValue::f64(next.accuracy, 4)),
],
);
}
if let Some(tasks) = &report.tasks {
log::emit(
"quality_tasks",
&[
("count", LogValue::Usize(tasks.tasks.len())),
("exact_match_rate", LogValue::f64(tasks.exact_match_rate, 4)),
(
"prefix_match_rate",
LogValue::f64(tasks.prefix_match_rate, 4),
),
("byte_accuracy", LogValue::f64(tasks.byte_accuracy, 4)),
],
);
for task in &tasks.tasks {
let generated = task.generated.as_str();
log::emit(
"quality_task",
&[
("id", LogValue::Str(&task.id)),
("exact", LogValue::Bool(task.exact_match)),
("prefix", LogValue::Bool(task.prefix_match)),
(
"byte_accuracy",
LogValue::f64(
if task.bytes_total == 0 {
0.0
} else {
task.bytes_correct as f64 / task.bytes_total as f64
},
4,
),
),
("generated", LogValue::Str(generated)),
],
);
}
}
}
fn tokenizer_report(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let mut positionals = Vec::new();
let mut index = 0;
while index < arguments.len() {
match arguments[index].as_str() {
"--repeats" => {
if index + 1 >= arguments.len() {
return Err("missing value for --repeats".into());
}
index += 2;
}
flag if flag.starts_with("--") => {
return Err(format!(
"unknown tokenizer-report option '{flag}'; usage: oxidelm --tokenizer-report model.bin|tokenizer.json heldout.txt [--repeats 3]"
)
.into());
}
value => {
positionals.push(value);
index += 1;
}
}
}
let model_path = positionals
.first()
.copied()
.ok_or("missing checkpoint or tokenizer path")?;
let text_path = positionals
.get(1)
.copied()
.ok_or("missing held-out text path")?;
if positionals.len() > 2 {
return Err("too many positional arguments for --tokenizer-report".into());
}
let repeats = parse_option::<usize>(arguments, "--repeats")?.unwrap_or(3);
if repeats == 0 {
return Err("--repeats must be positive".into());
}
let tokenizer = load_tokenizer_for_report(model_path)?;
let text_file = File::open(text_path)?;
// SAFETY: The mapping is read-only and the source file remains open for the
// lifetime of the mapping.
let text = unsafe { MmapOptions::new().map(&text_file)? };
let report = evaluate_bytes(&tokenizer, &text, repeats);
print!("{}", format_tokenizer_quality_report(&report));
Ok(())
}
fn load_tokenizer_for_report(path: &str) -> Result<BpeTokenizer, Box<dyn Error>> {
match BpeTokenizer::load(path) {
Ok(tokenizer) => Ok(tokenizer),
Err(tokenizer_error) => match load_checkpoint(path) {
Ok(checkpoint) => Ok(checkpoint.tokenizer),
Err(checkpoint_error) => Err(format!(
"failed to load tokenizer from {path}: tokenizer error: {tokenizer_error}; checkpoint error: {checkpoint_error}"
)
.into()),
},
}
}
#[cfg(feature = "training")]
fn emit_tokenizer_quality(tokenizer: &BpeTokenizer, held_out: &[u8], repeats: usize) {
let report = evaluate_bytes(tokenizer, held_out, repeats);
log::emit(
"tokenizer_quality",
&[
("bytes", LogValue::Usize(report.bytes)),
("characters", LogValue::Usize(report.characters)),
("tokens", LogValue::Usize(report.tokens)),
(
"compression_ratio",
LogValue::f64(report.compression_ratio, 4),
),
(
"tokens_per_character",
LogValue::f64(report.tokens_per_character, 4),
),
(
"unknown_token_rate",
LogValue::f64(report.unknown_token_rate, 6),
),
(
"vocabulary_utilization",
LogValue::f64(report.vocabulary_utilization, 4),
),
(
"encode_bytes_per_second",
LogValue::f64(report.encode_bytes_per_second, 1),
),
(
"encode_tokens_per_second",
LogValue::f64(report.encode_tokens_per_second, 1),
),
(
"decode_bytes_per_second",
LogValue::f64(report.decode_bytes_per_second, 1),
),
(
"decode_tokens_per_second",
LogValue::f64(report.decode_tokens_per_second, 1),
),
],
);
}
fn export_weights(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let checkpoint_path = arguments
.first()
.filter(|value| !value.starts_with("--"))
.ok_or("missing checkpoint path")?;
let weights_path = arguments
.get(1)
.filter(|value| !value.starts_with("--"))
.ok_or("missing output safetensors path")?;
let format = weight_format(arguments);
let config_out = option_value(arguments, "--config-out").map(Path::new);
export_checkpoint_weights(checkpoint_path, weights_path, format, config_out)?;
log::emit(
"export",
&[
("format", LogValue::Str(weight_format_name(format))),
("checkpoint", LogValue::Str(checkpoint_path)),
("weights", LogValue::Str(weights_path)),
],
);
if let Some(path) = config_out {
let config_display = path.display().to_string();
log::emit("export", &[("config", LogValue::Str(&config_display))]);
}
Ok(())
}
fn export_quantized(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let checkpoint_path = arguments
.first()
.filter(|value| !value.starts_with("--"))
.ok_or("missing checkpoint path")?;
let weights_path = arguments
.get(1)
.filter(|value| !value.starts_with("--"))
.ok_or("missing output safetensors path")?;
let format = weight_format(arguments);
export_quantized_safetensors(checkpoint_path, weights_path, format)?;
log::emit(
"export_quantized",
&[
("format", LogValue::Str(weight_format_name(format))),
("checkpoint", LogValue::Str(checkpoint_path)),
("weights", LogValue::Str(weights_path)),
],
);
Ok(())
}
fn import_quantized(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let weights_path = arguments
.first()
.filter(|value| !value.starts_with("--"))
.ok_or("missing safetensors path")?;
let checkpoint_path = arguments
.get(1)
.filter(|value| !value.starts_with("--"))
.ok_or("missing checkpoint path")?;
let format = weight_format(arguments);
import_quantized_safetensors(weights_path, checkpoint_path, format)?;
log::emit(
"import_quantized",
&[
("format", LogValue::Str(weight_format_name(format))),
("weights", LogValue::Str(weights_path)),
("checkpoint", LogValue::Str(checkpoint_path)),
],
);
Ok(())
}
fn export_gguf(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let checkpoint_path = arguments
.first()
.filter(|value| !value.starts_with("--"))
.ok_or("missing checkpoint path")?;
let gguf_path = arguments
.get(1)
.filter(|value| !value.starts_with("--"))
.ok_or("missing output gguf path")?;
let mut options = GgufExportOptions::default();
let mut index = 2usize;
while index < arguments.len() {
match arguments[index].as_str() {
"--type" => {
index += 1;
let value = arguments
.get(index)
.ok_or("missing value for --type")?;
options.quant_type = GgmlType::parse(value)?;
}
"--quantize-embed" => options.keep_embed_f32 = false,
other if other.starts_with("--") => {
return Err(format!(
"unknown --export-gguf option '{other}'; usage: oxidelm --export-gguf model.bin model.gguf [--type Q4_0|Q4_K|Q5_K|Q6_K|Q8_0] [--quantize-embed]"
)
.into());
}
_ => {}
}
index += 1;
}
export_checkpoint_gguf(checkpoint_path, gguf_path, &options)?;
log::emit(
"export_gguf",
&[
("type", LogValue::Str(options.quant_type.as_str())),
("checkpoint", LogValue::Str(checkpoint_path)),
("gguf", LogValue::Str(gguf_path)),
],
);
Ok(())
}
fn import_gguf(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let gguf_path = arguments
.first()
.filter(|value| !value.starts_with("--"))
.ok_or("missing gguf path")?;
let checkpoint_path = arguments
.get(1)
.filter(|value| !value.starts_with("--"))
.ok_or("missing output checkpoint path")?;
let mut options = GgufImportOptions::default();
let mut index = 2usize;
while index < arguments.len() {
match arguments[index].as_str() {
"--tokenizer" => {
index += 1;
let value = arguments
.get(index)
.ok_or("missing value for --tokenizer")?;
options.tokenizer_path = Some(PathBuf::from(value));
}
"--pack-native" => options.pack_native = true,
other if other.starts_with("--") => {
return Err(format!(
"unknown --import-gguf option '{other}'; usage: oxidelm --import-gguf model.gguf model.bin [--tokenizer tokenizer.json] [--pack-native]"
)
.into());
}
_ => {}
}
index += 1;
}
import_gguf_checkpoint(gguf_path, checkpoint_path, &options)?;
log::emit(
"import_gguf",
&[
("gguf", LogValue::Str(gguf_path)),
("checkpoint", LogValue::Str(checkpoint_path)),
],
);
Ok(())
}
fn export_adapters(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let checkpoint_path = arguments
.first()
.filter(|value| !value.starts_with("--"))
.ok_or("missing checkpoint path")?;
let adapters_path = arguments
.get(1)
.filter(|value| !value.starts_with("--"))
.ok_or("missing output adapters path")?;
let checkpoint = oxidelm::trainer::load_checkpoint(checkpoint_path)?;
let config = checkpoint
.trainer_config
.low_memory
.lora
.as_ref()
.ok_or("checkpoint has no LoRA config to export")?;
oxidelm::peft::save_adapters(adapters_path, &checkpoint.model, config)?;
log::emit(
"export_adapters",
&[
("checkpoint", LogValue::Str(checkpoint_path)),
("adapters", LogValue::Str(adapters_path)),
],
);
Ok(())
}
fn import_adapters(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let checkpoint_path = arguments
.first()
.filter(|value| !value.starts_with("--"))
.ok_or("missing checkpoint path")?;
let adapters_path = arguments
.get(1)
.filter(|value| !value.starts_with("--"))
.ok_or("missing adapters path")?;
let output_path = arguments
.get(2)
.filter(|value| !value.starts_with("--"))
.ok_or("missing output checkpoint path")?;
let mut checkpoint = oxidelm::trainer::load_checkpoint(checkpoint_path)?;
if checkpoint
.model
.named_parameters()
.iter()
.all(|(name, _)| !name.contains(".lora."))
{
let config = oxidelm::peft::load_adapter_config(adapters_path)?
.or_else(|| checkpoint.trainer_config.low_memory.lora.clone())
.ok_or("no LoRA config in adapters metadata or checkpoint")?;
checkpoint.model.install_lora(&config)?;
checkpoint.trainer_config.low_memory.lora = Some(config);
}
oxidelm::peft::load_adapters(adapters_path, &checkpoint.model)?;
let micro_batches_processed = checkpoint
.training_step
.saturating_mul(checkpoint.trainer_config.gradient_accumulation_steps);
oxidelm::trainer::save_checkpoint_with_best(
output_path,
&checkpoint.model,
&checkpoint.optimizer,
&checkpoint.tokenizer,
&checkpoint.schedule,
&checkpoint.trainer_config,
checkpoint.training_step,
checkpoint.cursor,
checkpoint.tokens_processed,
micro_batches_processed,
checkpoint.best_validation_loss,
checkpoint.evaluations_without_improvement,
checkpoint.loss_scaler.clone(),
)?;
log::emit(
"import_adapters",
&[
("checkpoint", LogValue::Str(checkpoint_path)),
("adapters", LogValue::Str(adapters_path)),
("output", LogValue::Str(output_path)),
],
);
Ok(())
}
/// Packs QAT masters into inference weights and writes an inference-only checkpoint.
fn export_qat_cmd(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let checkpoint_path = arguments
.first()
.filter(|value| !value.starts_with("--"))
.ok_or("missing checkpoint path")?;
let output_path = arguments
.get(1)
.filter(|value| !value.starts_with("--"))
.ok_or("missing output checkpoint path")?;
let mut checkpoint = oxidelm::trainer::load_checkpoint(checkpoint_path)?;
if checkpoint.trainer_config.low_memory.qat.is_none()
&& checkpoint
.model
.named_linears()
.iter()
.all(|(_, linear)| linear.qat_format().is_none())
{
return Err("checkpoint has no QAT state to export".into());
}
// Re-enable QAT from trainer config if the packed flag was cleared on load edge cases.
if let Some(qat) = checkpoint.trainer_config.low_memory.qat.clone() {
if checkpoint
.model
.named_linears()
.iter()
.all(|(_, linear)| linear.qat_format().is_none())
{
checkpoint.model.enable_qat_with(qat)?;
}
}
checkpoint.model.export_qat_packed()?;
checkpoint.trainer_config.low_memory.qat = None;
let quantization = qat_export_quantization_metadata(&checkpoint.model)?;
oxidelm::trainer::save_inference_checkpoint(
output_path,
&checkpoint.model,
&checkpoint.tokenizer,
&quantization,
)?;
log::emit(
"export_qat",
&[
("checkpoint", LogValue::Str(checkpoint_path)),
("output", LogValue::Str(output_path)),
],
);
Ok(())
}
fn qat_export_quantization_metadata(
model: &oxidelm::model::CausalLM,
) -> Result<oxidelm::trainer::CheckpointQuantization, Box<dyn Error>> {
let mut layers = Vec::new();
let mut bits = 8u32;
let mut group_size = None;
for (name, linear) in model.named_linears() {
let Some(quant) = linear.quantized_weight() else {
continue;
};
let scheme = quant.scheme();
bits = scheme.format.bits();
group_size = scheme.granularity.group_size();
layers.push(oxidelm::trainer::QuantizedLayerEntry {
name: name.clone(),
format: quant.format(),
in_features: quant.in_features(),
out_features: quant.out_features(),
packed_name: format!("{name}.weight.qpacked"),
scales_name: format!("{name}.weight.qscales"),
zero_points_name: if scheme.symmetric {
None
} else {
Some(format!("{name}.weight.qzeros"))
},
group_size: scheme.granularity.group_size(),
symmetric: scheme.symmetric,
scale_method: Some(scheme.scale_method.as_str().into()),
granularity: Some(scheme.granularity.as_str()),
});
}
if layers.is_empty() {
return Err("no packed Linear weights after QAT export".into());
}
Ok(oxidelm::trainer::CheckpointQuantization {
schema_version: oxidelm::trainer::QUANTIZATION_SCHEMA_VERSION,
method: "qat".into(),
bits,
group_size,
symmetric: true,
zero_points: false,
scale_method: Some("absmax".into()),
granularity: Some(
group_size
.map(|size| format!("group:{size}"))
.unwrap_or_else(|| "channel".into()),
),
format: layers.first().map(|layer| layer.format.as_str().into()),
inference_only: true,
excluded: Vec::new(),
included: None,
layers,
activation: None,
kv_cache: None,
})
}
fn import_weights(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let weights_path = arguments
.first()
.filter(|value| !value.starts_with("--"))
.ok_or("missing safetensors path")?;
let checkpoint_path = arguments
.get(1)
.filter(|value| !value.starts_with("--"))
.ok_or("missing checkpoint path")?;
let format = weight_format(arguments);
import_checkpoint_weights(weights_path, checkpoint_path, format)?;
log::emit(
"import",
&[
("format", LogValue::Str(weight_format_name(format))),
("weights", LogValue::Str(weights_path)),
("checkpoint", LogValue::Str(checkpoint_path)),
],
);
Ok(())
}
fn import_tokenizer(arguments: &[String]) -> Result<(), Box<dyn Error>> {
let positional: Vec<&str> = arguments
.iter()
.map(String::as_str)
.filter(|value| !value.starts_with("--"))
.collect();
let (tokenizer, format_name, output_path) = match positional.as_slice() {
[hf_tokenizer, output] => {
let tokenizer = BpeTokenizer::import_huggingface(hf_tokenizer)?;
(tokenizer, "huggingface-bpe", *output)
}
[vocab, merges, output] => {
let tokenizer = BpeTokenizer::import_gpt2(vocab, merges)?;
(tokenizer, "gpt2-bpe", *output)
}
_ => {
return Err(
"usage: oxidelm --import-tokenizer tokenizer.json out.json | --import-tokenizer vocab.json merges.txt out.json"
.into(),
);
}
};
tokenizer.save(output_path)?;
log::emit(
"import_tokenizer",
&[
("format", LogValue::Str(format_name)),
("output", LogValue::Str(output_path)),
("vocab_size", LogValue::Usize(tokenizer.vocab_size())),
("merges", LogValue::Usize(tokenizer.merges.len())),
],
);
Ok(())
}
fn weight_format(arguments: &[String]) -> WeightFormat {
if arguments.iter().any(|argument| argument == "--hf") {
WeightFormat::HuggingFace
} else {
WeightFormat::OxideLm
}
}
fn weight_format_name(format: WeightFormat) -> &'static str {
match format {
WeightFormat::OxideLm => "oxidelm",
WeightFormat::HuggingFace => "huggingface-llama",
}
}
fn option_value<'a>(arguments: &'a [String], name: &str) -> Option<&'a str> {
arguments
.windows(2)
.find(|window| window[0] == name)
.map(|window| window[1].as_str())
}
fn parse_option<T>(arguments: &[String], name: &str) -> Result<Option<T>, Box<dyn Error>>
where
T: std::str::FromStr,
T::Err: Error + 'static,
{
option_value(arguments, name)
.map(str::parse)
.transpose()
.map_err(Into::into)
}
fn positional_prompt(arguments: &[String]) -> String {
let value_flags = [
"--tokens",
"--temperature",
"--top-k",
"--top-p",
"--repetition-penalty",
"--repetition-penalty-window",
"--frequency-penalty",
"--presence-penalty",
"--seed",
"--memory-budget",
];
let mut prompt = Vec::new();
let mut index = 1;
while index < arguments.len() {
if arguments[index] == "--gpu" {
index += 1;
} else if value_flags.contains(&arguments[index].as_str()) {
index += 2;
} else {
prompt.push(arguments[index].clone());
index += 1;
}
}
prompt.join(" ")
}
fn parse_memory_budget(arguments: &[String]) -> Result<Option<usize>, Box<dyn Error>> {
let Some(value) = option_value(arguments, "--memory-budget") else {
return Ok(None);
};
Ok(Some(parse_byte_size(value)?))
}
#[cfg(feature = "training")]
fn parse_fit_memory(arguments: &[String]) -> Result<(bool, Option<usize>), Box<dyn Error>> {
let mut index = 0;
while index < arguments.len() {
if arguments[index] == "--fit-memory" {
let next = arguments.get(index + 1).map(String::as_str);
if let Some(value) = next.filter(|value| !value.starts_with("--")) {
if parse_byte_size(value).is_ok() {
return Ok((true, Some(parse_byte_size(value)?)));
}
}
return Ok((true, None));
}
index += 1;
}
Ok((false, None))
}
fn parse_byte_size(value: &str) -> Result<usize, Box<dyn Error>> {
let trimmed = value.trim();
if trimmed.is_empty() {
return Err("memory budget must not be empty".into());
}
let (number, multiplier) = if let Some(number) = trimmed.strip_suffix("GiB") {
(number.trim(), 1024usize * 1024 * 1024)
} else if let Some(number) = trimmed.strip_suffix("MiB") {
(number.trim(), 1024usize * 1024)
} else if let Some(number) = trimmed.strip_suffix("KiB") {
(number.trim(), 1024usize)
} else if let Some(number) = trimmed.strip_suffix('G') {
(number.trim(), 1024usize * 1024 * 1024)
} else if let Some(number) = trimmed.strip_suffix('M') {
(number.trim(), 1024usize * 1024)
} else if let Some(number) = trimmed.strip_suffix('K') {
(number.trim(), 1024usize)
} else if let Some(number) = trimmed.strip_suffix('B') {
(number.trim(), 1usize)
} else {
(trimmed, 1usize)
};
if number.is_empty() {
return Err(format!("invalid memory budget: {value}").into());
}
let amount: f64 = number
.parse()
.map_err(|_| format!("invalid memory budget: {value}"))?;
if !amount.is_finite() || amount <= 0.0 {
return Err("memory budget must be positive".into());
}
let bytes = amount * multiplier as f64;
if bytes > usize::MAX as f64 {
return Err("memory budget is too large".into());
}
Ok(bytes.round() as usize)
}
#[cfg(feature = "training")]
fn default_best_checkpoint_path(checkpoint_path: impl AsRef<Path>) -> PathBuf {
let checkpoint_path = checkpoint_path.as_ref();
let stem = checkpoint_path
.file_stem()
.and_then(|value| value.to_str())
.unwrap_or("model");
let filename = match checkpoint_path.extension().and_then(|value| value.to_str()) {
Some(extension) => format!("{stem}.best.{extension}"),
None => format!("{stem}.best"),
};
checkpoint_path.with_file_name(filename)
}
fn resolve_execution_device(use_gpu: bool) -> ExecutionDevice {
if use_gpu {
ExecutionDevice::from_gpu_result(GpuContext::global())
} else {
ExecutionDevice::cpu()
}
}
/// Soft budget + `GpuContext::set_cache_limits` for a low `--memory-budget`.
fn apply_gpu_cli_memory_budget(
gpu: &GpuContext,
budget_bytes: Option<usize>,
) -> Result<(), Box<dyn Error>> {
gpu.apply_memory_budget(budget_bytes)
.map_err(|error| -> Box<dyn Error> { error.into() })?;
emit_gpu_cache_limits(budget_bytes);
Ok(())
}
fn emit_gpu_cache_limits(budget_bytes: Option<usize>) {
let limits = GpuCacheLimits::for_memory_budget(budget_bytes);
if !limits.is_low_memory() {
return;
}
log::emit_message(
"gpu_cache",
&format!(
"low --memory-budget: output_pool={} bind_group_entries={}",
oxidelm::memory::format_bytes(limits.output_buffer_bytes),
limits.bind_group_entries
),
);
}
fn emit_backend_selection(execution: &ExecutionDevice) {
match execution {
ExecutionDevice::Gpu(gpu) => {
log::emit(
"backend",
&[
("device", LogValue::Str("gpu")),
("adapter", LogValue::Str(gpu.adapter_name())),
],
);
}
ExecutionDevice::Cpu {
fallback_reason: Some(reason),
} => {
log::emit(
"backend",
&[
("device", LogValue::Str("cpu")),
("reason", LogValue::Str(reason)),
],
);
}
ExecutionDevice::Cpu {
fallback_reason: None,
} => {
log::emit("backend", &[("device", LogValue::Str("cpu"))]);
}
}
}
#[cfg(feature = "gpu")]
fn require_gpu_feature() -> Result<(), Box<dyn Error>> {
Ok(())
}
#[cfg(not(feature = "gpu"))]
fn require_gpu_feature() -> Result<(), Box<dyn Error>> {
Err("the 'gpu' feature is disabled; rebuild with `--features gpu` to use --gpu".into())
}
fn cli_error(error: impl std::fmt::Display) -> Box<dyn Error> {
Box::new(CliError(error.to_string()))
}
struct CliError(String);
impl std::fmt::Debug for CliError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.0)
}
}
impl std::fmt::Display for CliError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.0)
}
}
impl Error for CliError {}
fn print_usage() {
eprintln!("usage:");
eprintln!(
" oxidelm --train init [train.json] [--architecture llama-tiny|llama-micro|llama-small|gpt-tiny|gpt-micro|gpt-small] [--memory-budget 6GiB] [--force] [--gpu] | oxidelm --train init --schema [training/schema.json] [--force]"
);
eprintln!(
" oxidelm --train [input.txt|data.jsonl|corpus_dir/ [model.bin]] [--config train.json|training/tiny.json|training/low-ram-8gb.json|training/cpu-laptop.json|training/gpu-16gb.json] [--architecture llama-tiny|llama-micro|llama-small|gpt-tiny|gpt-micro|gpt-small] [--resume] [--eval-only] [--dry-run|--plan-only] [--fit-memory [6GiB]] [--steps|--max-steps 100] [--hidden-size 64] [--num-layers 2] [--num-heads 4] [--num-key-value-heads 2] [--intermediate-size 128] [--use-bias|--no-bias] [--initializer-std 0.02] [--attention-dropout 0.0] [--residual-dropout 0.0] [--embedding-dropout 0.0] [--sequence-length 32] [--batch-size 1] [--gradient-accumulation-steps 1] [--data-parallel-workers 1] [--activation-checkpointing] [--mixed-precision] [--amp-format fp16|bf16] [--initial-loss-scale 65536] [--qat int8|int4] [--qat-group-size 32|64|128] [--qat-learnable-scales] [--qlora-base nf4|int4] [--qlora-group-size 32|64|128] [--vocab-size 516] [--pre-tokenization raw|documents|whitespace] [--lr 3e-4] [--weight-decay 0.01] [--no-decay-rms-norm|--decay-rms-norm] [--no-decay-embeddings name[,name...]|--decay-embeddings] [--embedding-lr-scale 1.0] [--rms-norm-lr-scale 1.0] [--validation-split 0.1] [--eval-every 10] [--early-stopping-patience 0] [--early-stopping-min-delta 0.0] [--token-cache model.bin.tokens] [--shard-tokens 1048576] [--tokenize-chunk-bytes 1048576] [--prefetch-batches 2] [--best-checkpoint model.best.bin] [--metrics-jsonl metrics.jsonl] [--memory-budget 2GiB] [--log human|json|quiet] [--profile [summary|trace]] [--detect-anomaly] [--gpu]"
);
eprintln!(
" oxidelm --generate model.bin \"prompt text\" [--tokens 128] [--temperature 0.8] [--top-k 40] [--top-p 0.95] [--repetition-penalty 1.1] [--repetition-penalty-window N] [--frequency-penalty 0.0] [--presence-penalty 0.0] [--memory-budget 2GiB] [--log human|json|quiet] [--profile [summary|trace]] [--detect-anomaly] [--gpu]"
);
eprintln!(" oxidelm --inspect model.bin");
eprintln!(
" oxidelm --analyze <path> [--format oxide|safetensors|hf|auto] [--json out.json] [--fail-on nan|inf|schema|forward|quality] [--corpus text.txt] [--eval-corpus text.txt] [--reference model.bin] [--eval-sequence-length N] [--eval-batch-size N] [--max-perplexity X] [--max-kl X] [--probe-device]"
);
eprintln!(
" oxidelm --analyze-diff <a> <b> [--format oxide|safetensors|hf|auto] [--json out.json]"
);
eprintln!(
" oxidelm --quantize <in> <out> --method rtn|gptq|awq|smoothquant|hqq|spqr|quip|aqlm [--bits 8|4|3|2 | --format nf4|fp8_e4m3|fp8_e5m2] [--group-size 0|1|32|64|128] [--asymmetric] [--scale-method absmax|minmax|percentile|mse] [--percentile P] [--inference-mode W8A8|W4A8|W4A16] [--activation-format int8] [--activation-granularity tensor|token] [--activation-mode static|dynamic] [--kv-format int8|nf4|fp8_e4m3|fp8_e5m2] [--kv-granularity layer|head] [--calibration path] [--calibration-sequences N] [--calibration-seed N] [--smoothquant-alpha A] [--exclude embed,lm_head] [--include <glob>] [--eval-corpus text.txt] [--eval-sequence-length N] [--eval-batch-size N] [--max-ppl-delta X] [--max-kl X] [--json out.json]"
);
eprintln!(
" oxidelm --eval model.bin [heldout.txt] [--tasks builtin|none|path.json] [--sequence-length N] [--batch-size N] [--memory-budget 2GiB] [--log human|json|quiet] [--profile [summary|trace]] [--detect-anomaly] [--gpu]"
);
eprintln!(
" oxidelm --tokenizer-report model.bin|tokenizer.json heldout.txt [--repeats 3] [--log human|json|quiet]"
);
eprintln!(
" oxidelm --import-tokenizer tokenizer.json out.json | --import-tokenizer vocab.json merges.txt out.json [--log human|json|quiet]"
);
eprintln!(
" oxidelm --export-weights model.bin model.safetensors [--hf] [--config-out config.json] [--log human|json|quiet] [--profile [summary|trace]] [--detect-anomaly]"
);
eprintln!(
" oxidelm --import-weights model.safetensors model.bin [--hf] [--log human|json|quiet] [--profile [summary|trace]] [--detect-anomaly]"
);
eprintln!(
" oxidelm --export-quantized model.bin model.quant.safetensors [--hf]"
);
eprintln!(
" oxidelm --import-quantized model.quant.safetensors model.bin [--hf]"
);
eprintln!(
" oxidelm --export-gguf model.bin model.gguf [--type Q4_0|Q4_K|Q5_K|Q6_K|Q8_0] [--quantize-embed]"
);
eprintln!(
" oxidelm --import-gguf model.gguf model.bin [--tokenizer tokenizer.json] [--pack-native]"
);
eprintln!(
" oxidelm --export-adapters model.bin adapters.safetensors"
);
eprintln!(
" oxidelm --import-adapters model.bin adapters.safetensors out.bin"
);
eprintln!(
" oxidelm --export-qat model.bin model.qat.bin"
);
}