use crate::inspect::tokenizer_stats;
use crate::layers::{Linear, QuantizedLinearWeight};
use crate::memory::{
count_parameters, estimate_inference_memory, estimate_kv_cache_bytes, estimate_training_memory,
format_bytes, MemoryEstimate, BYTES_PER_F32,
};
use crate::model::{CausalLM, CausalLmConfig, ModelError};
use crate::peft::LoraConfig;
use crate::quality::{
evaluate_held_out_text, evaluate_kl_held_out_text, validate_forward, ForwardValidationResult,
QualityError, FORWARD_VALIDATE_SEED,
};
use crate::state_dict::{
from_hf_state_dict, load_hf_config, load_safetensors, StateDict, StateDictError, WeightFormat,
};
use crate::tokenizer::BpeTokenizer;
use crate::trainer::{load_checkpoint, TrainerConfig, TrainerError};
use crate::{cpu_memory_stats, reset_cpu_peak_memory};
use memmap2::MmapOptions;
use safetensors::{Dtype, SafeTensors};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::error::Error;
use std::fmt;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};
pub const ANALYZE_SCHEMA_VERSION: u32 = 1;
const DEFAULT_DEAD_RMS: f64 = 1e-8;
const DEFAULT_EXPLODED_RMS: f64 = 1e3;
const FORWARD_VALIDATE_BATCH: usize = 1;
const FORWARD_VALIDATE_SEQ: usize = 8;
const INFERENCE_PRESETS: &[(usize, usize)] = &[(1, 128), (1, 512), (1, 2048), (4, 512)];
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum AnalyzeFormat {
Auto,
Oxide,
Safetensors,
Hf,
}
impl AnalyzeFormat {
pub fn parse(value: &str) -> Result<Self, AnalyzeError> {
match value.trim().to_ascii_lowercase().as_str() {
"auto" => Ok(Self::Auto),
"oxide" | "oxidelm" | "bin" => Ok(Self::Oxide),
"safetensors" | "st" => Ok(Self::Safetensors),
"hf" | "huggingface" | "llama" => Ok(Self::Hf),
other => Err(AnalyzeError::Invalid(format!(
"unknown format '{other}'; expected oxide|safetensors|hf|auto"
))),
}
}
pub fn as_str(self) -> &'static str {
match self {
Self::Auto => "auto",
Self::Oxide => "oxide",
Self::Safetensors => "safetensors",
Self::Hf => "hf",
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum FailOn {
Nan,
Inf,
Schema,
Forward,
Quality,
}
impl FailOn {
pub fn parse(value: &str) -> Result<Self, AnalyzeError> {
match value.trim().to_ascii_lowercase().as_str() {
"nan" => Ok(Self::Nan),
"inf" => Ok(Self::Inf),
"schema" => Ok(Self::Schema),
"forward" => Ok(Self::Forward),
"quality" | "eval" => Ok(Self::Quality),
other => Err(AnalyzeError::Invalid(format!(
"unknown --fail-on '{other}'; expected nan|inf|schema|forward|quality"
))),
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct AnalyzeReport {
pub schema_version: u32,
pub path: String,
pub format: String,
pub identity: IdentityReport,
pub architecture: ArchitectureReport,
pub parameters: ParameterInventory,
pub storage: StorageAccounting,
pub inference_memory: Vec<MemorySnapshot>,
#[serde(skip_serializing_if = "Option::is_none")]
pub training_memory: Option<MemorySnapshot>,
pub health: HealthReport,
pub modules: ModuleBreakdown,
pub quantization: Vec<QuantLinearReport>,
#[serde(skip_serializing_if = "Option::is_none")]
pub peft: Option<PeftInventory>,
pub kv_cache: KvCacheReport,
#[serde(skip_serializing_if = "Option::is_none")]
pub tokenizer: Option<TokenizerReport>,
#[serde(skip_serializing_if = "Option::is_none")]
pub device: Option<DeviceProbeReport>,
#[serde(skip_serializing_if = "Option::is_none")]
pub training: Option<TrainingStateReport>,
#[serde(skip_serializing_if = "Option::is_none")]
pub quality: Option<QualityGateReport>,
pub warnings: Vec<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct IdentityReport {
pub file_bytes: u64,
pub sha256: String,
pub tensor_count: usize,
pub dtype_histogram: BTreeMap<String, usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub packing: Option<BTreeMap<String, String>>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ArchitectureReport {
pub vocab_size: usize,
pub hidden_size: usize,
pub num_layers: usize,
pub num_heads: usize,
pub num_key_value_heads: usize,
pub head_dim: usize,
pub intermediate_size: usize,
pub max_sequence_length: usize,
pub rms_norm_epsilon: f32,
pub rope_theta: f32,
#[serde(skip_serializing_if = "Option::is_none")]
pub rope_scaling: Option<serde_json::Value>,
pub activation: String,
pub norm: String,
pub tie_word_embeddings: bool,
pub use_bias: bool,
pub eos_token_id: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ParameterInventory {
pub total: usize,
pub trainable: usize,
pub frozen: usize,
pub lora: usize,
pub tied_embedding_savings: usize,
pub tensors: Vec<TensorStat>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct TensorStat {
pub name: String,
pub shape: Vec<usize>,
pub dtype: String,
pub numel: usize,
pub bytes: usize,
pub nan_count: usize,
pub inf_count: usize,
pub min: f64,
pub max: f64,
pub mean: f64,
pub std: f64,
pub rms: f64,
pub sparsity: f64,
pub trainable: bool,
pub module: String,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct StorageAccounting {
pub on_disk_bytes: u64,
pub fp32_bytes: usize,
pub fp16_bytes: usize,
pub bf16_bytes: usize,
pub current_bytes: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct MemorySnapshot {
pub batch_size: usize,
pub sequence_length: usize,
pub mode: String,
pub parameter_bytes: usize,
pub amp_master_bytes: usize,
pub activation_bytes: usize,
pub cache_bytes: usize,
pub staging_bytes: usize,
pub optimizer_bytes: usize,
pub gradient_bytes: usize,
pub total_bytes: usize,
pub largest_tensor_bytes: usize,
pub notes: Vec<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct HealthReport {
pub nan_count: usize,
pub inf_count: usize,
pub layer_norms: Vec<LayerNormReport>,
pub dead_layers: Vec<usize>,
pub exploded_layers: Vec<usize>,
pub dead_rms_threshold: f64,
pub exploded_rms_threshold: f64,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct LayerNormReport {
pub layer: usize,
pub weight_rms: f64,
pub attention_rms: f64,
pub mlp_rms: f64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ModuleBreakdown {
pub attention: ModuleStats,
pub mlp: ModuleStats,
pub embed: ModuleStats,
pub lm_head: ModuleStats,
pub norm: ModuleStats,
pub lora: ModuleStats,
pub other: ModuleStats,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ModuleStats {
pub parameters: usize,
pub bytes: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct QuantLinearReport {
pub name: String,
pub scheme: String,
pub bits: u32,
pub group_size: Option<usize>,
pub in_features: usize,
pub out_features: usize,
pub packed_bytes: usize,
pub scale_count: usize,
pub scale_min: f64,
pub scale_max: f64,
pub scale_mean: f64,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct PeftInventory {
pub kind: String,
pub rank: usize,
pub alpha: f32,
pub targets: Vec<String>,
pub trainable_fraction: f64,
pub adapter_parameters: usize,
pub adapter_storage_bytes: usize,
pub dora: bool,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct KvCacheReport {
pub bytes_per_token_fp32: usize,
pub bytes_per_token_fp16: usize,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bytes_per_token_int8: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bytes_per_token_fp8: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bytes_per_token_nf4: Option<usize>,
pub presets: Vec<KvCachePreset>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct KvCachePreset {
pub batch_size: usize,
pub sequence_length: usize,
pub fp32_bytes: usize,
pub fp16_bytes: usize,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub int8_bytes: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fp8_bytes: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub nf4_bytes: Option<usize>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct TokenizerReport {
pub vocab_size: usize,
pub merge_count: usize,
pub merge_depth: usize,
pub byte_token_count: usize,
pub special_token_count: usize,
pub merge_token_count: usize,
pub specials: BTreeMap<String, usize>,
pub min_token_bytes: usize,
pub max_token_bytes: usize,
pub average_token_bytes: f64,
pub sample_tokens_per_character: f64,
pub sample_bytes_per_token: f64,
pub round_trip_failures: usize,
pub round_trip_checked_segments: usize,
#[serde(skip_serializing_if = "Option::is_none")]
pub corpus: Option<TokenizerCorpusReport>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct TokenizerCorpusReport {
pub path: String,
pub bytes: usize,
pub characters: usize,
pub tokens: usize,
pub fertility_tokens_per_character: f64,
pub bytes_per_token: f64,
pub unknown_tokens: usize,
pub round_trip_failures: usize,
pub round_trip_checked_segments: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct DeviceProbeReport {
pub cpu_parallelism: usize,
pub rayon_threads: usize,
pub cpu_live_alloc_bytes: usize,
pub cpu_peak_alloc_bytes: usize,
#[serde(skip_serializing_if = "Option::is_none")]
pub wgpu: Option<WgpuProbeReport>,
#[serde(skip_serializing_if = "Option::is_none")]
pub wgpu_error: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct WgpuProbeReport {
pub adapter_name: String,
pub backend: String,
pub max_buffer_bytes: usize,
pub max_buffer_size: u64,
pub max_storage_buffer_binding_size: u64,
pub max_compute_workgroup_storage_size: u32,
pub max_compute_invocations_per_workgroup: u32,
pub max_compute_workgroup_size_x: u32,
pub max_bind_groups: u32,
pub fp16: bool,
pub live_tensor_bytes: usize,
pub peak_tensor_bytes: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct AnalyzeDiffReport {
pub schema_version: u32,
pub left: String,
pub right: String,
pub config_delta: BTreeMap<String, ValueDelta>,
#[serde(skip_serializing_if = "Option::is_none")]
pub param_l2: Option<ParamL2Report>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tokenizer_drift: Option<TokenizerDriftReport>,
pub warnings: Vec<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ValueDelta {
pub left: serde_json::Value,
pub right: serde_json::Value,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ParamL2Report {
pub matched_tensors: usize,
pub left_only: Vec<String>,
pub right_only: Vec<String>,
pub shape_mismatches: Vec<String>,
pub total_l2: f64,
pub mean_l2: f64,
pub per_tensor: Vec<TensorL2Entry>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct TensorL2Entry {
pub name: String,
pub l2: f64,
pub numel: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct TokenizerDriftReport {
pub left_vocab_size: usize,
pub right_vocab_size: usize,
pub left_merge_count: usize,
pub right_merge_count: usize,
pub left_merge_depth: usize,
pub right_merge_depth: usize,
pub specials_match: bool,
pub vocab_prefix_equal: usize,
pub vocab_bytes_equal: bool,
pub merge_set_jaccard: f64,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct TrainingStateReport {
pub version: u32,
pub training_step: usize,
pub cursor: usize,
pub tokens_processed: usize,
pub best_validation_loss: Option<f32>,
pub optimizer_step: usize,
pub batch_size: usize,
pub sequence_length: usize,
pub gradient_accumulation_steps: usize,
pub activation_checkpointing: bool,
pub mixed_precision: bool,
pub amp_format: String,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct QualityGateReport {
pub forward: ForwardValidationReport,
#[serde(skip_serializing_if = "Option::is_none")]
pub eval: Option<EvalQualityReport>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ForwardValidationReport {
pub seed: u64,
pub batch_size: usize,
pub sequence_length: usize,
pub logit_nan_count: usize,
pub logit_inf_count: usize,
pub loss: f32,
pub loss_finite: bool,
pub passed: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct EvalQualityReport {
pub corpus_path: String,
pub loss: f32,
pub perplexity: f32,
pub tokens_processed: usize,
pub sequences: usize,
pub sequence_length: usize,
pub next_token_accuracy: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub kl: Option<KlQualityReport>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct KlQualityReport {
pub reference_path: String,
pub mean_kl: f64,
pub tokens_compared: usize,
}
#[derive(Debug)]
pub enum AnalyzeError {
Invalid(String),
FailGate(String),
Io(std::io::Error),
Json(serde_json::Error),
StateDict(StateDictError),
Trainer(TrainerError),
Model(ModelError),
Quality(QualityError),
}
impl fmt::Display for AnalyzeError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Invalid(message) | Self::FailGate(message) => write!(formatter, "{message}"),
Self::Io(error) => error.fmt(formatter),
Self::Json(error) => error.fmt(formatter),
Self::StateDict(error) => error.fmt(formatter),
Self::Trainer(error) => error.fmt(formatter),
Self::Model(error) => error.fmt(formatter),
Self::Quality(error) => error.fmt(formatter),
}
}
}
impl Error for AnalyzeError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Io(error) => Some(error),
Self::Json(error) => Some(error),
Self::StateDict(error) => Some(error),
Self::Trainer(error) => Some(error),
Self::Model(error) => Some(error),
Self::Quality(error) => Some(error),
Self::Invalid(_) | Self::FailGate(_) => None,
}
}
}
impl From<std::io::Error> for AnalyzeError {
fn from(error: std::io::Error) -> Self {
Self::Io(error)
}
}
impl From<serde_json::Error> for AnalyzeError {
fn from(error: serde_json::Error) -> Self {
Self::Json(error)
}
}
impl From<StateDictError> for AnalyzeError {
fn from(error: StateDictError) -> Self {
Self::StateDict(error)
}
}
impl From<TrainerError> for AnalyzeError {
fn from(error: TrainerError) -> Self {
Self::Trainer(error)
}
}
impl From<ModelError> for AnalyzeError {
fn from(error: ModelError) -> Self {
Self::Model(error)
}
}
impl From<QualityError> for AnalyzeError {
fn from(error: QualityError) -> Self {
Self::Quality(error)
}
}
#[derive(Clone, Debug)]
pub struct AnalyzeOptions {
pub format: AnalyzeFormat,
pub fail_on: Vec<FailOn>,
pub dead_rms: f64,
pub exploded_rms: f64,
pub corpus: Option<PathBuf>,
pub eval_corpus: Option<PathBuf>,
pub reference: Option<PathBuf>,
pub eval_sequence_length: Option<usize>,
pub eval_batch_size: usize,
pub max_perplexity: Option<f64>,
pub max_kl: Option<f64>,
pub probe_device: bool,
}
impl Default for AnalyzeOptions {
fn default() -> Self {
Self {
format: AnalyzeFormat::Auto,
fail_on: Vec::new(),
dead_rms: DEFAULT_DEAD_RMS,
exploded_rms: DEFAULT_EXPLODED_RMS,
corpus: None,
eval_corpus: None,
reference: None,
eval_sequence_length: None,
eval_batch_size: 1,
max_perplexity: None,
max_kl: None,
probe_device: false,
}
}
}
pub fn analyze_path(
path: impl AsRef<Path>,
options: &AnalyzeOptions,
) -> Result<AnalyzeReport, AnalyzeError> {
let path = path.as_ref();
let resolved = resolve_format(path, options.format)?;
let mut report = match resolved {
AnalyzeFormat::Oxide => analyze_oxide(path, options)?,
AnalyzeFormat::Safetensors => analyze_safetensors(path, WeightFormat::OxideLm, options)?,
AnalyzeFormat::Hf => analyze_hf(path, options)?,
AnalyzeFormat::Auto => unreachable!("resolve_format never returns Auto"),
};
if options.probe_device {
report.device = Some(probe_device());
}
apply_fail_gates(&mut report, options)?;
Ok(report)
}
fn resolve_format(path: &Path, requested: AnalyzeFormat) -> Result<AnalyzeFormat, AnalyzeError> {
if requested != AnalyzeFormat::Auto {
return Ok(requested);
}
if path.is_dir() {
return Ok(AnalyzeFormat::Hf);
}
if path
.extension()
.and_then(|ext| ext.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("safetensors"))
{
let parent = path.parent().unwrap_or_else(|| Path::new("."));
if parent.join("config.json").is_file() {
return Ok(AnalyzeFormat::Hf);
}
return Ok(AnalyzeFormat::Safetensors);
}
Ok(AnalyzeFormat::Oxide)
}
fn analyze_oxide(path: &Path, options: &AnalyzeOptions) -> Result<AnalyzeReport, AnalyzeError> {
let file_bytes = std::fs::metadata(path)?.len();
let sha256 = sha256_file(path)?;
let checkpoint = load_checkpoint(path)?;
let model = &checkpoint.model;
let config = &model.config;
let named = model.named_parameters();
let mut tensors = Vec::with_capacity(named.len());
let mut dtype_histogram = BTreeMap::new();
*dtype_histogram.entry("F32".into()).or_insert(0) += named.len();
let mut total_nan = 0usize;
let mut total_inf = 0usize;
let mut modules = ModuleBreakdown::default();
let mut trainable = 0usize;
let mut frozen = 0usize;
let mut lora_params = 0usize;
for (name, tensor) in &named {
let data = tensor.shared_data();
let stats = tensor_stats(name, &tensor.shape(), "F32", &data, tensor.requires_grad());
total_nan = total_nan.saturating_add(stats.nan_count);
total_inf = total_inf.saturating_add(stats.inf_count);
accumulate_module(&mut modules, &stats.module, stats.numel, stats.bytes);
if name.contains(".lora.") {
lora_params = lora_params.saturating_add(stats.numel);
}
if stats.trainable {
trainable = trainable.saturating_add(stats.numel);
} else {
frozen = frozen.saturating_add(stats.numel);
}
tensors.push(stats);
}
let total: usize = tensors.iter().map(|t| t.numel).sum();
let tied_savings = if config.tie_word_embeddings {
config.vocab_size.saturating_mul(config.hidden_size)
} else {
0
};
let quantization = collect_quantization(model);
let current_bytes = if quantization.is_empty() {
total.saturating_mul(BYTES_PER_F32)
} else {
quantized_current_bytes(model, total)
};
let peft = collect_peft(model, &checkpoint.trainer_config, total, lora_params);
let health = build_health(&tensors, config.num_layers, options);
let packing = Some(BTreeMap::from([
("checkpoint".into(), "oxidelm-binary".into()),
("version".into(), checkpoint.version.to_string()),
]));
let tokenizer = Some(build_tokenizer_report(
&checkpoint.tokenizer,
options.corpus.as_deref(),
)?);
let training = Some(TrainingStateReport {
version: checkpoint.version,
training_step: checkpoint.training_step,
cursor: checkpoint.cursor,
tokens_processed: checkpoint.tokens_processed,
best_validation_loss: checkpoint.best_validation_loss,
optimizer_step: checkpoint.optimizer.state().step,
batch_size: checkpoint.trainer_config.batch_size,
sequence_length: checkpoint.trainer_config.sequence_length,
gradient_accumulation_steps: checkpoint.trainer_config.gradient_accumulation_steps,
activation_checkpointing: checkpoint.trainer_config.activation_checkpointing,
mixed_precision: checkpoint.trainer_config.mixed_precision.enabled,
amp_format: checkpoint
.trainer_config
.mixed_precision
.format
.as_str()
.into(),
});
let training_memory = Some(memory_snapshot(
&estimate_training_memory(config, &checkpoint.trainer_config),
));
let mut report = build_report(
path,
"oxide",
IdentityReport {
file_bytes,
sha256,
tensor_count: tensors.len(),
dtype_histogram,
packing,
},
config,
ParameterInventory {
total,
trainable,
frozen,
lora: lora_params,
tied_embedding_savings: tied_savings,
tensors,
},
StorageAccounting {
on_disk_bytes: file_bytes,
fp32_bytes: total.saturating_mul(4),
fp16_bytes: total.saturating_mul(2),
bf16_bytes: total.saturating_mul(2),
current_bytes,
},
training_memory,
HealthReport {
nan_count: total_nan,
inf_count: total_inf,
layer_norms: health.0,
dead_layers: health.1,
exploded_layers: health.2,
dead_rms_threshold: options.dead_rms,
exploded_rms_threshold: options.exploded_rms,
},
modules,
quantization,
peft,
tokenizer,
training,
Vec::new(),
);
report.quality = Some(run_quality_gates(
model,
Some(&checkpoint.tokenizer),
Some(&checkpoint.trainer_config),
options,
)?);
Ok(report)
}
fn analyze_safetensors(
path: &Path,
weight_format: WeightFormat,
options: &AnalyzeOptions,
) -> Result<AnalyzeReport, AnalyzeError> {
let file_bytes = std::fs::metadata(path)?.len();
let sha256 = sha256_file(path)?;
let (dtype_histogram, packing) = safetensors_identity(path)?;
let state = load_safetensors(path)?;
let config = resolve_config_for_weights(path, &state, weight_format)?;
let tokenizer = try_load_sidecar_tokenizer(path, options.corpus.as_deref())?;
analyze_state_dict(
path,
if weight_format == WeightFormat::HuggingFace {
"hf"
} else {
"safetensors"
},
file_bytes,
sha256,
dtype_histogram,
packing,
&state,
&config,
weight_format,
None,
tokenizer,
None,
options,
)
}
fn analyze_hf(path: &Path, options: &AnalyzeOptions) -> Result<AnalyzeReport, AnalyzeError> {
let (weights_path, config_path) = if path.is_dir() {
let config_path = path.join("config.json");
if !config_path.is_file() {
return Err(AnalyzeError::Invalid(format!(
"HF directory missing config.json: {}",
path.display()
)));
}
let weights_path = find_safetensors_in_dir(path)?;
(weights_path, config_path)
} else {
let parent = path.parent().unwrap_or_else(|| Path::new("."));
let config_path = parent.join("config.json");
if !config_path.is_file() {
return Err(AnalyzeError::Invalid(format!(
"HF analyze requires config.json next to {}",
path.display()
)));
}
(path.to_path_buf(), config_path)
};
let file_bytes = std::fs::metadata(&weights_path)?.len();
let sha256 = sha256_file(&weights_path)?;
let (dtype_histogram, packing) = safetensors_identity(&weights_path)?;
let hf = load_hf_config(&config_path)?;
let config = hf.to_causal_lm(0)?;
let loaded = load_safetensors(&weights_path)?;
let tokenizer_root = if path.is_dir() {
path
} else {
path.parent().unwrap_or_else(|| Path::new("."))
};
let tokenizer = try_load_sidecar_tokenizer(tokenizer_root, options.corpus.as_deref())?;
analyze_state_dict(
&weights_path,
"hf",
file_bytes,
sha256,
dtype_histogram,
packing,
&loaded,
&config,
WeightFormat::HuggingFace,
None,
tokenizer,
None,
options,
)
}
fn analyze_state_dict(
path: &Path,
format: &str,
file_bytes: u64,
sha256: String,
dtype_histogram: BTreeMap<String, usize>,
packing: Option<BTreeMap<String, String>>,
state: &StateDict,
config: &CausalLmConfig,
weight_format: WeightFormat,
training_memory: Option<MemorySnapshot>,
tokenizer: Option<TokenizerReport>,
training: Option<TrainingStateReport>,
options: &AnalyzeOptions,
) -> Result<AnalyzeReport, AnalyzeError> {
let mut tensors = Vec::with_capacity(state.len());
let mut modules = ModuleBreakdown::default();
let mut total_nan = 0usize;
let mut total_inf = 0usize;
let mut lora_params = 0usize;
let mut total = 0usize;
for (name, storage) in state {
let stats = tensor_stats(name, &storage.shape, "F32", &storage.data, true);
total = total.saturating_add(stats.numel);
total_nan = total_nan.saturating_add(stats.nan_count);
total_inf = total_inf.saturating_add(stats.inf_count);
accumulate_module(&mut modules, &stats.module, stats.numel, stats.bytes);
if name.contains("lora") {
lora_params = lora_params.saturating_add(stats.numel);
}
tensors.push(stats);
}
let expected = count_parameters(config);
let mut warnings = Vec::new();
if total != expected && lora_params == 0 {
warnings.push(format!(
"tensor numel {total} differs from architecture estimate {expected}"
));
}
let tied_savings = if config.tie_word_embeddings {
config.vocab_size.saturating_mul(config.hidden_size)
} else {
0
};
let health = build_health(&tensors, config.num_layers, options);
let mut report = build_report(
path,
format,
IdentityReport {
file_bytes,
sha256,
tensor_count: tensors.len(),
dtype_histogram,
packing,
},
config,
ParameterInventory {
total,
trainable: total.saturating_sub(lora_params), frozen: 0,
lora: lora_params,
tied_embedding_savings: tied_savings,
tensors,
},
StorageAccounting {
on_disk_bytes: file_bytes,
fp32_bytes: total.saturating_mul(4),
fp16_bytes: total.saturating_mul(2),
bf16_bytes: total.saturating_mul(2),
current_bytes: file_bytes as usize,
},
training_memory,
HealthReport {
nan_count: total_nan,
inf_count: total_inf,
layer_norms: health.0,
dead_layers: health.1,
exploded_layers: health.2,
dead_rms_threshold: options.dead_rms,
exploded_rms_threshold: options.exploded_rms,
},
modules,
Vec::new(),
None,
tokenizer,
training,
warnings,
);
match materialize_model(config, state, weight_format) {
Ok(model) => {
let tokenizer = load_sidecar_tokenizer_instance(path);
report.quality = Some(run_quality_gates(
&model,
tokenizer.as_ref(),
None,
options,
)?);
}
Err(error) => {
report.warnings.push(format!(
"quality gates skipped: could not materialize model ({error})"
));
report.quality = Some(forward_failed_report(error.to_string()));
}
}
Ok(report)
}
fn build_report(
path: &Path,
format: &str,
identity: IdentityReport,
config: &CausalLmConfig,
parameters: ParameterInventory,
storage: StorageAccounting,
training_memory: Option<MemorySnapshot>,
health: HealthReport,
modules: ModuleBreakdown,
quantization: Vec<QuantLinearReport>,
peft: Option<PeftInventory>,
tokenizer: Option<TokenizerReport>,
training: Option<TrainingStateReport>,
warnings: Vec<String>,
) -> AnalyzeReport {
let head_dim = config.hidden_size / config.num_heads.max(1);
let architecture = ArchitectureReport {
vocab_size: config.vocab_size,
hidden_size: config.hidden_size,
num_layers: config.num_layers,
num_heads: config.num_heads,
num_key_value_heads: config.num_key_value_heads,
head_dim,
intermediate_size: config.intermediate_size,
max_sequence_length: config.max_sequence_length,
rms_norm_epsilon: config.rms_norm_epsilon,
rope_theta: config.rope_theta,
rope_scaling: config
.rope_scaling
.as_ref()
.and_then(|scaling| serde_json::to_value(scaling).ok()),
activation: "silu".into(),
norm: "rms_norm".into(),
tie_word_embeddings: config.tie_word_embeddings,
use_bias: config.use_bias,
eos_token_id: config.eos_token_id,
};
let inference_memory = INFERENCE_PRESETS
.iter()
.map(|(batch, seq)| {
let seq = (*seq).min(config.max_sequence_length).max(1);
memory_snapshot(&estimate_inference_memory(config, *batch, seq))
})
.collect();
let bytes_per_token_fp32 = estimate_kv_cache_bytes(config, 1, 1);
let head_dim = config.hidden_size / config.num_heads.max(1);
let kv_heads = if config.num_key_value_heads == 0 {
config.num_heads
} else {
config.num_key_value_heads
};
let quant_bytes = |format: crate::kv_quant::KvQuantization| {
crate::kv_quant::estimate_kv_cache_bytes_quantized(
config.num_layers,
1,
1,
kv_heads,
head_dim,
crate::kv_quant::KvQuantScheme {
format,
granularity: crate::kv_quant::KvGranularity::PerHead,
},
)
};
let kv_cache = KvCacheReport {
bytes_per_token_fp32,
bytes_per_token_fp16: bytes_per_token_fp32 / 2,
bytes_per_token_int8: Some(quant_bytes(crate::kv_quant::KvQuantization::Int8)),
bytes_per_token_fp8: Some(quant_bytes(crate::kv_quant::KvQuantization::Fp8E4M3)),
bytes_per_token_nf4: Some(quant_bytes(crate::kv_quant::KvQuantization::Nf4)),
presets: INFERENCE_PRESETS
.iter()
.map(|(batch, seq)| {
let seq = (*seq).min(config.max_sequence_length).max(1);
let fp32 = estimate_kv_cache_bytes(config, *batch, seq);
let quant_preset = |format: crate::kv_quant::KvQuantization| {
crate::kv_quant::estimate_kv_cache_bytes_quantized(
config.num_layers,
*batch,
seq,
kv_heads,
head_dim,
crate::kv_quant::KvQuantScheme {
format,
granularity: crate::kv_quant::KvGranularity::PerHead,
},
)
};
KvCachePreset {
batch_size: *batch,
sequence_length: seq,
fp32_bytes: fp32,
fp16_bytes: fp32 / 2,
int8_bytes: Some(quant_preset(crate::kv_quant::KvQuantization::Int8)),
fp8_bytes: Some(quant_preset(crate::kv_quant::KvQuantization::Fp8E4M3)),
nf4_bytes: Some(quant_preset(crate::kv_quant::KvQuantization::Nf4)),
}
})
.collect(),
};
AnalyzeReport {
schema_version: ANALYZE_SCHEMA_VERSION,
path: path.display().to_string(),
format: format.into(),
identity,
architecture,
parameters,
storage,
inference_memory,
training_memory,
health,
modules,
quantization,
peft,
kv_cache,
tokenizer,
device: None,
training,
quality: None,
warnings,
}
}
fn apply_fail_gates(
report: &mut AnalyzeReport,
options: &AnalyzeOptions,
) -> Result<(), AnalyzeError> {
let fail_on = &options.fail_on;
if fail_on.is_empty() {
return Ok(());
}
let mut failures = Vec::new();
if fail_on.contains(&FailOn::Nan) && report.health.nan_count > 0 {
failures.push(format!("nan_count={}", report.health.nan_count));
}
if fail_on.contains(&FailOn::Inf) && report.health.inf_count > 0 {
failures.push(format!("inf_count={}", report.health.inf_count));
}
if fail_on.contains(&FailOn::Schema) {
if report.schema_version != ANALYZE_SCHEMA_VERSION {
failures.push(format!(
"schema_version={} (expected {ANALYZE_SCHEMA_VERSION})",
report.schema_version
));
}
if report.architecture.hidden_size == 0 || report.architecture.num_layers == 0 {
failures.push("architecture schema invalid (zero hidden_size or num_layers)".into());
}
if report.parameters.total == 0 {
failures.push("schema: zero parameters".into());
}
}
if fail_on.contains(&FailOn::Forward) {
match report.quality.as_ref().map(|q| &q.forward) {
Some(forward) if forward.passed => {}
Some(forward) => {
failures.push(format!(
"forward validation failed (nan={} inf={} loss_finite={} error={:?})",
forward.logit_nan_count,
forward.logit_inf_count,
forward.loss_finite,
forward.error
));
}
None => failures.push("forward validation missing from report".into()),
}
}
if fail_on.contains(&FailOn::Quality) {
match report.quality.as_ref() {
None => failures.push("quality gates missing from report".into()),
Some(quality) => {
if !quality.forward.passed {
failures.push("quality: forward validation failed".into());
}
if options.eval_corpus.is_some() {
match &quality.eval {
None => failures.push("quality eval missing (--eval-corpus set)".into()),
Some(eval) => {
if let Some(error) = &eval.error {
failures.push(format!("quality eval error: {error}"));
} else if !eval.perplexity.is_finite() || !eval.loss.is_finite() {
failures.push(format!(
"quality eval non-finite (loss={} ppl={})",
eval.loss, eval.perplexity
));
}
if let Some(max_ppl) = options.max_perplexity {
if (eval.perplexity as f64) > max_ppl {
failures.push(format!(
"perplexity {:.3} exceeds max_perplexity {max_ppl}",
eval.perplexity
));
}
}
if options.reference.is_some() {
match &eval.kl {
None => {
failures.push("quality KL missing (--reference set)".into())
}
Some(kl) => {
if !kl.mean_kl.is_finite() {
failures.push(format!(
"quality KL non-finite ({})",
kl.mean_kl
));
}
if let Some(max_kl) = options.max_kl {
if kl.mean_kl > max_kl {
failures.push(format!(
"mean_kl {:.6} exceeds max_kl {max_kl}",
kl.mean_kl
));
}
}
}
}
}
}
}
}
}
}
}
if failures.is_empty() {
Ok(())
} else {
Err(AnalyzeError::FailGate(format!(
"analyze --fail-on triggered: {}",
failures.join(", ")
)))
}
}
fn resolve_config_for_weights(
path: &Path,
state: &StateDict,
weight_format: WeightFormat,
) -> Result<CausalLmConfig, AnalyzeError> {
let parent = path.parent().unwrap_or_else(|| Path::new("."));
let hf_config = parent.join("config.json");
let oxide_config = parent.join("oxidelm_config.json");
if weight_format == WeightFormat::HuggingFace || hf_config.is_file() {
if hf_config.is_file() {
return Ok(load_hf_config(&hf_config)?.to_causal_lm(0)?);
}
}
if oxide_config.is_file() {
let file = File::open(&oxide_config)?;
let config: CausalLmConfig = serde_json::from_reader(file)?;
config.validate().map_err(|error| {
AnalyzeError::Invalid(format!("invalid oxide config: {error}"))
})?;
return Ok(config);
}
if hf_config.is_file() {
if let Ok(hf) = load_hf_config(&hf_config) {
if let Ok(config) = hf.to_causal_lm(0) {
return Ok(config);
}
}
let file = File::open(&hf_config)?;
if let Ok(config) = serde_json::from_reader::<_, CausalLmConfig>(file) {
config.validate().map_err(|error| {
AnalyzeError::Invalid(format!("invalid config.json: {error}"))
})?;
return Ok(config);
}
}
infer_config_from_state(state, weight_format)
}
fn infer_config_from_state(
state: &StateDict,
weight_format: WeightFormat,
) -> Result<CausalLmConfig, AnalyzeError> {
let embed_key = match weight_format {
WeightFormat::HuggingFace => "model.embed_tokens.weight",
WeightFormat::OxideLm => "tok_embeddings.weight",
};
let embed = state.get(embed_key).ok_or_else(|| {
AnalyzeError::Invalid(format!(
"cannot infer architecture: missing '{embed_key}' (provide config.json)"
))
})?;
if embed.shape.len() != 2 {
return Err(AnalyzeError::Invalid(
"embedding weight must be rank-2".into(),
));
}
let (vocab_size, hidden_size) = (embed.shape[0], embed.shape[1]);
let layer_prefix = match weight_format {
WeightFormat::HuggingFace => "model.layers.",
WeightFormat::OxideLm => "layers.",
};
let mut max_layer = 0usize;
let mut intermediate = 0usize;
let mut kv_width = 0usize;
for name in state.keys() {
if let Some(rest) = name.strip_prefix(layer_prefix) {
if let Some((index, _)) = rest.split_once('.') {
if let Ok(index) = index.parse::<usize>() {
max_layer = max_layer.max(index + 1);
}
}
}
let is_gate = name.ends_with("gate_proj.weight");
if is_gate {
if let Some(tensor) = state.get(name) {
if tensor.shape.len() == 2 {
intermediate = match weight_format {
WeightFormat::OxideLm => tensor.shape[1],
WeightFormat::HuggingFace => tensor.shape[0],
};
}
}
}
if name.ends_with("k_proj.weight") {
if let Some(tensor) = state.get(name) {
if tensor.shape.len() == 2 {
kv_width = match weight_format {
WeightFormat::OxideLm => tensor.shape[1],
WeightFormat::HuggingFace => tensor.shape[0],
};
}
}
}
}
if max_layer == 0 || intermediate == 0 {
return Err(AnalyzeError::Invalid(
"cannot infer layers/intermediate_size from weights; provide config.json".into(),
));
}
let num_heads = 1.max(hidden_size / 64.min(hidden_size).max(1));
let num_heads = (1..=hidden_size)
.rev()
.find(|h| hidden_size % h == 0 && (hidden_size / h) % 2 == 0)
.unwrap_or(num_heads);
let head_dim = hidden_size / num_heads;
let num_key_value_heads = if kv_width > 0 && head_dim > 0 {
(kv_width / head_dim).max(1)
} else {
num_heads
};
let tie = !state.contains_key("lm_head.weight")
|| state
.get("lm_head.weight")
.zip(state.get(embed_key))
.is_some_and(|(head, emb)| head.data == emb.data);
let config = CausalLmConfig {
vocab_size,
hidden_size,
num_layers: max_layer,
num_heads,
num_key_value_heads,
intermediate_size: intermediate,
max_sequence_length: 2048,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: vocab_size.saturating_sub(1),
seed: 0,
initializer_std: 0.02,
tie_word_embeddings: tie,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: state.keys().any(|name| name.ends_with(".bias")),
};
config
.validate()
.map_err(|error| AnalyzeError::Invalid(format!("inferred config invalid: {error}")))?;
Ok(config)
}
fn find_safetensors_in_dir(dir: &Path) -> Result<PathBuf, AnalyzeError> {
let preferred = [
"model.safetensors",
"model.oxide.safetensors",
"pytorch_model.safetensors",
];
for name in preferred {
let path = dir.join(name);
if path.is_file() {
return Ok(path);
}
}
let mut matches = Vec::new();
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
if path
.extension()
.and_then(|ext| ext.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("safetensors"))
{
matches.push(path);
}
}
matches.sort();
matches.into_iter().next().ok_or_else(|| {
AnalyzeError::Invalid(format!(
"no .safetensors file found in {}",
dir.display()
))
})
}
fn safetensors_identity(
path: &Path,
) -> Result<(BTreeMap<String, usize>, Option<BTreeMap<String, String>>), AnalyzeError> {
let file = File::open(path)?;
let mmap = unsafe { MmapOptions::new().map(&file)? };
let tensors = SafeTensors::deserialize(&mmap).map_err(|error| {
AnalyzeError::Invalid(format!("safetensors deserialize failed: {error}"))
})?;
let mut histogram = BTreeMap::new();
for name in tensors.names() {
let tensor = tensors.tensor(name).map_err(|error| {
AnalyzeError::Invalid(format!("safetensors tensor '{name}': {error}"))
})?;
let label = dtype_label(tensor.dtype());
*histogram.entry(label).or_insert(0) += 1;
}
let packing = Some(BTreeMap::from([("container".into(), "safetensors".into())]));
Ok((histogram, packing))
}
fn dtype_label(dtype: Dtype) -> String {
match dtype {
Dtype::F64 => "F64".into(),
Dtype::F32 => "F32".into(),
Dtype::F16 => "F16".into(),
Dtype::BF16 => "BF16".into(),
Dtype::I64 => "I64".into(),
Dtype::I32 => "I32".into(),
Dtype::I16 => "I16".into(),
Dtype::I8 => "I8".into(),
Dtype::U8 => "U8".into(),
Dtype::BOOL => "BOOL".into(),
Dtype::F8_E4M3 => "F8_E4M3".into(),
Dtype::F8_E5M2 => "F8_E5M2".into(),
Dtype::U16 => "U16".into(),
Dtype::U32 => "U32".into(),
Dtype::U64 => "U64".into(),
_ => format!("{dtype:?}"),
}
}
fn tensor_stats(
name: &str,
shape: &[usize],
dtype: &str,
data: &[f32],
trainable: bool,
) -> TensorStat {
let numel = data.len();
let mut nan_count = 0usize;
let mut inf_count = 0usize;
let mut zero_count = 0usize;
let mut min = f64::INFINITY;
let mut max = f64::NEG_INFINITY;
let mut sum = 0.0f64;
let mut sum_sq = 0.0f64;
for &value in data {
if value.is_nan() {
nan_count += 1;
continue;
}
if value.is_infinite() {
inf_count += 1;
continue;
}
if value == 0.0 {
zero_count += 1;
}
let v = f64::from(value);
min = min.min(v);
max = max.max(v);
sum += v;
sum_sq += v * v;
}
let finite = numel.saturating_sub(nan_count).saturating_sub(inf_count);
let (mean, std, rms) = if finite == 0 {
(0.0, 0.0, 0.0)
} else {
let n = finite as f64;
let mean = sum / n;
let variance = (sum_sq / n - mean * mean).max(0.0);
(mean, variance.sqrt(), (sum_sq / n).sqrt())
};
if !min.is_finite() {
min = 0.0;
max = 0.0;
}
TensorStat {
name: name.into(),
shape: shape.to_vec(),
dtype: dtype.into(),
numel,
bytes: numel.saturating_mul(BYTES_PER_F32),
nan_count,
inf_count,
min,
max,
mean,
std,
rms,
sparsity: if numel == 0 {
0.0
} else {
zero_count as f64 / numel as f64
},
trainable,
module: classify_module(name),
}
}
fn classify_module(name: &str) -> String {
if name.contains(".lora.") || name.contains("lora_") {
"lora".into()
} else if name.starts_with("tok_embeddings")
|| name.contains("embed_tokens")
|| name.contains("tok_embeddings")
{
"embed".into()
} else if name.starts_with("lm_head") || name == "lm_head.weight" {
"lm_head".into()
} else if name.contains("attention") || name.contains("self_attn") {
"attention".into()
} else if name.contains("feed_forward") || name.contains("mlp") {
"mlp".into()
} else if name.contains("norm") {
"norm".into()
} else {
"other".into()
}
}
fn accumulate_module(modules: &mut ModuleBreakdown, module: &str, params: usize, bytes: usize) {
let target = match module {
"attention" => &mut modules.attention,
"mlp" => &mut modules.mlp,
"embed" => &mut modules.embed,
"lm_head" => &mut modules.lm_head,
"norm" => &mut modules.norm,
"lora" => &mut modules.lora,
_ => &mut modules.other,
};
target.parameters = target.parameters.saturating_add(params);
target.bytes = target.bytes.saturating_add(bytes);
}
fn build_health(
tensors: &[TensorStat],
num_layers: usize,
options: &AnalyzeOptions,
) -> (Vec<LayerNormReport>, Vec<usize>, Vec<usize>) {
let mut layer_norms = Vec::with_capacity(num_layers);
let mut dead = Vec::new();
let mut exploded = Vec::new();
for layer in 0..num_layers {
let prefix_oxide = format!("layers.{layer}.");
let prefix_hf = format!("model.layers.{layer}.");
let mut weight_sum_sq = 0.0;
let mut weight_n = 0.0;
let mut attn_sum_sq = 0.0;
let mut attn_n = 0.0;
let mut mlp_sum_sq = 0.0;
let mut mlp_n = 0.0;
for tensor in tensors {
let belongs = tensor.name.starts_with(&prefix_oxide)
|| tensor.name.starts_with(&prefix_hf);
if !belongs {
continue;
}
let contrib = tensor.rms * tensor.rms * tensor.numel as f64;
weight_sum_sq += contrib;
weight_n += tensor.numel as f64;
if tensor.module == "attention" {
attn_sum_sq += contrib;
attn_n += tensor.numel as f64;
} else if tensor.module == "mlp" {
mlp_sum_sq += contrib;
mlp_n += tensor.numel as f64;
}
}
let weight_rms = if weight_n > 0.0 {
(weight_sum_sq / weight_n).sqrt()
} else {
0.0
};
let attention_rms = if attn_n > 0.0 {
(attn_sum_sq / attn_n).sqrt()
} else {
0.0
};
let mlp_rms = if mlp_n > 0.0 {
(mlp_sum_sq / mlp_n).sqrt()
} else {
0.0
};
if weight_rms < options.dead_rms {
dead.push(layer);
}
if weight_rms > options.exploded_rms {
exploded.push(layer);
}
layer_norms.push(LayerNormReport {
layer,
weight_rms,
attention_rms,
mlp_rms,
});
}
(layer_norms, dead, exploded)
}
fn collect_quantization(model: &CausalLM) -> Vec<QuantLinearReport> {
let mut reports = Vec::new();
for (layer_index, layer) in model.layers.iter().enumerate() {
let prefix = format!("layers.{layer_index}");
push_quant(
&mut reports,
&format!("{prefix}.attention.q_proj"),
&layer.attention.q_proj,
);
push_quant(
&mut reports,
&format!("{prefix}.attention.k_proj"),
&layer.attention.k_proj,
);
push_quant(
&mut reports,
&format!("{prefix}.attention.v_proj"),
&layer.attention.v_proj,
);
push_quant(
&mut reports,
&format!("{prefix}.attention.o_proj"),
&layer.attention.o_proj,
);
push_quant(
&mut reports,
&format!("{prefix}.feed_forward.gate_proj"),
&layer.feed_forward.gate_proj,
);
push_quant(
&mut reports,
&format!("{prefix}.feed_forward.up_proj"),
&layer.feed_forward.up_proj,
);
push_quant(
&mut reports,
&format!("{prefix}.feed_forward.down_proj"),
&layer.feed_forward.down_proj,
);
}
push_quant(&mut reports, "lm_head", &model.lm_head);
reports
}
fn push_quant(reports: &mut Vec<QuantLinearReport>, name: &str, linear: &Linear) {
if let Some(quant) = linear.quantized_weight() {
reports.push(quant_report(name, quant));
} else if let Some(config) = linear.qat_config() {
reports.push(QuantLinearReport {
name: name.into(),
scheme: format!("qat-{}", config.as_str()),
bits: config.format.bits(),
group_size: config.group_size,
in_features: linear.in_features,
out_features: linear.out_features,
packed_bytes: 0,
scale_count: 0,
scale_min: 0.0,
scale_max: 0.0,
scale_mean: 0.0,
});
}
}
fn quant_report(name: &str, quant: &QuantizedLinearWeight) -> QuantLinearReport {
let scales = quant.scales();
let mut min = f64::INFINITY;
let mut max = f64::NEG_INFINITY;
let mut sum = 0.0;
for &scale in scales {
let v = f64::from(scale);
min = min.min(v);
max = max.max(v);
sum += v;
}
if !min.is_finite() {
min = 0.0;
max = 0.0;
}
QuantLinearReport {
name: name.into(),
scheme: format!("weight-only-{}", quant.format().as_str()),
bits: quant.format().bits(),
group_size: quant.group_size(),
in_features: quant.in_features(),
out_features: quant.out_features(),
packed_bytes: quant.storage_bytes(),
scale_count: scales.len(),
scale_min: min,
scale_max: max,
scale_mean: if scales.is_empty() {
0.0
} else {
sum / scales.len() as f64
},
}
}
fn quantized_current_bytes(model: &CausalLM, total_params: usize) -> usize {
let mut packed = 0usize;
let mut covered_fp32 = 0usize;
let mut visit = |linear: &Linear| {
if let Some(quant) = linear.quantized_weight() {
packed = packed.saturating_add(quant.storage_bytes());
covered_fp32 = covered_fp32
.saturating_add(linear.in_features.saturating_mul(linear.out_features) * 4);
if let Some(bias) = &linear.bias {
covered_fp32 = covered_fp32.saturating_add(bias.numel() * 4);
}
}
};
for layer in &model.layers {
visit(&layer.attention.q_proj);
visit(&layer.attention.k_proj);
visit(&layer.attention.v_proj);
visit(&layer.attention.o_proj);
visit(&layer.feed_forward.gate_proj);
visit(&layer.feed_forward.up_proj);
visit(&layer.feed_forward.down_proj);
}
visit(&model.lm_head);
let base = total_params.saturating_mul(BYTES_PER_F32);
base.saturating_sub(covered_fp32).saturating_add(packed)
}
fn collect_peft(
model: &CausalLM,
trainer: &TrainerConfig,
total_params: usize,
lora_params: usize,
) -> Option<PeftInventory> {
let config = trainer.low_memory.lora.as_ref()?;
Some(peft_from_config(config, total_params, lora_params, model))
}
fn peft_from_config(
config: &LoraConfig,
total_params: usize,
lora_params: usize,
_model: &CausalLM,
) -> PeftInventory {
let adapter_parameters = if lora_params > 0 {
lora_params
} else {
lora_params
};
PeftInventory {
kind: if config.dora { "dora" } else { "lora" }.into(),
rank: config.rank,
alpha: config.alpha,
targets: config.targets.clone(),
trainable_fraction: if total_params == 0 {
0.0
} else {
adapter_parameters as f64 / total_params as f64
},
adapter_parameters,
adapter_storage_bytes: adapter_parameters.saturating_mul(BYTES_PER_F32),
dora: config.dora,
}
}
fn memory_snapshot(estimate: &MemoryEstimate) -> MemorySnapshot {
MemorySnapshot {
batch_size: estimate.batch_size,
sequence_length: estimate.sequence_length,
mode: match estimate.mode {
crate::memory::MemoryMode::Inference => "inference".into(),
crate::memory::MemoryMode::Training => "training".into(),
},
parameter_bytes: estimate.parameter_bytes,
amp_master_bytes: estimate.amp_master_bytes,
activation_bytes: estimate.activation_bytes,
cache_bytes: estimate.cache_bytes,
staging_bytes: estimate.staging_bytes,
optimizer_bytes: estimate.optimizer_bytes,
gradient_bytes: estimate.gradient_bytes,
total_bytes: estimate.total_bytes,
largest_tensor_bytes: estimate.largest_tensor_bytes,
notes: estimate.notes.clone(),
}
}
pub fn format_analyze_report(report: &AnalyzeReport) -> String {
use std::fmt::Write as _;
let mut out = String::new();
let _ = writeln!(
out,
"analyze schema_version={} format={} path={}",
report.schema_version, report.format, report.path
);
let _ = writeln!(
out,
"identity file_bytes={} sha256={} tensors={} dtypes={:?}",
report.identity.file_bytes,
report.identity.sha256,
report.identity.tensor_count,
report.identity.dtype_histogram
);
let arch = &report.architecture;
let _ = writeln!(
out,
"architecture layers={} d_model={} heads={} kv_heads={} head_dim={} ffn={} vocab={} max_seq={} act={} norm={} tie={}",
arch.num_layers,
arch.hidden_size,
arch.num_heads,
arch.num_key_value_heads,
arch.head_dim,
arch.intermediate_size,
arch.vocab_size,
arch.max_sequence_length,
arch.activation,
arch.norm,
arch.tie_word_embeddings
);
let params = &report.parameters;
let _ = writeln!(
out,
"parameters total={} trainable={} frozen={} lora={} tied_savings={}",
params.total, params.trainable, params.frozen, params.lora, params.tied_embedding_savings
);
let storage = &report.storage;
let _ = writeln!(
out,
"storage on_disk={} fp32={} fp16={} bf16={} current={}",
format_bytes(storage.on_disk_bytes as usize),
format_bytes(storage.fp32_bytes),
format_bytes(storage.fp16_bytes),
format_bytes(storage.bf16_bytes),
format_bytes(storage.current_bytes)
);
let _ = writeln!(
out,
"health nan={} inf={} dead_layers={:?} exploded_layers={:?}",
report.health.nan_count,
report.health.inf_count,
report.health.dead_layers,
report.health.exploded_layers
);
let modules = &report.modules;
let _ = writeln!(
out,
"modules attention_params={} mlp_params={} embed_params={} lm_head_params={} norm_params={} lora_params={}",
modules.attention.parameters,
modules.mlp.parameters,
modules.embed.parameters,
modules.lm_head.parameters,
modules.norm.parameters,
modules.lora.parameters
);
if !report.quantization.is_empty() {
let _ = writeln!(out, "quantization linears={}", report.quantization.len());
for quant in &report.quantization {
let _ = writeln!(
out,
" quant name={} scheme={} bits={} packed={}",
quant.name,
quant.scheme,
quant.bits,
format_bytes(quant.packed_bytes)
);
}
}
if let Some(peft) = &report.peft {
let _ = writeln!(
out,
"peft kind={} rank={} alpha={} adapter_params={} trainable_fraction={:.6}",
peft.kind, peft.rank, peft.alpha, peft.adapter_parameters, peft.trainable_fraction
);
}
let _ = writeln!(
out,
"kv_cache bytes_per_token_fp32={} bytes_per_token_fp16={} int8={:?} fp8={:?} nf4={:?}",
report.kv_cache.bytes_per_token_fp32,
report.kv_cache.bytes_per_token_fp16,
report.kv_cache.bytes_per_token_int8,
report.kv_cache.bytes_per_token_fp8,
report.kv_cache.bytes_per_token_nf4
);
for preset in &report.inference_memory {
let _ = writeln!(
out,
"inference_memory batch={} seq={} total={}",
preset.batch_size,
preset.sequence_length,
format_bytes(preset.total_bytes)
);
}
if let Some(training) = &report.training_memory {
let _ = writeln!(
out,
"training_memory batch={} seq={} total={}",
training.batch_size,
training.sequence_length,
format_bytes(training.total_bytes)
);
}
if let Some(tokenizer) = &report.tokenizer {
let _ = writeln!(
out,
"tokenizer vocab={} merges={} merge_depth={} fertility_tokens_per_char={:.3} round_trip_failures={}",
tokenizer.vocab_size,
tokenizer.merge_count,
tokenizer.merge_depth,
tokenizer.sample_tokens_per_character,
tokenizer.round_trip_failures
);
if let Some(corpus) = &tokenizer.corpus {
let _ = writeln!(
out,
"tokenizer_corpus path={} tokens={} fertility_tokens_per_char={:.3} round_trip_failures={}",
corpus.path,
corpus.tokens,
corpus.fertility_tokens_per_character,
corpus.round_trip_failures
);
}
}
if let Some(device) = &report.device {
let _ = writeln!(
out,
"device cpu_parallelism={} rayon_threads={} cpu_peak_alloc={} cpu_live_alloc={}",
device.cpu_parallelism,
device.rayon_threads,
format_bytes(device.cpu_peak_alloc_bytes),
format_bytes(device.cpu_live_alloc_bytes)
);
if let Some(wgpu) = &device.wgpu {
let _ = writeln!(
out,
"device_wgpu adapter={} backend={} max_buffer={} max_storage_binding={} peak_alloc={}",
wgpu.adapter_name,
wgpu.backend,
format_bytes(wgpu.max_buffer_bytes),
format_bytes(wgpu.max_storage_buffer_binding_size as usize),
format_bytes(wgpu.peak_tensor_bytes)
);
} else if let Some(error) = &device.wgpu_error {
let _ = writeln!(out, "device_wgpu unavailable error={error}");
}
}
if let Some(quality) = &report.quality {
let forward = &quality.forward;
let _ = writeln!(
out,
"quality_forward passed={} seed={} seq={} loss={:.5} logit_nan={} logit_inf={}",
forward.passed,
forward.seed,
forward.sequence_length,
forward.loss,
forward.logit_nan_count,
forward.logit_inf_count
);
if let Some(error) = &forward.error {
let _ = writeln!(out, "quality_forward_error {error}");
}
if let Some(eval) = &quality.eval {
let _ = writeln!(
out,
"quality_eval corpus={} loss={:.5} perplexity={:.3} tokens={} accuracy={:.4}",
eval.corpus_path,
eval.loss,
eval.perplexity,
eval.tokens_processed,
eval.next_token_accuracy
);
if let Some(kl) = &eval.kl {
let _ = writeln!(
out,
"quality_kl reference={} mean_kl={:.6} tokens={}",
kl.reference_path, kl.mean_kl, kl.tokens_compared
);
}
if let Some(error) = &eval.error {
let _ = writeln!(out, "quality_eval_error {error}");
}
}
}
for warning in &report.warnings {
let _ = writeln!(out, "warning {warning}");
}
out
}
pub fn write_analyze_json(report: &AnalyzeReport, path: impl AsRef<Path>) -> Result<(), AnalyzeError> {
let file = File::create(path)?;
serde_json::to_writer_pretty(file, report)?;
Ok(())
}
pub fn analyze_diff(
left: impl AsRef<Path>,
right: impl AsRef<Path>,
options: &AnalyzeOptions,
) -> Result<AnalyzeDiffReport, AnalyzeError> {
let left = left.as_ref();
let right = right.as_ref();
let left_report = analyze_path(left, options)?;
let right_report = analyze_path(right, options)?;
let config_delta = config_delta(&left_report.architecture, &right_report.architecture);
let mut warnings = Vec::new();
let param_l2 = match (load_weight_map(left, options.format), load_weight_map(right, options.format))
{
(Ok(left_map), Ok(right_map)) => Some(param_l2_maps(&left_map, &right_map)),
(Err(left_err), Ok(_)) => {
warnings.push(format!("left weights unavailable for L2: {left_err}"));
None
}
(Ok(_), Err(right_err)) => {
warnings.push(format!("right weights unavailable for L2: {right_err}"));
None
}
(Err(left_err), Err(right_err)) => {
warnings.push(format!(
"weights unavailable for L2: left={left_err}; right={right_err}"
));
None
}
};
let tokenizer_drift = match (
load_tokenizer_for_diff(left),
load_tokenizer_for_diff(right),
) {
(Some(left_tok), Some(right_tok)) => Some(tokenizer_drift(&left_tok, &right_tok)),
(None, None) => {
warnings.push("tokenizer drift skipped: neither side has a tokenizer".into());
None
}
(None, Some(_)) => {
warnings.push("tokenizer drift skipped: left tokenizer missing".into());
None
}
(Some(_), None) => {
warnings.push("tokenizer drift skipped: right tokenizer missing".into());
None
}
};
Ok(AnalyzeDiffReport {
schema_version: ANALYZE_SCHEMA_VERSION,
left: left.display().to_string(),
right: right.display().to_string(),
config_delta,
param_l2,
tokenizer_drift,
warnings,
})
}
pub fn format_analyze_diff_report(report: &AnalyzeDiffReport) -> String {
use std::fmt::Write as _;
let mut out = String::new();
let _ = writeln!(
out,
"analyze_diff schema_version={} left={} right={}",
report.schema_version, report.left, report.right
);
if report.config_delta.is_empty() {
let _ = writeln!(out, "config_delta (none)");
} else {
let _ = writeln!(out, "config_delta fields={}", report.config_delta.len());
for (key, delta) in &report.config_delta {
let _ = writeln!(out, " {key}: {} -> {}", delta.left, delta.right);
}
}
if let Some(l2) = &report.param_l2 {
let _ = writeln!(
out,
"param_l2 matched={} total_l2={:.6} mean_l2={:.6} left_only={} right_only={} shape_mismatches={}",
l2.matched_tensors,
l2.total_l2,
l2.mean_l2,
l2.left_only.len(),
l2.right_only.len(),
l2.shape_mismatches.len()
);
for entry in l2.per_tensor.iter().take(16) {
let _ = writeln!(
out,
" tensor name={} l2={:.6} numel={}",
entry.name, entry.l2, entry.numel
);
}
if l2.per_tensor.len() > 16 {
let _ = writeln!(out, " ... {} more tensors", l2.per_tensor.len() - 16);
}
}
if let Some(drift) = &report.tokenizer_drift {
let _ = writeln!(
out,
"tokenizer_drift vocab={}->{} merges={}->{} depth={}->{} specials_match={} vocab_bytes_equal={} merge_jaccard={:.4}",
drift.left_vocab_size,
drift.right_vocab_size,
drift.left_merge_count,
drift.right_merge_count,
drift.left_merge_depth,
drift.right_merge_depth,
drift.specials_match,
drift.vocab_bytes_equal,
drift.merge_set_jaccard
);
}
for warning in &report.warnings {
let _ = writeln!(out, "warning {warning}");
}
out
}
pub fn write_analyze_diff_json(
report: &AnalyzeDiffReport,
path: impl AsRef<Path>,
) -> Result<(), AnalyzeError> {
let file = File::create(path)?;
serde_json::to_writer_pretty(file, report)?;
Ok(())
}
fn build_tokenizer_report(
tokenizer: &BpeTokenizer,
corpus: Option<&Path>,
) -> Result<TokenizerReport, AnalyzeError> {
let stats = tokenizer_stats(tokenizer);
let (sample_failures, sample_segments) =
round_trip_check(tokenizer, crate::inspect::TOKENIZER_SAMPLE.as_bytes());
let corpus_report = if let Some(path) = corpus {
let bytes = std::fs::read(path)?;
Some(corpus_fertility(tokenizer, path, &bytes))
} else {
None
};
let (round_trip_failures, round_trip_checked_segments) = if let Some(ref corpus) = corpus_report
{
(
sample_failures.saturating_add(corpus.round_trip_failures),
sample_segments.saturating_add(corpus.round_trip_checked_segments),
)
} else {
(sample_failures, sample_segments)
};
Ok(TokenizerReport {
vocab_size: stats.vocab_size,
merge_count: stats.merge_count,
merge_depth: merge_depth(tokenizer),
byte_token_count: stats.byte_token_count,
special_token_count: stats.special_token_count,
merge_token_count: stats.merge_token_count,
specials: BTreeMap::from([
("pad".into(), stats.special_tokens.pad),
("unk".into(), stats.special_tokens.unk),
("bos".into(), stats.special_tokens.bos),
("eos".into(), stats.special_tokens.eos),
]),
min_token_bytes: stats.min_token_bytes,
max_token_bytes: stats.max_token_bytes,
average_token_bytes: stats.average_token_bytes,
sample_tokens_per_character: stats.sample_tokens_per_character,
sample_bytes_per_token: stats.sample_bytes_per_token,
round_trip_failures,
round_trip_checked_segments,
corpus: corpus_report,
})
}
fn corpus_fertility(
tokenizer: &BpeTokenizer,
path: &Path,
bytes: &[u8],
) -> TokenizerCorpusReport {
let characters = match std::str::from_utf8(bytes) {
Ok(text) => text.chars().count(),
Err(_) => bytes.len(),
};
let tokens = tokenizer.encode_bytes(bytes);
let unk = tokenizer.special_tokens.unk;
let unknown_tokens = tokens.iter().filter(|&&token| token == unk).count();
let token_count = tokens.len();
let fertility_tokens_per_character = if characters == 0 {
0.0
} else {
token_count as f64 / characters as f64
};
let bytes_per_token = if token_count == 0 {
0.0
} else {
bytes.len() as f64 / token_count as f64
};
let (round_trip_failures, round_trip_checked_segments) = round_trip_check(tokenizer, bytes);
TokenizerCorpusReport {
path: path.display().to_string(),
bytes: bytes.len(),
characters,
tokens: token_count,
fertility_tokens_per_character,
bytes_per_token,
unknown_tokens,
round_trip_failures,
round_trip_checked_segments,
}
}
fn round_trip_check(tokenizer: &BpeTokenizer, bytes: &[u8]) -> (usize, usize) {
if bytes.is_empty() {
return (0, 0);
}
let mut failures = 0usize;
let mut segments = 0usize;
for segment in bytes.split(|&b| b == b'\n') {
if segment.is_empty() {
continue;
}
segments += 1;
let encoded = tokenizer.encode_bytes(segment);
let decoded = tokenizer.decode_bytes(&encoded);
if decoded.as_slice() != segment {
failures += 1;
}
}
if segments == 0 {
segments = 1;
let encoded = tokenizer.encode_bytes(bytes);
let decoded = tokenizer.decode_bytes(&encoded);
if decoded.as_slice() != bytes {
failures = 1;
}
}
(failures, segments)
}
fn merge_depth(tokenizer: &BpeTokenizer) -> usize {
if tokenizer.merges.is_empty() {
return 0;
}
let mut depths = vec![0usize; tokenizer.vocab.len()];
let mut ordered: Vec<(usize, usize, usize, usize)> = tokenizer
.merge_ranks
.iter()
.filter_map(|(&(left, right), &rank)| {
let token_id = *tokenizer.merges.get(&(left, right))?;
Some((rank, left, right, token_id))
})
.collect();
ordered.sort_unstable_by_key(|(rank, ..)| *rank);
for (_, left, right, token_id) in ordered {
if token_id >= depths.len() {
depths.resize(token_id + 1, 0);
}
let left_depth = depths.get(left).copied().unwrap_or(0);
let right_depth = depths.get(right).copied().unwrap_or(0);
depths[token_id] = 1 + left_depth.max(right_depth);
}
depths.into_iter().max().unwrap_or(0)
}
fn try_load_sidecar_tokenizer(
path: &Path,
corpus: Option<&Path>,
) -> Result<Option<TokenizerReport>, AnalyzeError> {
match load_sidecar_tokenizer_instance(path) {
Some(tokenizer) => Ok(Some(build_tokenizer_report(&tokenizer, corpus)?)),
None => Ok(None),
}
}
fn load_sidecar_tokenizer_instance(path: &Path) -> Option<BpeTokenizer> {
let candidates = [
path.join("tokenizer.json"),
path.with_file_name("tokenizer.json"),
path.parent()
.unwrap_or_else(|| Path::new("."))
.join("tokenizer.json"),
];
for candidate in candidates {
if !candidate.is_file() {
continue;
}
if let Ok(tokenizer) = BpeTokenizer::load(&candidate) {
return Some(tokenizer);
}
if let Ok(tokenizer) = BpeTokenizer::import_huggingface(&candidate) {
return Some(tokenizer);
}
}
None
}
fn materialize_model(
config: &CausalLmConfig,
state: &StateDict,
weight_format: WeightFormat,
) -> Result<CausalLM, AnalyzeError> {
let model = CausalLM::new(config.clone()).map_err(|error| {
AnalyzeError::Invalid(format!("failed to construct model for quality gates: {error}"))
})?;
match weight_format {
WeightFormat::OxideLm => model.load_state_dict(state)?,
WeightFormat::HuggingFace => {
let native = from_hf_state_dict(config, state)?;
model.load_state_dict(&native)?;
}
}
Ok(model)
}
fn run_quality_gates(
model: &CausalLM,
tokenizer: Option<&BpeTokenizer>,
trainer_config: Option<&TrainerConfig>,
options: &AnalyzeOptions,
) -> Result<QualityGateReport, AnalyzeError> {
let forward_seq = FORWARD_VALIDATE_SEQ
.min(model.config.max_sequence_length)
.max(2);
let forward = match validate_forward(
model,
FORWARD_VALIDATE_SEED,
FORWARD_VALIDATE_BATCH,
forward_seq,
) {
Ok(result) => forward_from_result(result),
Err(error) => ForwardValidationReport {
seed: FORWARD_VALIDATE_SEED,
batch_size: FORWARD_VALIDATE_BATCH,
sequence_length: forward_seq,
logit_nan_count: 0,
logit_inf_count: 0,
loss: f32::NAN,
loss_finite: false,
passed: false,
error: Some(error.to_string()),
},
};
let eval = if let Some(corpus_path) = options.eval_corpus.as_ref() {
Some(run_eval_quality(
model,
tokenizer,
trainer_config,
corpus_path,
options,
)?)
} else {
None
};
Ok(QualityGateReport { forward, eval })
}
fn run_eval_quality(
model: &CausalLM,
tokenizer: Option<&BpeTokenizer>,
trainer_config: Option<&TrainerConfig>,
corpus_path: &Path,
options: &AnalyzeOptions,
) -> Result<EvalQualityReport, AnalyzeError> {
let path_string = corpus_path.display().to_string();
let Some(tokenizer) = tokenizer else {
return Ok(EvalQualityReport {
corpus_path: path_string,
loss: f32::NAN,
perplexity: f32::NAN,
tokens_processed: 0,
sequences: 0,
sequence_length: 0,
next_token_accuracy: 0.0,
kl: None,
error: Some("tokenizer required for --eval-corpus".into()),
});
};
let text = std::fs::read_to_string(corpus_path)?;
let default_seq = trainer_config
.map(|config| config.sequence_length)
.unwrap_or(32)
.min(model.config.max_sequence_length)
.max(2);
let sequence_length = options
.eval_sequence_length
.unwrap_or(default_seq)
.min(model.config.max_sequence_length)
.max(2);
let batch_size = options.eval_batch_size.max(1);
let (ppl, next) = match evaluate_held_out_text(
model,
tokenizer,
&text,
sequence_length,
batch_size,
None,
) {
Ok(value) => value,
Err(error) => {
return Ok(EvalQualityReport {
corpus_path: path_string,
loss: f32::NAN,
perplexity: f32::NAN,
tokens_processed: 0,
sequences: 0,
sequence_length,
next_token_accuracy: 0.0,
kl: None,
error: Some(error.to_string()),
});
}
};
let kl = if let Some(reference_path) = options.reference.as_ref() {
match load_reference_model(reference_path, options.format) {
Ok(reference) => {
match evaluate_kl_held_out_text(
model,
&reference,
tokenizer,
&text,
sequence_length,
batch_size,
) {
Ok(report) => Some(KlQualityReport {
reference_path: reference_path.display().to_string(),
mean_kl: report.mean_kl,
tokens_compared: report.tokens_compared,
}),
Err(error) => {
return Ok(EvalQualityReport {
corpus_path: path_string,
loss: ppl.loss,
perplexity: ppl.perplexity,
tokens_processed: ppl.tokens_processed,
sequences: ppl.sequences,
sequence_length: ppl.sequence_length,
next_token_accuracy: next.accuracy,
kl: None,
error: Some(format!("KL vs reference failed: {error}")),
});
}
}
}
Err(error) => {
return Ok(EvalQualityReport {
corpus_path: path_string,
loss: ppl.loss,
perplexity: ppl.perplexity,
tokens_processed: ppl.tokens_processed,
sequences: ppl.sequences,
sequence_length: ppl.sequence_length,
next_token_accuracy: next.accuracy,
kl: None,
error: Some(format!("failed to load --reference: {error}")),
});
}
}
} else {
None
};
Ok(EvalQualityReport {
corpus_path: path_string,
loss: ppl.loss,
perplexity: ppl.perplexity,
tokens_processed: ppl.tokens_processed,
sequences: ppl.sequences,
sequence_length: ppl.sequence_length,
next_token_accuracy: next.accuracy,
kl,
error: None,
})
}
fn load_reference_model(
path: &Path,
format: AnalyzeFormat,
) -> Result<CausalLM, AnalyzeError> {
let resolved = resolve_format(path, format)?;
match resolved {
AnalyzeFormat::Oxide => Ok(load_checkpoint(path)?.model),
AnalyzeFormat::Safetensors => {
let state = load_safetensors(path)?;
let config = resolve_config_for_weights(path, &state, WeightFormat::OxideLm)?;
materialize_model(&config, &state, WeightFormat::OxideLm)
}
AnalyzeFormat::Hf => {
let (weights_path, config_path) = if path.is_dir() {
(find_safetensors_in_dir(path)?, path.join("config.json"))
} else {
let parent = path.parent().unwrap_or_else(|| Path::new("."));
(path.to_path_buf(), parent.join("config.json"))
};
let hf = load_hf_config(&config_path)?;
let config = hf.to_causal_lm(0)?;
let state = load_safetensors(&weights_path)?;
materialize_model(&config, &state, WeightFormat::HuggingFace)
}
AnalyzeFormat::Auto => unreachable!("resolve_format never returns Auto"),
}
}
fn forward_from_result(result: ForwardValidationResult) -> ForwardValidationReport {
ForwardValidationReport {
seed: result.seed,
batch_size: result.batch_size,
sequence_length: result.sequence_length,
logit_nan_count: result.logit_nan_count,
logit_inf_count: result.logit_inf_count,
loss: result.loss,
loss_finite: result.loss_finite,
passed: result.passed,
error: None,
}
}
fn forward_failed_report(error: String) -> QualityGateReport {
QualityGateReport {
forward: ForwardValidationReport {
seed: FORWARD_VALIDATE_SEED,
batch_size: FORWARD_VALIDATE_BATCH,
sequence_length: FORWARD_VALIDATE_SEQ,
logit_nan_count: 0,
logit_inf_count: 0,
loss: f32::NAN,
loss_finite: false,
passed: false,
error: Some(error),
},
eval: None,
}
}
fn probe_device() -> DeviceProbeReport {
reset_cpu_peak_memory();
let cpu = cpu_memory_stats();
let cpu_parallelism = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1);
let rayon_threads = rayon::current_num_threads();
#[cfg(feature = "gpu")]
let (wgpu, wgpu_error) = match crate::gpu::GpuContext::new() {
Ok(gpu) => {
let limits = gpu.adapter_limits();
let mem = gpu.memory_stats().ok();
(
Some(WgpuProbeReport {
adapter_name: gpu.adapter_name().to_string(),
backend: format!("{:?}", gpu.adapter_backend()),
max_buffer_bytes: gpu.max_buffer_bytes(),
max_buffer_size: limits.max_buffer_size,
max_storage_buffer_binding_size: limits.max_storage_buffer_binding_size,
max_compute_workgroup_storage_size: limits.max_compute_workgroup_storage_size,
max_compute_invocations_per_workgroup: limits
.max_compute_invocations_per_workgroup,
max_compute_workgroup_size_x: limits.max_compute_workgroup_size_x,
max_bind_groups: limits.max_bind_groups,
fp16: gpu.float_format_support().fp16,
live_tensor_bytes: mem.map(|m| m.live_tensor_bytes).unwrap_or(0),
peak_tensor_bytes: mem.map(|m| m.peak_tensor_bytes).unwrap_or(0),
}),
None,
)
}
Err(error) => (None, Some(error.to_string())),
};
#[cfg(not(feature = "gpu"))]
let (wgpu, wgpu_error): (Option<WgpuProbeReport>, Option<String>) = (
None,
Some("GPU support is not compiled in; rebuild with `--features gpu`".into()),
);
let cpu_after = cpu_memory_stats();
DeviceProbeReport {
cpu_parallelism,
rayon_threads,
cpu_live_alloc_bytes: cpu_after.live_tensor_bytes.max(cpu.live_tensor_bytes),
cpu_peak_alloc_bytes: cpu_after.peak_tensor_bytes.max(cpu.peak_tensor_bytes),
wgpu,
wgpu_error,
}
}
fn config_delta(
left: &ArchitectureReport,
right: &ArchitectureReport,
) -> BTreeMap<String, ValueDelta> {
let left_value = serde_json::to_value(left).unwrap_or(serde_json::Value::Null);
let right_value = serde_json::to_value(right).unwrap_or(serde_json::Value::Null);
let mut delta = BTreeMap::new();
let (Some(left_obj), Some(right_obj)) = (left_value.as_object(), right_value.as_object())
else {
return delta;
};
for (key, left_field) in left_obj {
let right_field = right_obj
.get(key)
.cloned()
.unwrap_or(serde_json::Value::Null);
if left_field != &right_field {
delta.insert(
key.clone(),
ValueDelta {
left: left_field.clone(),
right: right_field,
},
);
}
}
for (key, right_field) in right_obj {
if !left_obj.contains_key(key) {
delta.insert(
key.clone(),
ValueDelta {
left: serde_json::Value::Null,
right: right_field.clone(),
},
);
}
}
delta
}
fn load_tokenizer_for_diff(path: &Path) -> Option<BpeTokenizer> {
if let Ok(checkpoint) = load_checkpoint(path) {
return Some(checkpoint.tokenizer);
}
let candidates = [
path.join("tokenizer.json"),
path.with_file_name("tokenizer.json"),
path.parent()
.unwrap_or_else(|| Path::new("."))
.join("tokenizer.json"),
];
for candidate in candidates {
if !candidate.is_file() {
continue;
}
if let Ok(tokenizer) = BpeTokenizer::load(&candidate) {
return Some(tokenizer);
}
if let Ok(tokenizer) = BpeTokenizer::import_huggingface(&candidate) {
return Some(tokenizer);
}
}
None
}
fn tokenizer_drift(left: &BpeTokenizer, right: &BpeTokenizer) -> TokenizerDriftReport {
let specials_match = left.special_tokens == right.special_tokens;
let prefix = left
.vocab
.iter()
.zip(right.vocab.iter())
.take_while(|(a, b)| a == b)
.count();
let vocab_bytes_equal = left.vocab == right.vocab;
let left_merges: std::collections::HashSet<_> = left.merges.keys().copied().collect();
let right_merges: std::collections::HashSet<_> = right.merges.keys().copied().collect();
let intersection = left_merges.intersection(&right_merges).count();
let union = left_merges.union(&right_merges).count();
let merge_set_jaccard = if union == 0 {
1.0
} else {
intersection as f64 / union as f64
};
TokenizerDriftReport {
left_vocab_size: left.vocab_size(),
right_vocab_size: right.vocab_size(),
left_merge_count: left.merges.len(),
right_merge_count: right.merges.len(),
left_merge_depth: merge_depth(left),
right_merge_depth: merge_depth(right),
specials_match,
vocab_prefix_equal: prefix,
vocab_bytes_equal,
merge_set_jaccard,
}
}
fn load_weight_map(
path: &Path,
format: AnalyzeFormat,
) -> Result<BTreeMap<String, (Vec<usize>, Vec<f32>)>, AnalyzeError> {
let resolved = resolve_format(path, format)?;
match resolved {
AnalyzeFormat::Oxide => {
let checkpoint = load_checkpoint(path)?;
Ok(checkpoint
.model
.named_parameters()
.into_iter()
.map(|(name, tensor)| (name, (tensor.shape(), tensor.data())))
.collect())
}
AnalyzeFormat::Safetensors | AnalyzeFormat::Hf => {
let weights_path = if path.is_dir() {
find_safetensors_in_dir(path)?
} else {
path.to_path_buf()
};
let state = load_safetensors(&weights_path)?;
Ok(state
.into_iter()
.map(|(name, storage)| (name, (storage.shape, storage.data)))
.collect())
}
AnalyzeFormat::Auto => unreachable!("resolve_format never returns Auto"),
}
}
fn param_l2_maps(
left: &BTreeMap<String, (Vec<usize>, Vec<f32>)>,
right: &BTreeMap<String, (Vec<usize>, Vec<f32>)>,
) -> ParamL2Report {
let mut left_only = Vec::new();
let mut right_only = Vec::new();
let mut shape_mismatches = Vec::new();
let mut per_tensor = Vec::new();
let mut total_l2 = 0.0;
for (name, (shape, data)) in left {
match right.get(name) {
None => left_only.push(name.clone()),
Some((right_shape, right_data)) if right_shape != shape => {
shape_mismatches.push(name.clone());
}
Some((_, right_data)) => {
let n = data.len().min(right_data.len());
let mut sum = 0.0f64;
for index in 0..n {
let diff = f64::from(data[index]) - f64::from(right_data[index]);
sum += diff * diff;
}
let l2 = sum.sqrt();
total_l2 += sum;
per_tensor.push(TensorL2Entry {
name: name.clone(),
l2,
numel: n,
});
}
}
}
for name in right.keys() {
if !left.contains_key(name) {
right_only.push(name.clone());
}
}
per_tensor.sort_by(|a, b| {
b.l2
.partial_cmp(&a.l2)
.unwrap_or(std::cmp::Ordering::Equal)
});
let matched = per_tensor.len();
let mean_l2 = if matched == 0 {
0.0
} else {
per_tensor.iter().map(|e| e.l2).sum::<f64>() / matched as f64
};
ParamL2Report {
matched_tensors: matched,
left_only,
right_only,
shape_mismatches,
total_l2: total_l2.sqrt(),
mean_l2,
per_tensor,
}
}
fn sha256_file(path: &Path) -> Result<String, AnalyzeError> {
let mut file = File::open(path)?;
let mut hasher = Sha256::new();
let mut buffer = [0u8; 1024 * 64];
loop {
let read = file.read(&mut buffer)?;
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
}
Ok(hasher.finalize_hex())
}
struct Sha256 {
state: [u32; 8],
total_len: u64,
buffer: [u8; 64],
buffer_len: usize,
}
impl Sha256 {
fn new() -> Self {
Self {
state: [
0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab,
0x5be0cd19,
],
total_len: 0,
buffer: [0; 64],
buffer_len: 0,
}
}
fn update(&mut self, data: &[u8]) {
self.total_len = self.total_len.saturating_add(data.len() as u64);
self.absorb(data);
}
fn absorb(&mut self, data: &[u8]) {
let mut offset = 0;
if self.buffer_len > 0 {
let take = (64 - self.buffer_len).min(data.len());
self.buffer[self.buffer_len..self.buffer_len + take].copy_from_slice(&data[..take]);
self.buffer_len += take;
offset = take;
if self.buffer_len == 64 {
let block = self.buffer;
self.compress(&block);
self.buffer_len = 0;
}
}
while offset + 64 <= data.len() {
self.compress(&data[offset..offset + 64]);
offset += 64;
}
if offset < data.len() {
let rest = &data[offset..];
self.buffer[..rest.len()].copy_from_slice(rest);
self.buffer_len = rest.len();
}
}
fn finalize_hex(mut self) -> String {
let bit_len = self.total_len.saturating_mul(8);
self.absorb(&[0x80]);
while self.buffer_len != 56 {
self.absorb(&[0x00]);
}
self.absorb(&bit_len.to_be_bytes());
let mut out = String::with_capacity(64);
for word in self.state {
use std::fmt::Write as _;
let _ = write!(out, "{word:08x}");
}
out
}
fn compress(&mut self, block: &[u8]) {
const K: [u32; 64] = [
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4,
0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe,
0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f,
0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc,
0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116,
0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
0xc67178f2,
];
let mut w = [0u32; 64];
for (index, chunk) in block.chunks_exact(4).enumerate().take(16) {
w[index] = u32::from_be_bytes(chunk.try_into().unwrap());
}
for index in 16..64 {
let s0 = w[index - 15].rotate_right(7)
^ w[index - 15].rotate_right(18)
^ (w[index - 15] >> 3);
let s1 = w[index - 2].rotate_right(17)
^ w[index - 2].rotate_right(19)
^ (w[index - 2] >> 10);
w[index] = w[index - 16]
.wrapping_add(s0)
.wrapping_add(w[index - 7])
.wrapping_add(s1);
}
let mut a = self.state[0];
let mut b = self.state[1];
let mut c = self.state[2];
let mut d = self.state[3];
let mut e = self.state[4];
let mut f = self.state[5];
let mut g = self.state[6];
let mut h = self.state[7];
for index in 0..64 {
let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25);
let ch = (e & f) ^ ((!e) & g);
let temp1 = h
.wrapping_add(s1)
.wrapping_add(ch)
.wrapping_add(K[index])
.wrapping_add(w[index]);
let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22);
let maj = (a & b) ^ (a & c) ^ (b & c);
let temp2 = s0.wrapping_add(maj);
h = g;
g = f;
f = e;
e = d.wrapping_add(temp1);
d = c;
c = b;
b = a;
a = temp1.wrapping_add(temp2);
}
self.state[0] = self.state[0].wrapping_add(a);
self.state[1] = self.state[1].wrapping_add(b);
self.state[2] = self.state[2].wrapping_add(c);
self.state[3] = self.state[3].wrapping_add(d);
self.state[4] = self.state[4].wrapping_add(e);
self.state[5] = self.state[5].wrapping_add(f);
self.state[6] = self.state[6].wrapping_add(g);
self.state[7] = self.state[7].wrapping_add(h);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::optimizer::{AdamW, AdamWConfig};
use crate::tokenizer::{BpeTokenizer, FIRST_MERGE_ID};
use crate::trainer::{save_checkpoint, LearningRateSchedule, TrainerConfig};
fn tiny_config() -> CausalLmConfig {
CausalLmConfig {
vocab_size: 16,
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 8,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: 15,
seed: 11,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
}
}
fn temp_dir(label: &str) -> PathBuf {
let path = std::env::temp_dir().join(format!(
"oxidelm-analyze-{label}-{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&path).unwrap();
path
}
#[test]
fn sha256_matches_known_vector() {
let mut hasher = Sha256::new();
hasher.update(b"abc");
assert_eq!(
hasher.finalize_hex(),
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
);
}
#[test]
fn analyze_oxide_checkpoint_json_round_trip() {
let directory = temp_dir("oxide");
let path = directory.join("model.bin");
let tokenizer = BpeTokenizer::train_from_bytes_with_min_frequency(
b"hello world hello rust",
FIRST_MERGE_ID + 8,
1,
)
.unwrap();
let mut config = tiny_config();
config.vocab_size = tokenizer.vocab_size();
config.eos_token_id = tokenizer.special_tokens.eos;
let model = CausalLM::new(config).unwrap();
let optimizer = AdamW::new(&model.parameters(), AdamWConfig::default()).unwrap();
let schedule = LearningRateSchedule {
base_learning_rate: 3e-4,
minimum_learning_rate: 3e-5,
warmup_steps: 1,
total_steps: 10,
};
let trainer_config = TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
save_checkpoint(
&path,
&model,
&optimizer,
&tokenizer,
&schedule,
&trainer_config,
3,
1,
12,
)
.unwrap();
let report = analyze_path(&path, &AnalyzeOptions::default()).unwrap();
assert_eq!(report.schema_version, ANALYZE_SCHEMA_VERSION);
assert_eq!(report.format, "oxide");
assert!(report.parameters.total > 0);
assert_eq!(report.identity.sha256.len(), 64);
assert!(report.training_memory.is_some());
assert!(report.tokenizer.is_some());
assert_eq!(report.inference_memory.len(), INFERENCE_PRESETS.len());
assert!(report.health.nan_count == 0);
assert!(report.modules.embed.parameters > 0);
let quality = report.quality.as_ref().unwrap();
assert!(quality.forward.passed);
assert_eq!(quality.forward.seed, FORWARD_VALIDATE_SEED);
assert!(quality.forward.loss.is_finite());
let json_path = directory.join("report.json");
write_analyze_json(&report, &json_path).unwrap();
let loaded: AnalyzeReport =
serde_json::from_reader(File::open(&json_path).unwrap()).unwrap();
assert_eq!(loaded.parameters.total, report.parameters.total);
assert_eq!(loaded.architecture.hidden_size, report.architecture.hidden_size);
assert!(loaded.quality.as_ref().unwrap().forward.passed);
let text = format_analyze_report(&report);
assert!(text.contains("analyze schema_version="));
assert!(text.contains("parameters total="));
assert!(text.contains("merge_depth="));
assert!(text.contains("quality_forward passed=true"));
let tok = report.tokenizer.as_ref().unwrap();
assert!(tok.merge_depth > 0 || tok.merge_count == 0);
assert_eq!(tok.round_trip_failures, 0);
let corpus_path = directory.join("corpus.txt");
std::fs::write(&corpus_path, b"hello world\nhello rust\nhello world hello rust\n").unwrap();
let with_corpus = analyze_path(
&path,
&AnalyzeOptions {
corpus: Some(corpus_path.clone()),
eval_corpus: Some(corpus_path.clone()),
reference: Some(path.clone()),
probe_device: true,
eval_sequence_length: Some(4),
..AnalyzeOptions::default()
},
)
.unwrap();
let corpus = with_corpus.tokenizer.as_ref().unwrap().corpus.as_ref().unwrap();
assert!(corpus.tokens > 0);
assert!(corpus.fertility_tokens_per_character > 0.0);
assert_eq!(corpus.round_trip_failures, 0);
assert!(with_corpus.device.is_some());
assert!(with_corpus.device.as_ref().unwrap().rayon_threads >= 1);
let eval = with_corpus.quality.as_ref().unwrap().eval.as_ref().unwrap();
assert!(eval.error.is_none(), "{:?}", eval.error);
assert!(eval.perplexity.is_finite());
let kl = eval.kl.as_ref().unwrap();
assert!(kl.mean_kl.abs() < 1e-5);
let gated = analyze_path(
&path,
&AnalyzeOptions {
fail_on: vec![FailOn::Nan, FailOn::Inf, FailOn::Schema, FailOn::Forward],
..AnalyzeOptions::default()
},
);
assert!(gated.is_ok());
let path_b = directory.join("model_b.bin");
std::fs::copy(&path, &path_b).unwrap();
let diff = analyze_diff(&path, &path_b, &AnalyzeOptions::default()).unwrap();
assert!(diff.config_delta.is_empty());
let l2 = diff.param_l2.as_ref().unwrap();
assert!(l2.matched_tensors > 0);
assert!(l2.total_l2 < 1e-6);
assert!(diff.tokenizer_drift.as_ref().unwrap().vocab_bytes_equal);
let diff_text = format_analyze_diff_report(&diff);
assert!(diff_text.contains("analyze_diff schema_version="));
let _ = std::fs::remove_dir_all(directory);
}
#[test]
fn fail_on_nan_triggers() {
let mut report = AnalyzeReport {
schema_version: ANALYZE_SCHEMA_VERSION,
path: "x".into(),
format: "oxide".into(),
identity: IdentityReport {
file_bytes: 0,
sha256: "0".repeat(64),
tensor_count: 0,
dtype_histogram: BTreeMap::new(),
packing: None,
},
architecture: ArchitectureReport {
vocab_size: 1,
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
head_dim: 4,
intermediate_size: 16,
max_sequence_length: 8,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
activation: "silu".into(),
norm: "rms_norm".into(),
tie_word_embeddings: true,
use_bias: false,
eos_token_id: 0,
},
parameters: ParameterInventory {
total: 1,
trainable: 1,
frozen: 0,
lora: 0,
tied_embedding_savings: 0,
tensors: Vec::new(),
},
storage: StorageAccounting {
on_disk_bytes: 0,
fp32_bytes: 4,
fp16_bytes: 2,
bf16_bytes: 2,
current_bytes: 4,
},
inference_memory: Vec::new(),
training_memory: None,
health: HealthReport {
nan_count: 2,
inf_count: 0,
layer_norms: Vec::new(),
dead_layers: Vec::new(),
exploded_layers: Vec::new(),
dead_rms_threshold: DEFAULT_DEAD_RMS,
exploded_rms_threshold: DEFAULT_EXPLODED_RMS,
},
modules: ModuleBreakdown::default(),
quantization: Vec::new(),
peft: None,
kv_cache: KvCacheReport {
bytes_per_token_fp32: 0,
bytes_per_token_fp16: 0,
bytes_per_token_int8: None,
bytes_per_token_fp8: None,
bytes_per_token_nf4: None,
presets: Vec::new(),
},
tokenizer: None,
device: None,
training: None,
quality: None,
warnings: Vec::new(),
};
let err = apply_fail_gates(
&mut report,
&AnalyzeOptions {
fail_on: vec![FailOn::Nan],
..AnalyzeOptions::default()
},
)
.unwrap_err();
assert!(err.to_string().contains("nan_count=2"));
}
#[test]
fn tensor_stats_detects_nan_inf_and_sparsity() {
let stats = tensor_stats(
"layers.0.attention.q_proj.weight",
&[2, 2],
"F32",
&[0.0, f32::NAN, f32::INFINITY, 1.0],
true,
);
assert_eq!(stats.nan_count, 1);
assert_eq!(stats.inf_count, 1);
assert!(stats.sparsity > 0.0);
assert_eq!(stats.module, "attention");
}
}