#[cfg(feature = "gpu")]
use crate::gpu::{GpuContext, GpuError};
use crate::initialization::kaiming_uniform_values;
use crate::layers::Linear;
use crate::model::CausalLM;
use crate::Tensor;
use serde::{Deserialize, Serialize};
use std::cell::Cell;
use std::error::Error;
use std::fmt;
pub const DORA_MAGNITUDE_EPS: f32 = 1e-6;
fn default_lora_dropout() -> f32 {
0.0
}
fn default_lora_targets() -> Vec<String> {
vec!["q_proj".into(), "v_proj".into()]
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct LoraConfig {
pub rank: usize,
pub alpha: f32,
#[serde(default = "default_lora_dropout")]
pub dropout: f32,
#[serde(default)]
pub dora: bool,
#[serde(default = "default_lora_targets")]
pub targets: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub base_quant: Option<crate::weight_quant::WeightQuantization>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub base_group_size: Option<usize>,
}
impl LoraConfig {
pub fn validate(&self) -> Result<(), PeftError> {
if self.rank == 0 {
return Err(PeftError::InvalidConfig(
"LoRA rank must be at least 1".into(),
));
}
if !self.alpha.is_finite() || self.alpha <= 0.0 {
return Err(PeftError::InvalidConfig(
"LoRA alpha must be finite and positive".into(),
));
}
if !self.dropout.is_finite() || !(0.0..1.0).contains(&self.dropout) {
return Err(PeftError::InvalidConfig(
"LoRA dropout must be finite and in [0, 1)".into(),
));
}
if self.targets.is_empty() {
return Err(PeftError::InvalidConfig(
"LoRA targets must not be empty".into(),
));
}
if let Some(size) = self.base_group_size {
if ![32, 64, 128].contains(&size) {
return Err(PeftError::InvalidConfig(format!(
"QLoRA base_group_size must be 32, 64, or 128 (got {size})"
)));
}
}
Ok(())
}
pub fn scale(&self) -> f32 {
self.alpha / self.rank as f32
}
pub fn is_qlora(&self) -> bool {
self.base_quant.is_some()
}
}
#[derive(Debug)]
pub enum PeftError {
InvalidConfig(String),
ModuleNotFound(String),
}
impl fmt::Display for PeftError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidConfig(message) => {
write!(formatter, "invalid LoRA configuration: {message}")
}
Self::ModuleNotFound(message) => write!(formatter, "LoRA attachment error: {message}"),
}
}
}
impl Error for PeftError {}
#[derive(Clone)]
pub struct LoraAdapter {
pub a: Tensor,
pub b: Tensor,
pub rank: usize,
pub scale: f32,
pub dora_magnitude: Option<Tensor>,
dropout: f32,
in_features: usize,
out_features: usize,
dropout_seed: Cell<u64>,
}
impl LoraAdapter {
pub fn new(in_features: usize, out_features: usize, config: &LoraConfig, seed: u64) -> Self {
assert!(config.rank >= 1, "LoRA rank must be at least 1");
assert!(
in_features > 0 && out_features > 0,
"LoRA adapter dimensions must be positive"
);
let a_values = kaiming_uniform_values(in_features * config.rank, in_features, seed);
let a = Tensor::new(a_values, vec![in_features, config.rank], true);
let b = Tensor::zeros(vec![config.rank, out_features], true);
let dora_magnitude = if config.dora {
Some(Tensor::ones(vec![out_features], true))
} else {
None
};
Self {
a,
b,
rank: config.rank,
scale: config.scale(),
dora_magnitude,
dropout: config.dropout,
in_features,
out_features,
dropout_seed: Cell::new(seed),
}
}
fn next_seed(&self) -> u64 {
let seed = self.dropout_seed.get();
self.dropout_seed.set(seed.wrapping_add(1));
crate::derive_seed(seed, 0)
}
pub fn forward_delta(&self, input: &Tensor) -> Tensor {
let dropped = if self.dropout > 0.0 {
input.dropout(self.dropout, self.next_seed())
} else {
input.clone()
};
let input_shape = dropped.shape();
let rows = dropped.numel() / self.in_features;
let flattened = dropped.reshape(vec![rows, self.in_features]);
let hidden = flattened.matmul(&self.a);
let delta = hidden.matmul(&self.b);
let mut output_shape = input_shape;
*output_shape.last_mut().unwrap() = self.out_features;
delta.reshape(output_shape).mul(&Tensor::scalar(self.scale, false))
}
pub fn forward(&self, input: &Tensor) -> Tensor {
self.forward_delta(input)
}
#[cfg(feature = "gpu")]
pub fn forward_gpu(&self, input: &Tensor, gpu: &GpuContext) -> Result<Tensor, GpuError> {
let input_gpu = input.to_gpu(gpu)?;
let dropped = if self.dropout > 0.0 {
input_gpu.gpu_dropout(self.dropout, self.next_seed(), gpu)?
} else {
input_gpu
};
let input_shape = dropped.shape();
let rows = dropped.numel() / self.in_features;
let flattened = dropped.reshape(vec![rows, self.in_features]);
let a_gpu = self.a.to_gpu(gpu)?;
let b_gpu = self.b.to_gpu(gpu)?;
let hidden = flattened.gpu_matmul(&a_gpu, gpu)?;
let delta = hidden.gpu_matmul(&b_gpu, gpu)?;
let mut output_shape = input_shape;
*output_shape.last_mut().unwrap() = self.out_features;
delta.reshape(output_shape).gpu_scale(self.scale, gpu)
}
pub fn parameters(&self) -> Vec<Tensor> {
let mut parameters = vec![self.a.clone(), self.b.clone()];
if let Some(magnitude) = &self.dora_magnitude {
parameters.push(magnitude.clone());
}
parameters
}
pub fn named_parameters(&self, prefix: &str) -> Vec<(String, Tensor)> {
let mut parameters = vec![
(format!("{prefix}.a"), self.a.clone()),
(format!("{prefix}.b"), self.b.clone()),
];
if let Some(magnitude) = &self.dora_magnitude {
parameters.push((format!("{prefix}.magnitude"), magnitude.clone()));
}
parameters
}
}
pub fn target_matches(param_or_module_name: &str, targets: &[String]) -> bool {
targets
.iter()
.any(|target| target == "all" || param_or_module_name.contains(target.as_str()))
}
pub fn freeze_parameters(named: &[(String, Tensor)], patterns: &[String]) {
for (name, tensor) in named {
if matches_freeze_pattern(name, patterns) {
tensor.set_requires_grad(false);
}
}
}
pub fn matches_freeze_pattern(name: &str, patterns: &[String]) -> bool {
patterns
.iter()
.any(|pattern| !pattern.is_empty() && name.contains(pattern.as_str()))
}
pub fn attach_lora_to_model(
model: &mut CausalLM,
config: &LoraConfig,
) -> Result<Vec<(String, LoraAdapter)>, PeftError> {
config.validate()?;
let base_seed = model.config.seed;
let mut attached = Vec::new();
for (layer_index, layer) in model.layers.iter_mut().enumerate() {
let prefix = format!("layers.{layer_index}");
attach_if_targeted(
&mut layer.attention.q_proj,
format!("{prefix}.attention.q_proj"),
config,
base_seed,
&mut attached,
);
attach_if_targeted(
&mut layer.attention.k_proj,
format!("{prefix}.attention.k_proj"),
config,
base_seed,
&mut attached,
);
attach_if_targeted(
&mut layer.attention.v_proj,
format!("{prefix}.attention.v_proj"),
config,
base_seed,
&mut attached,
);
attach_if_targeted(
&mut layer.attention.o_proj,
format!("{prefix}.attention.o_proj"),
config,
base_seed,
&mut attached,
);
attach_if_targeted(
&mut layer.feed_forward.gate_proj,
format!("{prefix}.feed_forward.gate_proj"),
config,
base_seed,
&mut attached,
);
attach_if_targeted(
&mut layer.feed_forward.up_proj,
format!("{prefix}.feed_forward.up_proj"),
config,
base_seed,
&mut attached,
);
attach_if_targeted(
&mut layer.feed_forward.down_proj,
format!("{prefix}.feed_forward.down_proj"),
config,
base_seed,
&mut attached,
);
}
if attached.is_empty() {
return Err(PeftError::ModuleNotFound(format!(
"no linear modules matched LoRA targets {:?}",
config.targets
)));
}
Ok(attached)
}
fn attach_if_targeted(
linear: &mut Linear,
name: String,
config: &LoraConfig,
base_seed: u64,
attached: &mut Vec<(String, LoraAdapter)>,
) {
if !target_matches(&name, &config.targets) {
return;
}
let seed = crate::derive_seed(base_seed, fnv1a64(&name));
let adapter = LoraAdapter::new(linear.in_features, linear.out_features, config, seed);
linear.lora = Some(adapter.clone());
attached.push((name, adapter));
}
fn fnv1a64(value: &str) -> u64 {
const OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325;
const PRIME: u64 = 0x0000_0100_0000_01b3;
let mut hash = OFFSET_BASIS;
for byte in value.as_bytes() {
hash ^= *byte as u64;
hash = hash.wrapping_mul(PRIME);
}
hash
}
pub fn estimate_lora_parameter_count(
config: &crate::model::CausalLmConfig,
lora: &LoraConfig,
) -> usize {
let num_key_value_heads = if config.num_key_value_heads == 0 {
config.num_heads
} else {
config.num_key_value_heads
};
let head_dim = if config.num_heads == 0 {
0
} else {
config.hidden_size / config.num_heads
};
let key_value_size = num_key_value_heads * head_dim;
let projections = [
("q_proj", config.hidden_size, config.hidden_size),
("k_proj", config.hidden_size, key_value_size),
("v_proj", config.hidden_size, key_value_size),
("o_proj", config.hidden_size, config.hidden_size),
("gate_proj", config.hidden_size, config.intermediate_size),
("up_proj", config.hidden_size, config.intermediate_size),
("down_proj", config.intermediate_size, config.hidden_size),
];
let per_layer: usize = projections
.iter()
.filter(|(name, _, _)| target_matches(name, &lora.targets))
.map(|(_, in_features, out_features)| {
lora_adapter_parameter_count(*in_features, *out_features, lora)
})
.sum();
per_layer * config.num_layers
}
fn lora_adapter_parameter_count(in_features: usize, out_features: usize, config: &LoraConfig) -> usize {
let mut count = in_features * config.rank + config.rank * out_features;
if config.dora {
count += out_features;
}
count
}
pub fn collect_adapter_state(model: &CausalLM) -> crate::state_dict::StateDict {
let mut state = crate::state_dict::StateDict::new();
for (name, parameter) in model.named_parameters() {
if !name.contains(".lora.") {
continue;
}
state.insert(
name,
crate::state_dict::TensorStorage {
shape: parameter.shape(),
data: parameter.data(),
},
);
}
state
}
pub fn save_adapters(
path: impl AsRef<std::path::Path>,
model: &CausalLM,
config: &LoraConfig,
) -> Result<(), PeftError> {
let state = collect_adapter_state(model);
if state.is_empty() {
return Err(PeftError::ModuleNotFound(
"model has no LoRA adapter tensors to save".into(),
));
}
let mut metadata = std::collections::HashMap::new();
metadata.insert("format".into(), "oxidelm-adapters".into());
metadata.insert(
"lora_config".into(),
serde_json::to_string(config).map_err(|error| {
PeftError::InvalidConfig(format!("failed to serialize LoRA config: {error}"))
})?,
);
crate::state_dict::save_safetensors(path, &state, Some(metadata)).map_err(|error| {
PeftError::InvalidConfig(format!("failed to save adapters: {error}"))
})
}
pub fn load_adapters(
path: impl AsRef<std::path::Path>,
model: &CausalLM,
) -> Result<(), PeftError> {
let state = crate::state_dict::load_safetensors(path).map_err(|error| {
PeftError::InvalidConfig(format!("failed to load adapters: {error}"))
})?;
let mut loaded = 0usize;
for (name, parameter) in model.named_parameters() {
if !name.contains(".lora.") {
continue;
}
let Some(storage) = state.get(&name) else {
return Err(PeftError::ModuleNotFound(format!(
"adapter file is missing tensor {name}"
)));
};
if storage.shape != parameter.shape() {
return Err(PeftError::InvalidConfig(format!(
"adapter tensor {name} has shape {:?} but model expects {:?}",
storage.shape,
parameter.shape()
)));
}
if storage.data.len() != parameter.numel() {
return Err(PeftError::InvalidConfig(format!(
"adapter tensor {name} has the wrong length"
)));
}
parameter.set_data(storage.data.clone());
loaded += 1;
}
if loaded == 0 {
return Err(PeftError::ModuleNotFound(
"model has no LoRA modules to load adapters into".into(),
));
}
Ok(())
}
pub fn load_adapter_config(
path: impl AsRef<std::path::Path>,
) -> Result<Option<LoraConfig>, PeftError> {
use memmap2::MmapOptions;
use safetensors::SafeTensors;
use std::fs::File;
let file = File::open(path.as_ref()).map_err(|error| {
PeftError::InvalidConfig(format!("failed to open adapter file: {error}"))
})?;
let mmap = unsafe { MmapOptions::new().map(&file) }.map_err(|error| {
PeftError::InvalidConfig(format!("failed to map adapter file: {error}"))
})?;
let (_, file_metadata) = SafeTensors::read_metadata(&mmap).map_err(|error| {
PeftError::InvalidConfig(format!("failed to parse adapter file: {error}"))
})?;
let Some(metadata) = file_metadata.metadata().as_ref() else {
return Ok(None);
};
let Some(json) = metadata.get("lora_config") else {
return Ok(None);
};
let config: LoraConfig = serde_json::from_str(json).map_err(|error| {
PeftError::InvalidConfig(format!("invalid lora_config metadata: {error}"))
})?;
Ok(Some(config))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::model::{ArchitecturePreset, CausalLM};
fn tiny_lora_config() -> LoraConfig {
LoraConfig {
rank: 4,
alpha: 8.0,
dropout: 0.0,
dora: false,
targets: default_lora_targets(),
base_quant: None,
base_group_size: None,
}
}
#[test]
fn adapter_starts_near_zero() {
let config = tiny_lora_config();
let adapter = LoraAdapter::new(16, 32, &config, 0);
let input = Tensor::new(vec![1.0; 2 * 16], vec![2, 16], false);
let delta = adapter.forward(&input);
assert_eq!(delta.shape(), vec![2, 32]);
for value in delta.data() {
assert_eq!(value, 0.0, "delta should be exactly zero while B is zero");
}
}
#[test]
fn adapter_scale_matches_alpha_over_rank() {
let config = LoraConfig {
rank: 8,
alpha: 16.0,
..tiny_lora_config()
};
let adapter = LoraAdapter::new(4, 4, &config, 1);
assert!((adapter.scale - 2.0).abs() < 1e-6);
}
#[test]
fn target_matches_substring_and_all() {
let targets = vec!["q_proj".to_string(), "v_proj".to_string()];
assert!(target_matches("layers.0.attention.q_proj", &targets));
assert!(target_matches("layers.3.attention.v_proj", &targets));
assert!(!target_matches("layers.0.attention.k_proj", &targets));
let all_targets = vec!["all".to_string()];
assert!(target_matches("layers.0.feed_forward.down_proj", &all_targets));
}
#[test]
fn freeze_pattern_matches_prefix_and_full_name() {
let patterns = vec!["tok_embeddings".to_string(), "layers.0".to_string()];
assert!(matches_freeze_pattern("tok_embeddings.weight", &patterns));
assert!(matches_freeze_pattern(
"layers.0.attention.q_proj.weight",
&patterns
));
assert!(!matches_freeze_pattern(
"layers.1.attention.q_proj.weight",
&patterns
));
}
#[test]
fn freeze_parameters_sets_requires_grad_false_on_matches_only() {
let frozen = Tensor::zeros(vec![2], true);
let trainable = Tensor::zeros(vec![2], true);
let named = vec![
("layers.0.attention.q_proj.weight".to_string(), frozen.clone()),
("layers.0.attention.q_proj.lora.a".to_string(), trainable.clone()),
];
freeze_parameters(&named, &["layers.0.attention.q_proj.weight".to_string()]);
assert!(!frozen.requires_grad());
assert!(trainable.requires_grad());
}
#[test]
fn install_lora_attaches_adapters_and_freezes_base_weights() {
let mut model = CausalLM::new(
ArchitecturePreset::LlamaTiny.causal_lm_config(32, 2, 0),
)
.unwrap();
let config = tiny_lora_config();
model.install_lora(&config).unwrap();
assert!(model.layers[0].attention.q_proj.lora.is_some());
assert!(!model.layers[0].attention.q_proj.weight.requires_grad());
assert!(model.layers[0].attention.k_proj.lora.is_none());
let adapter_params: Vec<_> = model
.named_parameters()
.into_iter()
.filter(|(name, _)| name.contains(".lora."))
.collect();
assert!(!adapter_params.is_empty());
for (_, parameter) in &adapter_params {
assert!(parameter.requires_grad());
}
}
#[test]
fn estimate_lora_parameter_count_matches_manual_computation() {
let causal_config = ArchitecturePreset::LlamaTiny.causal_lm_config(32, 2, 0);
let lora = tiny_lora_config();
let estimate = estimate_lora_parameter_count(&causal_config, &lora);
let per_layer = (128 * 4 + 4 * 128) + (128 * 4 + 4 * 64);
assert_eq!(estimate, per_layer * causal_config.num_layers);
}
#[test]
fn adapter_save_load_round_trip() {
let mut model = CausalLM::new(
ArchitecturePreset::LlamaTiny.causal_lm_config(32, 2, 0),
)
.unwrap();
let config = tiny_lora_config();
model.install_lora(&config).unwrap();
let adapter = model.layers[0].attention.q_proj.lora.as_ref().unwrap();
adapter.a.set_data(vec![0.5; adapter.a.numel()]);
adapter.b.set_data(vec![0.25; adapter.b.numel()]);
let path = std::env::temp_dir().join(format!(
"oxidelm-adapters-{}-{}.safetensors",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
save_adapters(&path, &model, &config).unwrap();
let loaded_config = load_adapter_config(&path).unwrap().unwrap();
assert_eq!(loaded_config.rank, config.rank);
let mut other = CausalLM::new(
ArchitecturePreset::LlamaTiny.causal_lm_config(32, 2, 0),
)
.unwrap();
other.install_lora(&config).unwrap();
load_adapters(&path, &other).unwrap();
let restored = other.layers[0].attention.q_proj.lora.as_ref().unwrap();
assert_eq!(restored.a.data(), vec![0.5; restored.a.numel()]);
assert_eq!(restored.b.data(), vec![0.25; restored.b.numel()]);
let _ = std::fs::remove_file(&path);
}
}