use alloc::collections::BTreeSet;
use burn::{
store::{
FloatCastAdapter,
ModuleAdapter,
ModuleSnapshot,
PyTorchToBurnAdapter,
burn_pack::Tensor as PackTensor,
},
tensor::{
DType,
FloatDType,
},
};
use crate::{
burner::module::{
Rekey,
Target,
TensorLoadRowSplit,
},
errors::{
BunsenError,
BunsenResult,
},
};
#[derive(Clone, Debug, Default)]
pub struct NamedTensorsApplied {
pub applied: Vec<String>,
pub unused: Vec<String>,
}
pub fn apply_named_tensors<M: ModuleSnapshot>(
module: &mut M,
tensors: impl IntoIterator<Item = PackTensor>,
rename: impl Fn(&str) -> Option<String>,
) -> BunsenResult<NamedTensorsApplied> {
apply_named_tensors_as(module, tensors, rename, None)
}
pub fn apply_named_tensors_as<M: ModuleSnapshot>(
module: &mut M,
tensors: impl IntoIterator<Item = PackTensor>,
rename: impl Fn(&str) -> Option<String>,
float_dtype: Option<DType>,
) -> BunsenResult<NamedTensorsApplied> {
apply_in_batches(module, tensors, rename, float_dtype, UPLOAD_BATCH_BYTES)
}
fn apply_in_batches<M: ModuleSnapshot>(
module: &mut M,
tensors: impl IntoIterator<Item = PackTensor>,
rename: impl Fn(&str) -> Option<String>,
float_dtype: Option<DType>,
batch_bytes: usize,
) -> BunsenResult<NamedTensorsApplied> {
let mut renamed = Vec::new();
let mut dropped = Vec::new();
for mut tensor in tensors {
match rename(&tensor.name) {
Some(path) => {
tensor.name = path;
renamed.push(tensor);
}
None => dropped.push(tensor.name),
}
}
let adapter = || -> Box<dyn ModuleAdapter> {
match float_dtype {
Some(dtype) => Box::new(PyTorchToBurnAdapter.chain(FloatCastAdapter::to(dtype))),
None => Box::new(PyTorchToBurnAdapter),
}
};
let paths: Vec<String> = module
.collect(None, None, true)
.into_iter()
.map(|t| t.name)
.collect();
let devices = module.devices();
let mut applied = Vec::new();
let mut unmatched: Option<BTreeSet<String>> = None;
let mut errors = Vec::new();
for batch in upload_batches(renamed, batch_bytes) {
let result = module.apply(batch, None, Some(adapter()), true);
errors.extend(result.errors.iter().map(|e| format!("{e:?}")));
applied.extend(result.applied);
let unused: BTreeSet<String> = result.unused.into_iter().collect();
unmatched = Some(match unmatched {
None => unused,
Some(before) => before.union(&unused).cloned().collect(),
});
for device in &devices {
let _ = device.sync();
device.memory_cleanup();
}
}
if !errors.is_empty() {
return Err(BunsenError::invalid_resource(format!(
"{} tensors do not fit the model",
errors.len()
))
.with_details(errors.join("\n")));
}
let loaded: BTreeSet<&str> = applied.iter().map(String::as_str).collect();
let missing: Vec<&str> = paths
.iter()
.map(String::as_str)
.filter(|p| !loaded.contains(p))
.collect();
if !missing.is_empty() {
return Err(BunsenError::invalid_resource(format!(
"the tensors lack {} of the model's parameters: {}",
missing.len(),
missing
.iter()
.take(8)
.copied()
.collect::<Vec<_>>()
.join(", ")
))
.with_details(missing.join("\n")));
}
let mut unused: Vec<String> = unmatched.unwrap_or_default().into_iter().collect();
unused.extend(dropped);
Ok(NamedTensorsApplied { applied, unused })
}
pub const UPLOAD_BATCH_BYTES: usize = 512 << 20;
fn upload_batches(
tensors: Vec<PackTensor>,
cap: usize,
) -> Vec<Vec<PackTensor>> {
let mut batches = vec![Vec::new()];
let mut bytes = 0;
for tensor in tensors {
let size = tensor.shape.iter().product::<usize>() * tensor.dtype.size();
if bytes > 0 && bytes + size > cap {
batches.push(Vec::new());
bytes = 0;
}
bytes += size;
batches.last_mut().expect("a batch").push(tensor);
}
batches
}
impl Rekey {
pub fn apply_to<M: ModuleSnapshot>(
&self,
module: &mut M,
tensors: impl IntoIterator<Item = PackTensor>,
) -> BunsenResult<NamedTensorsApplied> {
self.apply_as(module, tensors, None)
}
}
pub fn stored_float_dtype(tensors: &[PackTensor]) -> Option<FloatDType> {
let mut counts: Vec<(DType, usize)> = Vec::new();
for tensor in tensors {
if !tensor.dtype.is_float() {
continue;
}
let n = tensor.shape.iter().product::<usize>();
match counts.iter_mut().find(|(d, _)| *d == tensor.dtype) {
Some((_, count)) => *count += n,
None => counts.push((tensor.dtype, n)),
}
}
counts
.into_iter()
.max_by_key(|(_, n)| *n)
.map(|(dtype, _)| FloatDType::from(dtype))
}
pub fn split_rows(
tensors: impl IntoIterator<Item = PackTensor>,
splits: &[TensorLoadRowSplit],
) -> BunsenResult<Vec<PackTensor>> {
let mut out = Vec::new();
for tensor in tensors {
let Some(split) = splits.iter().find(|s| tensor.name.ends_with(s.suffix())) else {
out.push(tensor);
continue;
};
let name = tensor.name.clone();
let data = burn::store::bridge::into_data(tensor)
.map_err(|e| BunsenError::invalid_resource(format!("cannot read {name}: {e:?}")))?;
let parts = split
.split(&name, data)
.expect("the split's suffix matched")
.map_err(BunsenError::invalid_resource)?;
out.extend(
parts
.into_iter()
.map(|(part, data)| burn::store::bridge::from_data(data, part, None)),
);
}
Ok(out)
}
impl Rekey {
pub fn load<M: ModuleSnapshot>(
&self,
module: &mut M,
tensors: impl IntoIterator<Item = PackTensor>,
dtype: Option<FloatDType>,
) -> BunsenResult<NamedTensorsApplied> {
let dtype = dtype.or_else(|| crate::burner::module::module_float_dtype(module));
self.apply_as(module, tensors, dtype.map(DType::from))
}
}
impl Rekey {
pub fn apply_as<M: ModuleSnapshot>(
&self,
module: &mut M,
tensors: impl IntoIterator<Item = PackTensor>,
float_dtype: Option<DType>,
) -> BunsenResult<NamedTensorsApplied> {
let tensors: Vec<PackTensor> = tensors.into_iter().collect();
let module_paths = module
.collect(None, None, true)
.into_iter()
.map(|t| t.name)
.collect();
let plan = self.plan_for(tensors.iter().map(|t| t.name.as_str()), &module_paths);
plan.check()?;
let mut placed = Vec::new();
let mut dropped = Vec::new();
for (mut tensor, (_, target)) in tensors.into_iter().zip(plan.targets) {
match target {
None => dropped.push(tensor.name),
Some(Target {
path,
transform: Some(f),
..
}) => {
let (name, param_id) = (tensor.name.clone(), tensor.param_id);
let data = burn::store::bridge::into_data(tensor).map_err(|e| {
BunsenError::invalid_resource(format!("cannot read {name}: {e:?}"))
})?;
placed.push(burn::store::bridge::from_data(f(data), path, param_id));
}
Some(Target { path, .. }) => {
tensor.name = path;
placed.push(tensor);
}
}
}
let mut applied =
apply_named_tensors_as(module, placed, |name| Some(name.to_string()), float_dtype)?;
applied.unused.extend(dropped);
Ok(applied)
}
}
#[cfg(test)]
mod tests {
use burn::{
module::Module,
nn::{
Linear,
LinearConfig,
},
prelude::*,
store::bridge,
};
use super::*;
use crate::{
errors::{
BunsenErrorKind,
testing::ErrorMatcher,
},
support::testing::cpu_device,
};
#[derive(Module, Debug)]
struct Two {
a: Linear,
b: Linear,
}
fn two(device: &Device) -> Two {
Two {
a: LinearConfig::new(3, 2).init(device),
b: LinearConfig::new(2, 4).init(device),
}
}
fn tensor(
name: &str,
rows: usize,
cols: usize,
value: f32,
) -> PackTensor {
let data = TensorData::new(vec![value; rows * cols], [rows, cols]);
bridge::from_data(data, name.to_string(), None)
}
fn bias(
name: &str,
n: usize,
value: f32,
) -> PackTensor {
bridge::from_data(TensorData::new(vec![value; n], [n]), name.to_string(), None)
}
#[test]
fn test_upload_batches() {
let sizes = |batches: Vec<Vec<PackTensor>>| -> Vec<usize> {
batches.iter().map(Vec::len).collect()
};
let tensors = || {
vec![
tensor("x", 2, 2, 0.0),
tensor("y", 2, 2, 0.0),
tensor("z", 8, 8, 0.0),
tensor("w", 1, 1, 0.0),
]
};
assert_eq!(sizes(upload_batches(tensors(), 32)), [2, 1, 1]);
assert_eq!(sizes(upload_batches(tensors(), 1)), [1, 1, 1, 1]);
assert_eq!(sizes(upload_batches(tensors(), 1 << 20)), [4]);
}
#[test]
fn test_batches_load_alike() {
let device = cpu_device();
let tensors = || {
vec![
tensor("a.weight", 2, 3, 1.0),
bias("a.bias", 2, 2.0),
tensor("b.weight", 4, 2, 3.0),
bias("b.bias", 4, 4.0),
bias("extra", 1, 5.0),
]
};
let mut whole = two(&device);
let one = apply_in_batches(
&mut whole,
tensors(),
|n| Some(n.to_string()),
None,
1 << 20,
)
.unwrap();
let mut batched = two(&device);
let many =
apply_in_batches(&mut batched, tensors(), |n| Some(n.to_string()), None, 1).unwrap();
assert_eq!(one.unused, ["extra"]);
assert_eq!(many.unused, ["extra"]);
let sorted = |mut v: Vec<String>| {
v.sort();
v
};
assert_eq!(sorted(one.applied), sorted(many.applied.clone()));
assert_eq!(many.applied.len(), 4);
batched
.b
.weight
.val()
.to_data()
.assert_eq(&whole.b.weight.val().to_data(), true);
let partial = tensors().into_iter().filter(|t| t.name != "b.bias");
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("lack 1")
.assert_err(&apply_in_batches(
&mut two(&device),
partial,
|n| Some(n.to_string()),
None,
1,
));
}
#[derive(Module, Debug)]
struct Qkv {
q: Linear,
k: Linear,
v: Linear,
}
#[test]
fn test_split_rows_loads_the_parts() {
let device = cpu_device();
let fused: Vec<f32> = (0..24).map(|v| v as f32).collect();
let tensors = vec![
bridge::from_data(
TensorData::new(fused.clone(), [6, 4]),
"attn.qkv.weight".to_string(),
None,
),
bridge::from_data(
TensorData::new(vec![1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0], [6]),
"attn.qkv.bias".to_string(),
None,
),
bias("attn.other", 2, 9.0),
];
let splits = [
TensorLoadRowSplit::equal(".qkv.weight", [".q.weight", ".k.weight", ".v.weight"]),
TensorLoadRowSplit::equal(".qkv.bias", [".q.bias", ".k.bias", ".v.bias"]),
];
let split = split_rows(tensors, &splits).unwrap();
let names: Vec<&str> = split.iter().map(|t| t.name.as_str()).collect();
assert_eq!(
names,
[
"attn.q.weight",
"attn.k.weight",
"attn.v.weight",
"attn.q.bias",
"attn.k.bias",
"attn.v.bias",
"attn.other",
]
);
let mut module = Qkv {
q: LinearConfig::new(4, 2).init(&device),
k: LinearConfig::new(4, 2).init(&device),
v: LinearConfig::new(4, 2).init(&device),
};
let applied = Rekey::keep_rest()
.rename("attn.", "")
.load(&mut module, split, None)
.unwrap();
assert_eq!(applied.unused, ["other"]);
let rows = Tensor::<2>::from_floats(TensorData::new(fused, [6, 4]), &device);
for (linear, (range, b)) in [&module.q, &module.k, &module.v].into_iter().zip([
(0..2, [1.0f32, 2.0]),
(2..4, [3.0, 4.0]),
(4..6, [5.0, 6.0]),
]) {
linear
.weight
.val()
.into_data()
.assert_eq(&rows.clone().slice(range).transpose().into_data(), true);
linear
.bias
.as_ref()
.unwrap()
.val()
.into_data()
.assert_eq(&TensorData::from(b), true);
}
}
#[test]
fn test_split_rows_refuses_a_bad_split() {
let splits = [TensorLoadRowSplit::equal(
".qkv.weight",
[".q.weight", ".k.weight", ".v.weight"],
)];
ErrorMatcher::kind(BunsenErrorKind::InvalidResource)
.message_contains("attn.qkv.weight: 4 rows")
.assert_err(&split_rows(
vec![tensor("attn.qkv.weight", 4, 2, 0.0)],
&splits,
));
}
}