use burn::{
nn::{
Linear,
attention::MultiHeadAttention,
},
prelude::Backend,
};
use crate::burner::store::repair_pytorch_strided_weight;
pub trait FixPytorchLoadMappers {
fn fix_pytorch_load_mappers(self) -> Self;
}
impl<T: FixPytorchLoadMappers> FixPytorchLoadMappers for Vec<T> {
fn fix_pytorch_load_mappers(self) -> Self {
self.into_iter().map(T::fix_pytorch_load_mappers).collect()
}
}
impl<B: Backend> FixPytorchLoadMappers for Linear<B> {
fn fix_pytorch_load_mappers(mut self) -> Self {
self.weight = repair_pytorch_strided_weight(self.weight);
self
}
}
impl<B: Backend> FixPytorchLoadMappers for MultiHeadAttention<B> {
fn fix_pytorch_load_mappers(mut self) -> Self {
self.query = self.query.fix_pytorch_load_mappers();
self.key = self.key.fix_pytorch_load_mappers();
self.value = self.value.fix_pytorch_load_mappers();
self.output = self.output.fix_pytorch_load_mappers();
self
}
}
#[cfg(test)]
mod tests {
use burn::{
Tensor,
module::Param,
nn::{
LinearConfig,
attention::MultiHeadAttentionConfig,
},
prelude::{
Device,
TensorData,
},
tensor::DType,
};
use super::*;
use crate::support::testing::{
CpuBackend,
default_device,
param_load_mapping,
};
type B = CpuBackend;
fn probe(device: &Device<B>) -> Tensor<B, 2> {
Tensor::from_data(
TensorData::new((0..6).map(|v| v as f64).collect::<Vec<_>>(), [2, 3]),
device,
)
}
fn load_mapped_probe(
param: &Param<Tensor<B, 2>>,
device: &Device<B>,
) -> Vec<f32> {
param_load_mapping(param, probe(device))
.cast(DType::F32)
.to_data()
.to_vec()
.unwrap()
}
const UNMAPPED: [f32; 6] = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0];
const REPAIRED: [f32; 6] = [0.0, 3.0, 1.0, 4.0, 2.0, 5.0];
#[test]
fn test_linear_weight_is_repaired() {
let device = default_device();
let linear = LinearConfig::new(3, 5).init::<B>(&device);
let before = linear.weight.val().to_data();
let linear = linear.fix_pytorch_load_mappers();
assert_eq!(load_mapped_probe(&linear.weight, &device), REPAIRED);
assert_eq!(linear.weight.val().to_data(), before);
}
#[test]
fn test_attention_repairs_every_projection() {
let device: Device<B> = Default::default();
let attn = MultiHeadAttentionConfig::new(4, 2).init::<B>(&device);
assert_eq!(load_mapped_probe(&attn.query.weight, &device), UNMAPPED);
let attn = attn.fix_pytorch_load_mappers();
for weight in [
&attn.query.weight,
&attn.key.weight,
&attn.value.weight,
&attn.output.weight,
] {
assert_eq!(load_mapped_probe(weight, &device), REPAIRED);
}
}
#[test]
fn test_vec_repairs_each_element() {
let device: Device<B> = Default::default();
let linears: Vec<Linear<B>> = (0..3)
.map(|_| LinearConfig::new(3, 5).init::<B>(&device))
.collect();
for linear in linears.fix_pytorch_load_mappers() {
assert_eq!(load_mapped_probe(&linear.weight, &device), REPAIRED);
}
}
}