use burn::{
Tensor,
tensor::{
DType,
FloatDType,
},
};
use super::widen_to;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
#[serde(transparent)]
pub struct Widening {
pub at_least: Option<FloatDType>,
}
impl Widening {
pub const F32: Self = Self::to(FloatDType::F32);
pub const NONE: Self = Self { at_least: None };
pub const fn to(at_least: FloatDType) -> Self {
Self {
at_least: Some(at_least),
}
}
pub fn compute(
&self,
dtype: DType,
) -> DType {
match self.at_least {
Some(at_least) => widen_to(dtype, at_least),
None => dtype,
}
}
pub fn scope(
&self,
dtype: DType,
) -> WideningScope {
WideningScope {
dtype,
compute: self.compute(dtype),
}
}
pub fn around<const R: usize>(
&self,
dtype: DType,
f: impl FnOnce(&WideningScope) -> Tensor<R>,
) -> Tensor<R> {
let scope = self.scope(dtype);
scope.narrow(f(&scope))
}
}
impl From<Option<FloatDType>> for Widening {
fn from(at_least: Option<FloatDType>) -> Self {
Self { at_least }
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct WideningScope {
dtype: DType,
compute: DType,
}
impl WideningScope {
pub fn dtype(&self) -> DType {
self.dtype
}
pub fn compute(&self) -> DType {
self.compute
}
pub fn widen<const R: usize>(
&self,
x: Tensor<R>,
) -> Tensor<R> {
cast_to(x, self.compute)
}
pub fn narrow<const R: usize>(
&self,
x: Tensor<R>,
) -> Tensor<R> {
cast_to(x, self.dtype)
}
}
fn cast_to<const R: usize>(
x: Tensor<R>,
dtype: DType,
) -> Tensor<R> {
if x.dtype() == dtype { x } else { x.cast(dtype) }
}
#[cfg(test)]
mod tests {
use burn::tensor::Distribution;
use super::*;
use crate::support::testing::{
cpu_device,
seeded_tensor,
};
#[test]
fn test_compute() {
assert_eq!(Widening::F32.compute(DType::BF16), DType::F32);
assert_eq!(Widening::F32.compute(DType::F16), DType::F32);
assert_eq!(Widening::F32.compute(DType::F32), DType::F32);
assert_eq!(Widening::F32.compute(DType::F64), DType::F64);
assert_eq!(Widening::NONE.compute(DType::BF16), DType::BF16);
assert_eq!(
Widening::to(FloatDType::F64).compute(DType::F32),
DType::F64
);
assert_eq!(Widening::default(), Widening::NONE);
assert_eq!(Widening::from(Some(FloatDType::F32)), Widening::F32);
}
#[test]
fn test_scope() {
let device = cpu_device();
let scope = Widening::F32.scope(DType::BF16);
assert_eq!((scope.dtype(), scope.compute()), (DType::BF16, DType::F32));
let x = seeded_tensor::<2>(1, [2, 3], Distribution::Default, &device).cast(DType::BF16);
let wide = scope.widen(x.clone());
assert_eq!(wide.dtype(), DType::F32);
assert_eq!(scope.widen(wide.clone()).dtype(), DType::F32);
let back = scope.narrow(wide);
assert_eq!(back.dtype(), DType::BF16);
back.into_data().assert_eq(&x.into_data(), true);
}
#[test]
fn test_around() {
let device = cpu_device();
let a = seeded_tensor::<2>(2, [3, 4], Distribution::Default, &device).cast(DType::BF16);
let b = seeded_tensor::<2>(3, [3, 4], Distribution::Default, &device).cast(DType::BF16);
let got = Widening::F32.around(a.dtype(), |w| {
assert_eq!(w.compute(), DType::F32);
w.widen(a.clone()) * w.widen(b.clone()).exp()
});
let expected =
(a.clone().cast(DType::F32) * b.clone().cast(DType::F32).exp()).cast(DType::BF16);
assert_eq!(got.dtype(), DType::BF16);
got.into_data().assert_eq(&expected.into_data(), true);
let unwidened =
Widening::NONE.around(a.dtype(), |w| w.widen(a.clone()) * w.widen(b.clone()).exp());
unwidened
.into_data()
.assert_eq(&(a * b.exp()).into_data(), true);
}
#[test]
fn test_serde_form() {
assert_eq!(serde_json::to_string(&Widening::NONE).unwrap(), "null");
let f32 = serde_json::to_string(&Widening::F32).unwrap();
assert_eq!(f32, serde_json::to_string(&FloatDType::F32).unwrap());
assert_eq!(
serde_json::from_str::<Widening>(&f32).unwrap(),
Widening::F32
);
}
}