use crate::{
convert::widen_finite_high_word,
types::{Bf16, Bf4, F32, F4},
};
pub trait Packable4: Copy + 'static {
type Unpacked: Copy + Default;
fn pack_pair(low: Self, high: Self) -> u8;
fn unpack_pair(packed: u8) -> (Self, Self);
fn unpack_slice_packed(packed: &[u8], unpacked: &mut [Self::Unpacked]);
fn unpack_single(element: Self) -> Self::Unpacked;
}
impl Packable4 for Bf4 {
type Unpacked = Bf16;
#[inline(always)]
fn pack_pair(low: Self, high: Self) -> u8 {
Bf4::pack_pair(low, high)
}
#[inline(always)]
fn unpack_pair(packed: u8) -> (Self, Self) {
Bf4::unpack_pair(packed)
}
#[inline(always)]
fn unpack_slice_packed(packed: &[u8], unpacked: &mut [Bf16]) {
super::unpack::unpack_bf4_to_bf16_packed(packed, unpacked);
}
#[inline(always)]
fn unpack_single(element: Self) -> Bf16 {
Bf16(widen_finite_high_word::<2, 1>(element.0 as u32))
}
}
impl Packable4 for F4 {
type Unpacked = F32;
#[inline(always)]
fn pack_pair(low: Self, high: Self) -> u8 {
F4::pack_pair(low, high)
}
#[inline(always)]
fn unpack_pair(packed: u8) -> (Self, Self) {
F4::unpack_pair(packed)
}
#[inline(always)]
fn unpack_slice_packed(packed: &[u8], unpacked: &mut [F32]) {
super::unpack::unpack_f4_to_f32_packed(packed, unpacked);
}
#[inline(always)]
fn unpack_single(element: Self) -> F32 {
F32(element.to_f32())
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct Packed4Slice<'a, T: Packable4> {
pub(crate) data: &'a [u8],
pub(crate) len: usize,
pub(crate) _marker: core::marker::PhantomData<T>,
}
impl<'a, T: Packable4> Packed4Slice<'a, T> {
#[inline]
pub fn new(data: &'a [u8], len: usize) -> Option<Self> {
let required_bytes = len.div_ceil(2);
if data.len() < required_bytes {
None
} else {
Some(Self {
data,
len,
_marker: core::marker::PhantomData,
})
}
}
#[inline(always)]
pub fn len(&self) -> usize {
self.len
}
#[inline(always)]
pub fn as_packed_slice(&self) -> &'a [u8] {
self.data
}
#[inline(always)]
pub fn is_empty(&self) -> bool {
self.len == 0
}
#[inline]
pub fn get(&self, index: usize) -> Option<T> {
if index >= self.len {
None
} else {
let byte_idx = index / 2;
let byte = self.data[byte_idx];
let (low, high) = T::unpack_pair(byte);
if index.is_multiple_of(2) {
Some(low)
} else {
Some(high)
}
}
}
#[inline]
pub fn sub_slice(self, range: core::ops::Range<usize>) -> Option<Self> {
if range.start > range.end || range.end > self.len {
return None;
}
if range.start.is_multiple_of(2) {
let byte_start = range.start / 2;
let byte_end = range.end.div_ceil(2);
let sub_data = &self.data[byte_start..byte_end];
let sub_len = range.end - range.start;
Some(Self {
data: sub_data,
len: sub_len,
_marker: core::marker::PhantomData,
})
} else {
None
}
}
}
pub struct Packed4SliceMut<'a, T: Packable4> {
pub(crate) data: &'a mut [u8],
pub(crate) len: usize,
pub(crate) _marker: core::marker::PhantomData<T>,
}
impl<'a, T: Packable4> Packed4SliceMut<'a, T> {
#[inline]
pub fn new(data: &'a mut [u8], len: usize) -> Option<Self> {
let required_bytes = len.div_ceil(2);
if data.len() < required_bytes {
None
} else {
Some(Self {
data,
len,
_marker: core::marker::PhantomData,
})
}
}
#[inline(always)]
pub fn len(&self) -> usize {
self.len
}
#[inline(always)]
pub fn as_packed_slice(&self) -> &[u8] {
self.data
}
#[inline(always)]
pub fn as_packed_slice_mut(&mut self) -> &mut [u8] {
self.data
}
#[inline]
pub fn as_borrowed(&self) -> Packed4Slice<'_, T> {
Packed4Slice {
data: self.data,
len: self.len,
_marker: core::marker::PhantomData,
}
}
#[inline]
pub fn get(&self, index: usize) -> Option<T> {
if index >= self.len {
None
} else {
let byte_idx = index / 2;
let byte = self.data[byte_idx];
let (low, high) = T::unpack_pair(byte);
if index.is_multiple_of(2) {
Some(low)
} else {
Some(high)
}
}
}
#[inline(always)]
pub fn is_empty(&self) -> bool {
self.len == 0
}
#[inline]
pub fn set(&mut self, index: usize, val: T) {
if index < self.len {
let byte_idx = index / 2;
let byte = self.data[byte_idx];
let (mut low, mut high) = T::unpack_pair(byte);
if index.is_multiple_of(2) {
low = val;
} else {
high = val;
}
self.data[byte_idx] = T::pack_pair(low, high);
}
}
#[inline]
pub fn sub_slice_mut(self, range: core::ops::Range<usize>) -> Option<Self> {
if range.start > range.end || range.end > self.len {
return None;
}
if range.start.is_multiple_of(2) {
let byte_start = range.start / 2;
let byte_end = range.end.div_ceil(2);
let sub_data = &mut self.data[byte_start..byte_end];
let sub_len = range.end - range.start;
Some(Self {
data: sub_data,
len: sub_len,
_marker: core::marker::PhantomData,
})
} else {
None
}
}
}
impl<'a, T: Packable4> Packed4Slice<'a, T> {
#[inline]
pub fn unpack(&self, dest: &mut [T::Unpacked]) {
let n = self.len.min(dest.len());
let even_len = (n / 2) * 2;
T::unpack_slice_packed(&self.data[..even_len / 2], &mut dest[..even_len]);
if n % 2 != 0 {
if let Some(b) = self.get(n - 1) {
dest[n - 1] = T::unpack_single(b);
}
}
}
}
impl<'a> Packed4Slice<'a, Bf4> {
#[inline]
pub fn unpack_to_bf16(&self, dest: &mut [Bf16]) {
self.unpack(dest);
}
}
impl<'a> Packed4Slice<'a, F4> {
#[inline]
pub fn unpack_to_f32(&self, dest: &mut [F32]) {
self.unpack(dest);
}
}
pub type PackedBf4Slice<'a> = Packed4Slice<'a, Bf4>;
pub type PackedBf4SliceMut<'a> = Packed4SliceMut<'a, Bf4>;
pub type PackedF4Slice<'a> = Packed4Slice<'a, F4>;
pub type PackedF4SliceMut<'a> = Packed4SliceMut<'a, F4>;