use super::{Element, private};
use crate::{
Vector,
kernels::matmul,
utils::{Load, Store},
};
pub struct Matrix<T: Element<R, C>, const R: usize, const C: usize> {
pub(crate) storage: <T as private::SealedElement<C, R>>::Storage,
}
mod impls;
macro_rules! call {
(<$t:ty, $r:tt, $c:tt>::$f:ident $(::<$gen:ty>)? $(($($arg:expr),*))?) => {
<$t as $crate::private::SealedElement<$c, $r>>::$f $(::<$gen>)? $(($($arg),*))?
};
($w:ident(<$t:ty, $r:tt, $c:tt>::$f:ident $(::<$gen:ty>)? $(($($arg:expr),*))?)) => {
$w { storage: $crate::row_major::call!(<$t, $r, $c>::$f $(::<$gen>)? $(($($arg),*))?) }
};
}
pub(crate) use call;
pub trait MatrixProduct<const R: usize, const K: usize, const C: usize>:
Element<R, K> + OuterProduct<R, C> + VectorMatrixProduct<K, C>
{
#[doc(hidden)]
fn __matrix_product(lhs: Matrix<Self, R, K>, rhs: Matrix<Self, K, C>) -> Matrix<Self, R, C>;
}
pub trait VectorMatrixProduct<const R: usize, const C: usize>: Element<R, C> {
#[doc(hidden)]
fn __vector_matrix_product(lhs: Vector<Self, R>, rhs: Matrix<Self, R, C>) -> Vector<Self, C>;
}
pub trait OuterProduct<const R: usize, const C: usize>: Element<R, C> {
#[doc(hidden)]
fn __outer_product(lhs: Matrix<Self, R, 1>, rhs: Vector<Self, C>) -> Matrix<Self, R, C>;
}
impl<T: MatrixProduct<R, K, C>, const R: usize, const K: usize, const C: usize>
core::ops::Mul<Matrix<T, K, C>> for Matrix<T, R, K>
{
type Output = Matrix<T, R, C>;
#[inline]
fn mul(self, rhs: Matrix<T, K, C>) -> Self::Output {
MatrixProduct::__matrix_product(self, rhs)
}
}
impl<T: VectorMatrixProduct<R, C>, const R: usize, const C: usize> core::ops::Mul<Matrix<T, R, C>>
for Vector<T, R>
{
type Output = Vector<T, C>;
#[inline]
fn mul(self, rhs: Matrix<T, R, C>) -> Self::Output {
VectorMatrixProduct::__vector_matrix_product(self, rhs)
}
}
impl<T: OuterProduct<R, C>, const R: usize, const C: usize> core::ops::Mul<Vector<T, C>>
for Matrix<T, R, 1>
{
type Output = Matrix<T, R, C>;
#[inline]
fn mul(self, rhs: Vector<T, C>) -> Self::Output { OuterProduct::__outer_product(self, rhs) }
}
impl<T: VectorMatrixProduct<N, N>, const N: usize> core::ops::MulAssign<Matrix<T, N, N>>
for Vector<T, N>
{
#[inline]
fn mul_assign(&mut self, rhs: Matrix<T, N, N>) { *self = *self * rhs; }
}
macro_rules! impl_mat_mul_mat {
($self:ident, [$($a:literal),*]; $b:tt; $c:tt) => {
$(impl_mat_mul_mat!(@a $self, $a; $b; $c);)*
};
(@a $self:ident, $a:literal; [$($b:literal),*]; $c:tt) => {
$(impl_mat_mul_mat!(@ab $self, $a; $b; $c);)*
};
(@ab $self:ident, $a:literal; $b:literal; [$($c:literal),*]) => {
$(
paste::paste! {
impl_mat_mul_mat!(@c $self, $a, $b, $c, [<matmul $c x $b x $a>]);
}
)*
};
(@c $self:ident, $a:literal, $b:literal, $c:literal, $f:ident) => {
impl MatrixProduct<$a, $b, $c> for $self {
#[doc(hidden)]
#[inline(always)]
fn __matrix_product(
lhs: Matrix<Self, $a, $b>,
rhs: Matrix<Self, $b, $c>,
) -> Matrix<Self, $a, $c> {
Matrix { storage: matmul::$self::$f(rhs.storage.load(), lhs.storage.load()).store() }
}
}
};
}
macro_rules! impl_vec_mul_mat {
($self:ident, $a:literal, $b:literal, $f:ident) => {
impl VectorMatrixProduct<$a, $b> for $self {
#[doc(hidden)]
#[inline(always)]
fn __vector_matrix_product(
lhs: Vector<Self, $a>,
rhs: Matrix<Self, $a, $b>,
) -> Vector<Self, $b> {
Vector {
storage: matmul::$self::$f(rhs.storage.load(), lhs.storage.load()).store(),
}
}
}
};
}
macro_rules! impl_mat_mul_vec {
($self:ident, $a:literal, $b:literal, $f:ident) => {
impl OuterProduct<$a, $b> for $self {
#[doc(hidden)]
#[inline(always)]
fn __outer_product(
lhs: Matrix<Self, $a, 1>,
rhs: Vector<Self, $b>,
) -> Matrix<Self, $a, $b> {
Matrix {
storage: matmul::$self::$f(rhs.storage.load(), lhs.storage.load()).store(),
}
}
}
};
}
macro_rules! impl_mat_mul_float {
($($self:ident),+) => {
$(
impl_mat_mul_mat!($self, [1, 2, 3, 4]; [1, 2, 3, 4]; [1, 2, 3, 4]);
impl_vec_mul_mat!($self, 1, 1, matmul1x1x1);
impl_vec_mul_mat!($self, 1, 2, matmul2x1x1);
impl_vec_mul_mat!($self, 1, 3, matmul3x1x1);
impl_vec_mul_mat!($self, 1, 4, matmul4x1x1);
impl_vec_mul_mat!($self, 2, 1, matmul1x2x1);
impl_vec_mul_mat!($self, 2, 2, matmul2x2x1);
impl_vec_mul_mat!($self, 2, 3, matmul3x2x1);
impl_vec_mul_mat!($self, 2, 4, matmul4x2x1);
impl_vec_mul_mat!($self, 3, 1, matmul1x3x1);
impl_vec_mul_mat!($self, 3, 2, matmul2x3x1);
impl_vec_mul_mat!($self, 3, 3, matmul3x3x1);
impl_vec_mul_mat!($self, 3, 4, matmul4x3x1);
impl_vec_mul_mat!($self, 4, 1, matmul1x4x1);
impl_vec_mul_mat!($self, 4, 2, matmul2x4x1);
impl_vec_mul_mat!($self, 4, 3, matmul3x4x1);
impl_vec_mul_mat!($self, 4, 4, matmul4x4x1);
impl_mat_mul_vec!($self, 1, 1, matmul1x1x1);
impl_mat_mul_vec!($self, 1, 2, matmul2x1x1);
impl_mat_mul_vec!($self, 1, 3, matmul3x1x1);
impl_mat_mul_vec!($self, 1, 4, matmul4x1x1);
impl_mat_mul_vec!($self, 2, 1, matmul1x1x2);
impl_mat_mul_vec!($self, 2, 2, matmul2x1x2);
impl_mat_mul_vec!($self, 2, 3, matmul3x1x2);
impl_mat_mul_vec!($self, 2, 4, matmul4x1x2);
impl_mat_mul_vec!($self, 3, 1, matmul1x1x3);
impl_mat_mul_vec!($self, 3, 2, matmul2x1x3);
impl_mat_mul_vec!($self, 3, 3, matmul3x1x3);
impl_mat_mul_vec!($self, 3, 4, matmul4x1x3);
impl_mat_mul_vec!($self, 4, 1, matmul1x1x4);
impl_mat_mul_vec!($self, 4, 2, matmul2x1x4);
impl_mat_mul_vec!($self, 4, 3, matmul3x1x4);
impl_mat_mul_vec!($self, 4, 4, matmul4x1x4);
)+
};
}
impl_mat_mul_float!(f32, f64);