use crate::dist::Local;
use crate::err::Result;
use crate::exec::ExecutionDescriptor;
use crate::exec::catalog::{AddmmAttributes, AttentionAttributes, op};
use crate::exec::request::TensorHandle;
use crate::shapes::Dyn;
use crate::shapes::error::OperationKind;
use crate::shapes::prelude::{
Axis, BroadcastShape, Dim, DimCons, DimensionConstraint, DynShape, Nil, RankExpectation, Shape,
ShapeBuf, ShapeError,
};
use crate::shapes::shape::shape_buf_from_dims;
use crate::tensor::backend::Backend;
use crate::tensor::backend::Execute;
use crate::tensor::base::Tensor;
use crate::tensor::grad::{JoinedGrad, RequiresGrad};
use alloc::vec::Vec;
#[diagnostic::on_unimplemented(
message = "Cannot matrix-multiply shape `{Self}` with `{Rhs}`",
label = "Shape mismatch for matrix multiplication",
note = "Matrix multiplication requires the inner dimensions (last dim of lhs, second-to-last of rhs) to match"
)]
pub trait MatMulShape<Rhs: Shape>: Shape + DynShape {
type Output: Shape + DynShape;
fn output_shape(lhs: &ShapeBuf, rhs: &ShapeBuf) -> core::result::Result<ShapeBuf, ShapeError>;
}
pub trait StaticDim: Dim + Default + crate::shapes::ConcreteStaticExtent {}
impl<U, B> StaticDim for typenum::UInt<U, B>
where
U: typenum::Unsigned + Dim,
B: typenum::Bit
+ Default
+ Copy
+ Clone
+ core::fmt::Debug
+ Send
+ Sync
+ Eq
+ PartialEq
+ 'static,
typenum::UInt<U, B>: typenum::Unsigned
+ Default
+ Copy
+ Clone
+ core::fmt::Debug
+ Send
+ Sync
+ Eq
+ PartialEq
+ 'static,
{
}
impl StaticDim for typenum::UTerm {}
pub trait StaticOrNamedDim: Dim {}
impl<T: StaticDim> StaticOrNamedDim for T {}
impl<const N: usize> StaticOrNamedDim for crate::shapes::dim::ConstDim<N> {}
impl<Tag: crate::shapes::AxisTag, Extent: Dim> StaticOrNamedDim
for crate::shapes::dim::NamedDim<Tag, Extent>
{
}
#[inline]
fn checked_contraction(lhs_k: usize, rhs_k: usize) -> core::result::Result<(), ShapeError> {
if lhs_k == rhs_k {
Ok(())
} else {
Err(ShapeError::DimensionMismatch {
operation: OperationKind::MatMul,
axis: Axis::Named("k"),
lhs: lhs_k,
rhs: rhs_k,
constraint: DimensionConstraint::Equal,
})
}
}
#[diagnostic::on_unimplemented(
message = "Cannot contract dimension `{Self}` with `{Rhs}`",
label = "inner dimensions do not match",
note = "Matrix multiplication requires the last dim of lhs and the second-to-last of rhs to be the same, or one of them to be a runtime `usize`"
)]
pub trait ContractsWith<Rhs: Dim>: Dim {}
impl<D: StaticDim> ContractsWith<D> for D {}
impl<const N: usize> ContractsWith<crate::shapes::dim::ConstDim<N>>
for crate::shapes::dim::ConstDim<N>
{
}
impl<D: StaticOrNamedDim> ContractsWith<D> for usize {}
impl<D: StaticOrNamedDim> ContractsWith<usize> for D {}
impl<Tag, N> ContractsWith<crate::shapes::dim::NamedDim<Tag, N>> for N
where
Tag: crate::shapes::AxisTag,
N: StaticDim,
{
}
impl<Tag, N> ContractsWith<N> for crate::shapes::dim::NamedDim<Tag, N>
where
Tag: crate::shapes::AxisTag,
N: StaticDim,
{
}
impl<Tag, N> ContractsWith<crate::shapes::dim::NamedDim<Tag, usize>> for N
where
Tag: crate::shapes::AxisTag,
N: StaticDim,
{
}
impl<Tag, N> ContractsWith<N> for crate::shapes::dim::NamedDim<Tag, usize>
where
Tag: crate::shapes::AxisTag,
N: StaticDim,
{
}
impl<Tag, L, R> ContractsWith<crate::shapes::dim::NamedDim<Tag, R>>
for crate::shapes::dim::NamedDim<Tag, L>
where
Tag: crate::shapes::AxisTag,
L: Dim,
R: Dim,
{
}
impl ContractsWith<usize> for usize {}
impl<L, R, LP, RP, LM, LK, RK, RN, BOut, MatrixOut> MatMulShape<R> for L
where
L: Shape + DynShape + crate::shapes::SplitLast2<Prefix = LP, Penultimate = LM, Last = LK>,
R: Shape + DynShape + crate::shapes::SplitLast2<Prefix = RP, Penultimate = RK, Last = RN>,
LP: Shape + DynShape + BroadcastShape<RP, Output = BOut>,
RP: Shape + DynShape,
LM: Dim,
RK: Dim,
RN: Dim,
BOut: Shape
+ crate::shapes::StructuralConcatShape<DimCons<LM, DimCons<RN, Nil>>, Output = MatrixOut>,
MatrixOut: Shape + DynShape,
LK: ContractsWith<RK>,
{
type Output = MatrixOut;
fn output_shape(lhs: &ShapeBuf, rhs: &ShapeBuf) -> core::result::Result<ShapeBuf, ShapeError> {
let lhs_dims: Vec<usize> = lhs.clone().into();
let rhs_dims: Vec<usize> = rhs.clone().into();
let lhs_k = *lhs_dims.last().ok_or(ShapeError::RankMismatch {
operation: OperationKind::MatMul,
expected: RankExpectation::AtLeast(2),
actual: lhs_dims.len(),
})?;
let rhs_k = rhs_dims
.get(
rhs_dims
.len()
.checked_sub(2)
.ok_or(ShapeError::RankMismatch {
operation: OperationKind::MatMul,
expected: RankExpectation::AtLeast(2),
actual: rhs_dims.len(),
})?,
)
.copied()
.ok_or(ShapeError::RankMismatch {
operation: OperationKind::MatMul,
expected: RankExpectation::AtLeast(2),
actual: rhs_dims.len(),
})?;
checked_contraction(lhs_k, rhs_k)?;
let lhs_batch = &lhs_dims[..lhs_dims.len() - 2];
let rhs_batch = &rhs_dims[..rhs_dims.len() - 2];
let mut out = crate::shapes::broadcast::broadcast_dim_slices(lhs_batch, rhs_batch)?;
out.push(lhs_dims[lhs_dims.len() - 2]);
out.push(rhs_dims[rhs_dims.len() - 1]);
shape_buf_from_dims::<Self::Output>(OperationKind::MatMul, &out)
}
}
impl MatMulShape<Dyn> for Dyn {
type Output = Dyn;
fn output_shape(lhs: &ShapeBuf, rhs: &ShapeBuf) -> core::result::Result<ShapeBuf, ShapeError> {
const OP: OperationKind = OperationKind::MatMul;
if lhs.len() < 2 {
return Err(ShapeError::RankMismatch {
operation: OP,
expected: RankExpectation::AtLeast(2),
actual: lhs.len(),
});
}
if rhs.is_empty() {
return Err(ShapeError::RankMismatch {
operation: OP,
expected: RankExpectation::AtLeast(1),
actual: 0,
});
}
let vector_rhs = rhs.len() == 1;
let rhs_k = if vector_rhs {
rhs[0]
} else {
rhs[rhs.len() - 2]
};
checked_contraction(lhs[lhs.len() - 1], rhs_k)?;
let mut out: alloc::vec::Vec<usize> = lhs[..lhs.len() - 1].to_vec();
if !vector_rhs {
out.push(rhs[rhs.len() - 1]);
}
Ok(ShapeBuf::from_slice(&out))
}
}
impl<S1: Shape, B: Backend, K: crate::tensor::dtype::DType, G1: RequiresGrad> Tensor<S1, B, K, G1> {
#[allow(clippy::type_complexity)]
pub fn matmul<S2, G2>(
&self,
rhs: &Tensor<S2, B, K, G2>,
) -> Result<Tensor<S1::Output, B, K, crate::tensor::grad::JoinedGrad<G1, G2>>>
where
S2: Shape + DynShape,
G2: RequiresGrad,
G1: crate::tensor::grad::GradJoin<G2>,
S1: MatMulShape<S2>,
B: Execute<op::MatMulExact> + crate::exec::Capabilities,
<B as Execute<op::MatMulExact>>::Output: Into<B::Storage<K>>,
{
let validated = <crate::exec::MatMulRule as crate::exec::ShapeRule<(S1, S2)>>::lower(
&(self.shape_buf_value(), rhs.shape_buf_value()),
(),
)
.map_err(crate::err::Error::Shape)?;
let descriptor = validated.into_descriptor();
let output_dims = descriptor
.output_shape()
.cloned()
.ok_or(crate::err::Error::Shape(
crate::shapes::ShapeError::TargetShapeRejected {
operation: crate::shapes::OperationKind::MatMul,
rank: 0,
},
))?;
let output_shape = crate::shapes::ShapeValue::<S1::Output>::try_new(output_dims)
.map_err(crate::err::Error::Shape)?;
let joined_grad =
<G1 as crate::tensor::grad::GradJoin<G2>>::join_field(&self._grad, &rhs._grad);
let lhs = TensorHandle::from_storage::<B, K, Local>(&self.inner);
let rhs = TensorHandle::from_storage::<B, K, Local>(&rhs.inner);
let context = crate::tensor::grad::execution_context::<
B,
crate::tensor::grad::JoinedGrad<G1, G2>,
>(&joined_grad);
let inner = crate::tensor::grad::JoinedGrad::<G1, G2>::grad_mode(&joined_grad)
.restrict(|| {
crate::exec::dispatch::execute_shaped::<op::MatMulExact, B, S1::Output>(
&context,
crate::exec::catalog::NoAttributes,
&[lhs, rhs],
&output_shape,
)
})?
.into();
Tensor::from_shape_value(
inner,
output_shape,
self._dtype.clone(),
self._device.clone(),
joined_grad,
)
}
pub fn dot<S2: Shape>(
&self,
rhs: &Tensor<S2, B, K, G1>,
) -> Result<Tensor<crate::shapes::Nil, B, K, JoinedGrad<G1, G1>>>
where
S1: crate::tensor::ops::ShapeEq<S2>,
B: Execute<op::Mul> + Execute<op::SumAll> + crate::exec::Capabilities,
<B as Execute<op::Mul>>::Output: Into<B::Storage<K>>,
<B as Execute<op::SumAll>>::Output: Into<B::Storage<K>>,
{
<S1 as crate::tensor::ops::ShapeEq<S2>>::ASSERT_SHAPES_MATCH;
let mul = self.mul_exact(rhs)?;
mul.sum_all()
}
pub fn outer<S2: Shape + DynShape>(
&self,
rhs: &Tensor<S2, B, K, G1>,
) -> Result<Tensor<Dyn, B, K, JoinedGrad<G1, G1>>>
where
S1: DynShape,
B: Execute<op::Mul> + Execute<op::UnsqueezeExact> + crate::exec::Capabilities,
<B as Execute<op::Mul>>::Output: Into<B::Storage<K>>,
<B as Execute<op::UnsqueezeExact>>::Output: Into<B::Storage<K>>,
{
let u1 = self.unsqueeze_dyn(1)?;
let u2 = rhs.unsqueeze_dyn(0)?;
u1.broadcast_mul(&u2)
}
pub fn addmm<S2: Shape, S3: Shape>(
&self,
mat1: &Tensor<S2, B, K, G1>,
mat2: &Tensor<S3, B, K, G1>,
beta: f64,
alpha: f64,
) -> Result<Self>
where
S1: DynShape,
S2: Shape + DynShape,
S3: Shape + DynShape,
B: Execute<op::Addmm> + crate::exec::Capabilities,
<B as Execute<op::Addmm>>::Output: Into<B::Storage<K>>,
{
let output_shape = crate::shapes::ShapeValue::<S1>::try_new(self.shape_buf_value())
.map_err(crate::err::Error::Shape)?;
let bias = TensorHandle::from_storage::<B, K, Local>(&self.inner);
let lhs = TensorHandle::from_storage::<B, K, Local>(&mat1.inner);
let rhs = TensorHandle::from_storage::<B, K, Local>(&mat2.inner);
let context = crate::tensor::grad::execution_context::<B, G1>(&self._grad);
let inner = G1::grad_mode(&self._grad)
.restrict(|| {
crate::exec::dispatch::execute_shaped::<op::Addmm, B, S1>(
&context,
AddmmAttributes { alpha, beta },
&[bias, lhs, rhs],
&output_shape,
)
})?
.into();
Tensor::from_shape_value(
inner,
output_shape,
self._dtype.clone(),
self._device.clone(),
self._grad.clone(),
)
}
pub fn scaled_dot_product_attention<S2: Shape, S3: Shape, S4: Shape>(
q: &Tensor<S1, B, K, G1>,
k: &Tensor<S2, B, K, G1>,
v: &Tensor<S3, B, K, G1>,
mask: Option<&Tensor<S4, B, K, G1>>,
scale: Option<f64>,
) -> Result<Tensor<Dyn, B, K, G1>>
where
S1: DynShape,
S2: DynShape,
S3: DynShape,
S4: DynShape,
B: Execute<op::ScaledDotProductAttention> + crate::exec::Capabilities,
<B as Execute<op::ScaledDotProductAttention>>::Output: Into<B::Storage<K>>,
{
let output_shape = crate::shapes::ShapeValue::<S1>::try_new(q.shape_buf_value())
.map_err(crate::err::Error::Shape)?;
let q_handle = TensorHandle::from_storage::<B, K, Local>(&q.inner);
let k_handle = TensorHandle::from_storage::<B, K, Local>(&k.inner);
let v_handle = TensorHandle::from_storage::<B, K, Local>(&v.inner);
let mask_handle = mask.map(|m| TensorHandle::from_storage::<B, K, Local>(&m.inner));
let mut inputs = vec![q_handle, k_handle, v_handle];
if let Some(mask_handle) = mask_handle {
inputs.push(mask_handle);
}
let context = crate::tensor::grad::execution_context::<B, G1>(&q._grad);
let inner = G1::grad_mode(&q._grad)
.restrict(|| {
crate::exec::dispatch::execute_shaped::<op::ScaledDotProductAttention, B, S1>(
&context,
AttentionAttributes {
scale,
has_mask: mask.is_some(),
},
&inputs,
&output_shape,
)
})?
.into();
Tensor::<Dyn, B, K, G1>::from_shape_buf(
inner,
output_shape.shape_buf().clone(),
q._dtype.clone(),
q._device.clone(),
q._grad.clone(),
)
}
}