use crate::backend_ops::BackendOps;
use coeus_core::{Layout, Scalar, Shape, Strides};
use coeus_tensor::Tensor;
#[inline]
pub fn matmul<T: Scalar, B: BackendOps<T> + Default>(
a: &Tensor<T, B>,
b: &Tensor<T, B>,
backend: &B,
) -> Result<Tensor<T, B>, B::Error> {
let a_ndim = a.ndim();
let b_ndim = b.ndim();
assert!(a_ndim >= 2, "matmul: A must be ≥ 2-D, got {a_ndim}-D");
assert!(b_ndim >= 2, "matmul: B must be ≥ 2-D, got {b_ndim}-D");
let a_shape = a.shape();
let b_shape = b.shape();
let m = a_shape[a_ndim - 2];
let k = a_shape[a_ndim - 1];
let k2 = b_shape[b_ndim - 2];
let n = b_shape[b_ndim - 1];
assert_eq!(k, k2, "matmul: inner dimension mismatch: {} vs {}", k, k2);
if a_ndim == 2 && b_ndim == 2 {
let mut out = unsafe { Tensor::alloc_on([m, n], backend) }?;
let (out_storage, out_layout) = out.storage_mut_and_layout()?;
backend.matmul(
a.storage(),
a.layout(),
b.storage(),
b.layout(),
out_storage,
out_layout,
)?;
return Ok(out);
}
let a_slices: usize = if a_ndim > 2 {
a_shape[..a_ndim - 2].iter().product()
} else {
1
};
let b_slices: usize = if b_ndim > 2 {
b_shape[..b_ndim - 2].iter().product()
} else {
1
};
assert!(
a_slices == b_slices || a_slices == 1 || b_slices == 1,
"matmul: batch dimensions not broadcastable: {} vs {}",
a_slices,
b_slices
);
let batch_size = a_slices.max(b_slices);
let a_batch = &a_shape[..a_ndim.saturating_sub(2)];
let b_batch = &b_shape[..b_ndim.saturating_sub(2)];
let batch_shape = match (a_slices, b_slices) {
(1, 1) if a_ndim == 2 && b_ndim > 2 => b_batch.to_vec(),
(1, 1) if b_ndim == 2 && a_ndim > 2 => a_batch.to_vec(),
(1, _) => b_batch.to_vec(),
(_, 1) => a_batch.to_vec(),
_ => {
assert_eq!(
a_batch, b_batch,
"matmul: non-singleton batch dimensions must match exactly"
);
a_batch.to_vec()
}
};
let mut out_shape = batch_shape;
out_shape.extend([m, n]);
let mut out = unsafe { Tensor::alloc_on(out_shape, backend) }?;
let a_storage = a.storage();
let b_storage = b.storage();
let (out_storage, out_layout) = out.storage_mut_and_layout()?;
let a_layout = batch_layout(a.layout(), a_slices, m, k, a_ndim == 2);
let b_layout = batch_layout(b.layout(), b_slices, k, n, b_ndim == 2);
let c_layout = Layout::from_shape_strides(
Shape::from([batch_size, m, n].as_slice()),
Strides::from([m * n, n, 1].as_slice()),
out_layout.offset(),
);
backend.batched_matmul(
a_storage,
&a_layout,
b_storage,
&b_layout,
out_storage,
&c_layout,
)?;
Ok(out)
}
fn batch_layout(
layout: &Layout,
batches: usize,
rows: usize,
cols: usize,
is_rank2: bool,
) -> Layout {
let batch_stride = if is_rank2 { 0 } else { rows * cols };
Layout::from_shape_strides(
Shape::from([batches, rows, cols].as_slice()),
Strides::from([batch_stride, cols, 1].as_slice()),
layout.offset(),
)
}
#[inline]
pub fn matmul_accumulate<T: Scalar, B: BackendOps<T> + Default>(
a: &Tensor<T, B>,
b: &Tensor<T, B>,
out: &mut Tensor<T, B>,
backend: &B,
) -> Result<(), B::Error> {
let a_ndim = a.ndim();
let b_ndim = b.ndim();
assert!(
a_ndim >= 2,
"matmul_accumulate: A must be ≥ 2-D, got {a_ndim}-D"
);
assert!(
b_ndim >= 2,
"matmul_accumulate: B must be ≥ 2-D, got {b_ndim}-D"
);
let a_shape = a.shape();
let b_shape = b.shape();
let m = a_shape[a_ndim - 2];
let k = a_shape[a_ndim - 1];
let k2 = b_shape[b_ndim - 2];
let n = b_shape[b_ndim - 1];
assert_eq!(
k, k2,
"matmul_accumulate: inner dimension mismatch: {} vs {}",
k, k2
);
if a_ndim == 2 && b_ndim == 2 {
let (out_storage, out_layout) = out.storage_mut_and_layout()?;
backend.matmul_accumulate(
a.storage(),
a.layout(),
b.storage(),
b.layout(),
out_storage,
out_layout,
)?;
return Ok(());
}
let a_slices: usize = if a_ndim > 2 {
a_shape[..a_ndim - 2].iter().product()
} else {
1
};
let b_slices: usize = if b_ndim > 2 {
b_shape[..b_ndim - 2].iter().product()
} else {
1
};
assert!(
a_slices == b_slices || a_slices == 1 || b_slices == 1,
"matmul_accumulate: batch dimensions not broadcastable: {} vs {}",
a_slices,
b_slices
);
let a_layout = batch_layout(a.layout(), a_slices, m, k, a_ndim == 2);
let b_layout = batch_layout(b.layout(), b_slices, k, n, b_ndim == 2);
assert_eq!(
out.numel(),
a_slices.max(b_slices) * m * n,
"matmul_accumulate: output element count must match batched product"
);
let (out_storage, out_layout) = out.storage_mut_and_layout()?;
let c_layout = Layout::from_shape_strides(
Shape::from([a_slices.max(b_slices), m, n].as_slice()),
Strides::from([m * n, n, 1].as_slice()),
out_layout.offset(),
);
backend.batched_matmul_accumulate(
a.storage(),
&a_layout,
b.storage(),
&b_layout,
out_storage,
&c_layout,
)?;
Ok(())
}