#[cfg(feature = "parallel")]
use super::dot_outer_kernels::{parallel_dot_matmul, parallel_outer_matmul};
use super::dot_outer_kernels::{serial_dot_matmul, serial_outer_matmul};
#[cfg(feature = "parallel")]
use super::generic_kernel::parallel_matmul;
use super::generic_kernel::serial_matmul;
use super::packing::zero_output;
use super::types::validate_matmul;
#[cfg(feature = "parallel")]
use super::types::MatmulLayout;
use crate::domain::scalar::Scalar;
use crate::infrastructure::cache::MatmulTilePolicy;
#[cfg(feature = "parallel")]
use crate::infrastructure::parallel::{PARALLEL_MIN_MATMUL_MACS, PARALLEL_MIN_MATMUL_ROWS};
use leto::{ArrayView, ArrayViewMut, Result};
#[cfg(feature = "parallel")]
#[inline]
fn is_parallel_beneficial(layout: MatmulLayout) -> bool {
layout.rows * layout.cols * layout.shared >= PARALLEL_MIN_MATMUL_MACS
&& layout.rows >= PARALLEL_MIN_MATMUL_ROWS
}
pub(super) fn route_matmul<T: Scalar>(
lhs: &ArrayView<'_, T, 2>,
rhs: &ArrayView<'_, T, 2>,
out: &mut ArrayViewMut<'_, T, 2>,
accumulate: bool,
tile_policy: MatmulTilePolicy,
) -> Result<()> {
let layout = validate_matmul(lhs, rhs, out)?;
#[cfg(not(feature = "parallel"))]
let _ = layout;
if lhs.is_c_dense() && rhs.is_f_dense() && out.is_c_dense() {
#[cfg(feature = "parallel")]
{
if is_parallel_beneficial(layout) {
parallel_dot_matmul(lhs, rhs, out, accumulate);
return Ok(());
}
}
serial_dot_matmul(lhs, rhs, out, accumulate);
return Ok(());
}
if lhs.is_f_dense() && rhs.is_c_dense() && out.is_c_dense() {
#[cfg(feature = "parallel")]
{
if is_parallel_beneficial(layout) {
parallel_outer_matmul(lhs, rhs, out, accumulate);
return Ok(());
}
}
serial_outer_matmul(lhs, rhs, out, accumulate);
return Ok(());
}
let lhs_contig;
let lhs_view = if lhs.is_c_dense() {
lhs.reborrow()
} else {
lhs_contig = lhs.to_contiguous();
lhs_contig.view()
};
let rhs_contig;
let rhs_view = if rhs.is_c_dense() {
rhs.reborrow()
} else {
rhs_contig = rhs.to_contiguous();
rhs_contig.view()
};
let layout = validate_matmul(&lhs_view, &rhs_view, out)?;
if !accumulate {
zero_output(layout, out);
}
#[cfg(feature = "parallel")]
{
if is_parallel_beneficial(layout) {
parallel_matmul(&lhs_view, &rhs_view, out, layout, tile_policy);
return Ok(());
}
}
serial_matmul(&lhs_view, &rhs_view, out, layout, tile_policy);
Ok(())
}