use crate::domain::strategy::{SimdOperations, SimdStrategy};
use eunomia::{Bf16, Complex, NumericElement, F16};
use super::fallback::{
scalar_axpy_rows_batch_fallback, scalar_axpy_rows_fallback, scalar_gemv_strided_fallback,
scalar_gemv_transpose_strided_fallback, scalar_tiled_gemm_fallback,
};
use super::Scalar;
macro_rules! impl_scalar_simd {
($t:ty) => {
impl Scalar for $t {
#[inline]
fn add_slice(a: &[Self], b: &[Self], out: &mut [Self]) {
assert_eq!(a.len(), b.len(), "add_slice: a.len() != b.len()");
assert_eq!(a.len(), out.len(), "add_slice: output length mismatch");
if <SimdStrategy as SimdOperations<Self>>::add_slice(a, b, out).is_ok() {
return;
}
for ((o, &x), &y) in out.iter_mut().zip(a.iter()).zip(b.iter()) {
*o = x + y;
}
}
#[inline]
fn sub_slice(a: &[Self], b: &[Self], out: &mut [Self]) {
assert_eq!(a.len(), b.len(), "sub_slice: a.len() != b.len()");
assert_eq!(a.len(), out.len(), "sub_slice: output length mismatch");
if <SimdStrategy as SimdOperations<Self>>::sub_slice(a, b, out).is_ok() {
return;
}
for ((o, &x), &y) in out.iter_mut().zip(a.iter()).zip(b.iter()) {
*o = x - y;
}
}
#[inline]
fn mul_slice(a: &[Self], b: &[Self], out: &mut [Self]) {
assert_eq!(a.len(), b.len(), "mul_slice: a.len() != b.len()");
assert_eq!(a.len(), out.len(), "mul_slice: output length mismatch");
if <SimdStrategy as SimdOperations<Self>>::mul_slice(a, b, out).is_ok() {
return;
}
for ((o, &x), &y) in out.iter_mut().zip(a.iter()).zip(b.iter()) {
*o = x * y;
}
}
#[inline]
fn div_slice(a: &[Self], b: &[Self], out: &mut [Self]) {
assert_eq!(a.len(), b.len(), "div_slice: a.len() != b.len()");
assert_eq!(a.len(), out.len(), "div_slice: output length mismatch");
if <SimdStrategy as SimdOperations<Self>>::div_slice(a, b, out).is_ok() {
return;
}
for ((o, &x), &y) in out.iter_mut().zip(a.iter()).zip(b.iter()) {
*o = x / y;
}
}
#[inline]
fn scale_slice(data: &mut [Self], scalar: Self) {
if <SimdStrategy as SimdOperations<Self>>::scale_slice(data, scalar).is_ok() {
return;
}
for x in data.iter_mut() {
*x *= scalar;
}
}
#[inline]
fn sum_slice(s: &[Self]) -> Self {
if let Some(res) = <SimdStrategy as SimdOperations<Self>>::sum_slice(s) {
res
} else {
s.iter()
.copied()
.fold(<Self as NumericElement>::ZERO, |acc, x| acc + x)
}
}
#[inline]
fn dot_slice(a: &[Self], b: &[Self]) -> Self {
assert_eq!(a.len(), b.len(), "dot_slice: a.len() != b.len()");
if let Some(res) = <SimdStrategy as SimdOperations<Self>>::dot_slice(a, b) {
res
} else {
a.iter()
.copied()
.zip(b.iter().copied())
.fold(<Self as NumericElement>::ZERO, |acc, (x, y)| acc + x * y)
}
}
#[inline]
fn axpy_slice(alpha: Self, x: &[Self], out: &mut [Self]) {
assert_eq!(x.len(), out.len(), "axpy_slice: x.len() != out.len()");
if <SimdStrategy as SimdOperations<Self>>::axpy_slice(alpha, x, out).is_ok() {
return;
}
for (o, &xv) in out.iter_mut().zip(x.iter()) {
*o += alpha * xv;
}
}
#[inline]
fn axpy_rows(
alphas: &[Self],
x: &[Self],
out: &mut [Self],
row_stride: usize,
rows: usize,
cols: usize,
) {
if <SimdStrategy as SimdOperations<Self>>::axpy_rows(
alphas, x, out, row_stride, rows, cols,
)
.is_ok()
{
return;
}
scalar_axpy_rows_fallback(alphas, x, out, row_stride, rows, cols);
}
#[inline]
fn axpy_rows_batch(
alphas: &[Self],
x_panel: &[Self],
out: &mut [Self],
row_stride: usize,
rows: usize,
depth: usize,
cols: usize,
) {
if <SimdStrategy as SimdOperations<Self>>::axpy_rows_batch(
alphas, x_panel, out, row_stride, rows, depth, cols,
)
.is_ok()
{
return;
}
scalar_axpy_rows_batch_fallback(
alphas, x_panel, out, row_stride, rows, depth, cols,
);
}
#[inline]
fn tiled_gemm(a: &[Self], b: &[Self], c: &mut [Self], m: usize, n: usize, k: usize) {
if <SimdStrategy as SimdOperations<Self>>::tiled_gemm(a, b, c, m, n, k).is_ok() {
return;
}
scalar_tiled_gemm_fallback(a, b, c, m, n, k);
}
#[inline]
fn gemv_strided(
a: &[Self],
x: &[Self],
y: &mut [Self],
nrows: usize,
ncols: usize,
lda: usize,
) {
if <SimdStrategy as SimdOperations<Self>>::gemv_strided(a, x, y, nrows, ncols, lda)
.is_ok()
{
return;
}
scalar_gemv_strided_fallback(a, x, y, nrows, ncols, lda);
}
#[inline]
fn gemv_transpose_strided(
a: &[Self],
x: &[Self],
y: &mut [Self],
nrows: usize,
ncols: usize,
lda: usize,
) {
if <SimdStrategy as SimdOperations<Self>>::gemv_transpose_strided(
a, x, y, nrows, ncols, lda,
)
.is_ok()
{
return;
}
scalar_gemv_transpose_strided_fallback(a, x, y, nrows, ncols, lda);
}
#[inline]
fn min_slice(s: &[Self]) -> Self {
if let Some(res) = <SimdStrategy as SimdOperations<Self>>::min_slice(s) {
res
} else {
s.iter()
.copied()
.fold(<Self as NumericElement>::MAX_VALUE, |acc, x| {
if x < acc {
x
} else {
acc
}
})
}
}
#[inline]
fn max_slice(s: &[Self]) -> Self {
if let Some(res) = <SimdStrategy as SimdOperations<Self>>::max_slice(s) {
res
} else {
s.iter()
.copied()
.fold(<Self as NumericElement>::MIN_VALUE, |acc, x| {
if x > acc {
x
} else {
acc
}
})
}
}
#[inline]
fn jaccard_distance(a: &[Self], b: &[Self]) -> Option<f64> {
<SimdStrategy as SimdOperations<Self>>::jaccard_distance(a, b)
}
#[inline]
fn hamming_distance(a: &[Self], b: &[Self]) -> Option<u64> {
<SimdStrategy as SimdOperations<Self>>::hamming_distance(a, b)
}
}
};
}
macro_rules! impl_scalar_plain {
($t:ty) => {
impl Scalar for $t {}
};
}
impl_scalar_simd!(f32);
impl_scalar_simd!(f64);
impl_scalar_simd!(F16);
impl_scalar_simd!(Bf16);
impl<T> Scalar for Complex<T> where T: Scalar + core::ops::Neg<Output = T> {}
impl_scalar_plain!(i8);
impl_scalar_plain!(u8);
impl_scalar_plain!(i16);
impl_scalar_plain!(u16);
impl_scalar_plain!(i32);
impl_scalar_plain!(u32);
impl_scalar_plain!(i64);
impl_scalar_plain!(u64);
impl_scalar_plain!(isize);
impl_scalar_plain!(usize);