#![no_std]
#[cfg(jflib_module)]
extern crate modrt as rt;
#[cfg(target_arch = "x86_64")]
mod native;
#[cfg(target_arch = "x86_64")]
pub use native::{Sse2, Avx2};
mod scalar;
pub use scalar::{Scalar, Scalar256};
#[cfg(target_arch = "aarch64")] mod neon;
#[cfg(target_arch = "aarch64")] pub use neon::{Neon128, Neon128 as Sse2};
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] mod wasm;
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] pub use wasm::{Wasm128, Wasm128 as Sse2};
#[cfg(any(target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128")))]
pub use scalar::Scalar256 as Avx2;
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64", all(target_arch = "wasm32", target_feature = "simd128"))))]
pub use scalar::{Scalar as Sse2, Scalar256 as Avx2};
pub use rt::cpu::CPU_AVX2;
#[inline]
pub fn has_avx2() -> bool {
rt::cpu::features() & CPU_AVX2 != 0
}
#[cfg(target_arch = "x86_64")]
pub fn copy_stream(dst: &mut [u8], src: &[u8]) {
use core::arch::x86_64::{__m128i, _mm_loadu_si128, _mm_stream_si128};
let n = dst.len().min(src.len());
let (dst, src) = (&mut dst[..n], &src[..n]);
let head = (dst.as_ptr() as usize).wrapping_neg() & 15;
if n < head + 64 {
dst.copy_from_slice(src);
return;
}
dst[..head].copy_from_slice(&src[..head]);
let body = (n - head) & !63;
let (d, s) = (&mut dst[head..head + body], &src[head..head + body]);
let mut i = 0;
while i < body {
unsafe {
let (sp, dp) = (s.as_ptr().add(i) as *const __m128i, d.as_mut_ptr().add(i) as *mut __m128i);
let (a, b, c, e) = (_mm_loadu_si128(sp), _mm_loadu_si128(sp.add(1)), _mm_loadu_si128(sp.add(2)), _mm_loadu_si128(sp.add(3)));
_mm_stream_si128(dp, a);
_mm_stream_si128(dp.add(1), b);
_mm_stream_si128(dp.add(2), c);
_mm_stream_si128(dp.add(3), e);
}
i += 64;
}
dst[head + body..].copy_from_slice(&src[head + body..]);
}
#[cfg(not(target_arch = "x86_64"))]
pub fn copy_stream(dst: &mut [u8], src: &[u8]) {
let n = dst.len().min(src.len());
dst[..n].copy_from_slice(&src[..n]);
}
#[inline]
pub fn stream_fence() {
#[cfg(target_arch = "x86_64")]
unsafe {
core::arch::x86_64::_mm_sfence()
};
}
pub trait Kernel {
type Out;
fn run<S: Simd>(self, s: S) -> Self::Out;
}
#[inline]
#[cfg(target_arch = "x86_64")]
pub fn dispatch<K: Kernel>(k: K) -> K::Out {
match Avx2::new() {
Some(a) => unsafe { run_avx2(k, a) },
None => k.run(Sse2),
}
}
#[inline]
pub fn dispatch_sse2<K: Kernel>(k: K) -> K::Out {
k.run(Sse2)
}
#[cfg(not(target_arch = "x86_64"))]
#[inline]
pub fn dispatch<K: Kernel>(k: K) -> K::Out { k.run(Sse2) }
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2")]
fn run_avx2<K: Kernel>(k: K, a: Avx2) -> K::Out {
k.run(a)
}
pub trait Simd: Copy {
type V: Copy;
const BYTES: usize;
const PX: usize;
fn zero(self) -> Self::V;
fn splat8(self, v: u8) -> Self::V;
fn splat16(self, v: u16) -> Self::V;
fn splat32(self, v: u32) -> Self::V;
fn splat64(self, v: u64) -> Self::V;
fn splatf(self, v: f32) -> Self::V;
fn load(self, b: &[u8]) -> Self::V;
fn store(self, b: &mut [u8], v: Self::V);
fn load_px(self, b: &[u8]) -> Self::V;
fn store_px(self, b: &mut [u8], v: Self::V);
fn bcast_px(self, b: &[u8]) -> Self::V;
fn load_raw(self, b: &[u8]) -> Self::V;
fn store_raw(self, b: &mut [u8], v: Self::V);
fn widen_px(self, v: Self::V) -> Self::V;
fn load_lane32(self, b: &[u8]) -> Self::V;
fn load_i32(self, b: &[i32]) -> Self::V;
fn store_i32(self, b: &mut [i32], v: Self::V);
fn from_i32(self, v: i32) -> Self::V;
fn to_i32(self, v: Self::V) -> i32;
fn add8(self, a: Self::V, b: Self::V) -> Self::V;
fn sub8(self, a: Self::V, b: Self::V) -> Self::V;
fn addus8(self, a: Self::V, b: Self::V) -> Self::V;
fn subus8(self, a: Self::V, b: Self::V) -> Self::V;
fn maxu8(self, a: Self::V, b: Self::V) -> Self::V;
fn minu8(self, a: Self::V, b: Self::V) -> Self::V;
fn cmpeq8(self, a: Self::V, b: Self::V) -> Self::V;
fn avgu8(self, a: Self::V, b: Self::V) -> Self::V;
fn movemask8(self, a: Self::V) -> u32;
fn unpacklo8(self, a: Self::V, b: Self::V) -> Self::V;
fn unpackhi8(self, a: Self::V, b: Self::V) -> Self::V;
fn packus16(self, a: Self::V, b: Self::V) -> Self::V;
fn add16(self, a: Self::V, b: Self::V) -> Self::V;
fn sub16(self, a: Self::V, b: Self::V) -> Self::V;
fn subus16(self, a: Self::V, b: Self::V) -> Self::V;
fn mullo16(self, a: Self::V, b: Self::V) -> Self::V;
fn mulhiu16(self, a: Self::V, b: Self::V) -> Self::V;
fn mins16(self, a: Self::V, b: Self::V) -> Self::V;
fn maxs16(self, a: Self::V, b: Self::V) -> Self::V;
fn cmpeq16(self, a: Self::V, b: Self::V) -> Self::V;
fn cmpgts16(self, a: Self::V, b: Self::V) -> Self::V;
fn srl16<const N: i32>(self, a: Self::V) -> Self::V;
fn sll16<const N: i32>(self, a: Self::V) -> Self::V;
fn shuf16<const IMM: i32>(self, a: Self::V) -> Self::V;
fn unpacklo16(self, a: Self::V, b: Self::V) -> Self::V;
fn unpackhi16(self, a: Self::V, b: Self::V) -> Self::V;
fn packs32(self, a: Self::V, b: Self::V) -> Self::V;
fn madd16(self, a: Self::V, b: Self::V) -> Self::V;
fn add32(self, a: Self::V, b: Self::V) -> Self::V;
fn sub32(self, a: Self::V, b: Self::V) -> Self::V;
fn cmpeq32(self, a: Self::V, b: Self::V) -> Self::V;
fn cmpgts32(self, a: Self::V, b: Self::V) -> Self::V;
fn movemask32(self, a: Self::V) -> u32;
fn shuf32<const IMM: i32>(self, a: Self::V) -> Self::V;
fn shufps<const IMM: i32>(self, a: Self::V, b: Self::V) -> Self::V;
fn srl32<const N: i32>(self, a: Self::V) -> Self::V;
fn sll32<const N: i32>(self, a: Self::V) -> Self::V;
fn sra32<const N: i32>(self, a: Self::V) -> Self::V;
fn bsll<const N: i32>(self, a: Self::V) -> Self::V;
fn bsrl<const N: i32>(self, a: Self::V) -> Self::V;
fn and(self, a: Self::V, b: Self::V) -> Self::V;
fn andnot(self, a: Self::V, b: Self::V) -> Self::V;
fn or(self, a: Self::V, b: Self::V) -> Self::V;
fn xor(self, a: Self::V, b: Self::V) -> Self::V;
fn cvt_i32_f32(self, a: Self::V) -> Self::V;
fn cvtt_f32_i32(self, a: Self::V) -> Self::V;
fn cvtr_f32_i32(self, a: Self::V) -> Self::V;
fn addf(self, a: Self::V, b: Self::V) -> Self::V;
fn subf(self, a: Self::V, b: Self::V) -> Self::V;
fn mulf(self, a: Self::V, b: Self::V) -> Self::V;
fn divf(self, a: Self::V, b: Self::V) -> Self::V;
fn splatd(self, v: f64) -> Self::V;
fn addd(self, a: Self::V, b: Self::V) -> Self::V;
fn subd(self, a: Self::V, b: Self::V) -> Self::V;
fn muld(self, a: Self::V, b: Self::V) -> Self::V;
fn divd(self, a: Self::V, b: Self::V) -> Self::V;
fn sqrtd(self, a: Self::V) -> Self::V;
fn maxd(self, a: Self::V, b: Self::V) -> Self::V;
fn mind(self, a: Self::V, b: Self::V) -> Self::V;
fn cmpltd(self, a: Self::V, b: Self::V) -> Self::V;
fn cvt_i32_f64(self, a: Self::V) -> Self::V;
fn cvtt_f64_i32(self, a: Self::V) -> Self::V;
fn store_i32_half(self, b: &mut [i32], v: Self::V);
fn sra16<const N: i32>(self, a: Self::V) -> Self::V;
fn mulhis16(self, a: Self::V, b: Self::V) -> Self::V;
fn packs16(self, a: Self::V, b: Self::V) -> Self::V;
fn load_i16(self, b: &[i16]) -> Self::V;
fn store_i16(self, b: &mut [i16], v: Self::V);
fn store_lanes2(self, b: &mut [u8], lo: Self::V, hi: Self::V);
fn load_lanes2(self, b: &[u8]) -> (Self::V, Self::V);
fn load_i16_lanes(self, a: &[i16], b: &[i16]) -> Self::V;
fn store_raw_lanes(self, a: &mut [u8], b: &mut [u8], v: Self::V);
fn sadu8(self, a: Self::V, b: Self::V) -> Self::V;
fn add64(self, a: Self::V, b: Self::V) -> Self::V;
fn maxf(self, a: Self::V, b: Self::V) -> Self::V;
fn minf(self, a: Self::V, b: Self::V) -> Self::V;
fn sqrtf(self, a: Self::V) -> Self::V;
fn load_u16(self, b: &[u16]) -> Self::V;
fn store_u16(self, b: &mut [u16], v: Self::V);
fn load_u16_lo(self, b: &[u16]) -> Self::V;
fn store_u16_lo(self, b: &mut [u16], v: Self::V);
fn srl16x(self, a: Self::V, n: u32) -> Self::V;
fn load_f32(self, b: &[f32]) -> Self::V;
fn store_f32(self, b: &mut [f32], v: Self::V);
fn adds8(self, a: Self::V, b: Self::V) -> Self::V;
fn subs8(self, a: Self::V, b: Self::V) -> Self::V;
fn cmpltf(self, a: Self::V, b: Self::V) -> Self::V;
fn load_f32_lanes(self, a: &[f32], b: &[f32]) -> Self::V;
fn store_f32_lanes(self, a: &mut [f32], b: &mut [f32], v: Self::V);
fn packssdw(self, a: Self::V, b: Self::V) -> Self::V;
fn packuswb_px(self, v: Self::V) -> Self::V;
fn ptest_raw(self, a: Self::V, b: Self::V) -> u32;
fn ptestz_raw(self, a: Self::V, b: Self::V) -> bool;
}