Struct Avx512 

pub struct Avx512;

x86/x86_64 AVX-512F instruction set architecture marker.

Trait Implementations

impl Clone for Avx512

fn clone(&self) -> Avx512

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Copy for Avx512

impl Debug for Avx512

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Eq for Avx512

impl PartialEq for Avx512

fn eq(&self, other: &Avx512) -> bool

Equality operator ==.

fn ne(&self, other: &Rhs) -> bool

Inequality operator !=.

impl SimdArch for Avx512

const NAME: &'static str = "avx512"

Human-readable ISA name ("avx2", "avx512", "neon", "scalar").

const REGISTER_WIDTH_BITS: u32 = 512

Width of vector registers in bits.

const ISA_FAMILY: IsaFamily = hermes_simd_core::arch::IsaFamily::X86

ISA family for this architecture.

const FMA_THROUGHPUT_HINT: u32 = 8

Suggested TILE_M value for tiled_dot to saturate FMA throughput.

fn is_runtime_supported() -> bool

Returns true when the current host may execute this architecture’s native instructions from safe wrappers.

impl SimdKernel<Bf4> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 64

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [Bf4; 64]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 64]

Hardware-native mask type.

type IndexVector = [i32; 64]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const Bf4) -> <Avx512 as SimdKernel<Bf4>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const Bf4) -> <Avx512 as SimdKernel<Bf4>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut Bf4, val: <Avx512 as SimdKernel<Bf4>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned( ptr: *mut Bf4, val: <Avx512 as SimdKernel<Bf4>>::Vector, )

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<Bf4>>::Vector, b: <Avx512 as SimdKernel<Bf4>>::Vector, ) -> <Avx512 as SimdKernel<Bf4>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<Bf4>>::Vector, b: <Avx512 as SimdKernel<Bf4>>::Vector, ) -> <Avx512 as SimdKernel<Bf4>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<Bf4>>::Vector, b: <Avx512 as SimdKernel<Bf4>>::Vector, ) -> <Avx512 as SimdKernel<Bf4>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<Bf4>>::Vector, ) -> <Avx512 as SimdKernel<Bf4>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<Bf4>>::Vector, b: <Avx512 as SimdKernel<Bf4>>::Vector, c: <Avx512 as SimdKernel<Bf4>>::Vector, ) -> <Avx512 as SimdKernel<Bf4>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<Bf4>>::Vector) -> Bf4

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<Bf4>>::Vector, mask: <Avx512 as SimdKernel<Bf4>>::Mask, ) -> <Avx512 as SimdKernel<Bf4>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<Bf4>>::Vector, mask: <Avx512 as SimdKernel<Bf4>>::Mask, fill: <Avx512 as SimdKernel<Bf4>>::Vector, ) -> <Avx512 as SimdKernel<Bf4>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const Bf4, indices: <Avx512 as SimdKernel<Bf4>>::IndexVector, ) -> <Avx512 as SimdKernel<Bf4>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const Bf4, indices: <Avx512 as SimdKernel<Bf4>>::IndexVector, mask: <Avx512 as SimdKernel<Bf4>>::Mask, src: <Avx512 as SimdKernel<Bf4>>::Vector, ) -> <Avx512 as SimdKernel<Bf4>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<Bf4>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<Bf4>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<Bf4>>::Vector

Set all lanes to zero.

unsafe fn splat(val: Bf4) -> <Avx512 as SimdKernel<Bf4>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<Bf4>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<Bf4>>::Mask, ) -> <Avx512 as SimdKernel<Bf4>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<Bf4>>::Vector, ) -> <Avx512 as SimdKernel<Bf4>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<Bf8> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 64

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [Bf8; 64]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 64]

Hardware-native mask type.

type IndexVector = [i32; 64]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const Bf8) -> <Avx512 as SimdKernel<Bf8>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const Bf8) -> <Avx512 as SimdKernel<Bf8>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut Bf8, val: <Avx512 as SimdKernel<Bf8>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned( ptr: *mut Bf8, val: <Avx512 as SimdKernel<Bf8>>::Vector, )

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<Bf8>>::Vector, b: <Avx512 as SimdKernel<Bf8>>::Vector, ) -> <Avx512 as SimdKernel<Bf8>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<Bf8>>::Vector, b: <Avx512 as SimdKernel<Bf8>>::Vector, ) -> <Avx512 as SimdKernel<Bf8>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<Bf8>>::Vector, b: <Avx512 as SimdKernel<Bf8>>::Vector, ) -> <Avx512 as SimdKernel<Bf8>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<Bf8>>::Vector, ) -> <Avx512 as SimdKernel<Bf8>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<Bf8>>::Vector, b: <Avx512 as SimdKernel<Bf8>>::Vector, c: <Avx512 as SimdKernel<Bf8>>::Vector, ) -> <Avx512 as SimdKernel<Bf8>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<Bf8>>::Vector) -> Bf8

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<Bf8>>::Vector, mask: <Avx512 as SimdKernel<Bf8>>::Mask, ) -> <Avx512 as SimdKernel<Bf8>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<Bf8>>::Vector, mask: <Avx512 as SimdKernel<Bf8>>::Mask, fill: <Avx512 as SimdKernel<Bf8>>::Vector, ) -> <Avx512 as SimdKernel<Bf8>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const Bf8, indices: <Avx512 as SimdKernel<Bf8>>::IndexVector, ) -> <Avx512 as SimdKernel<Bf8>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const Bf8, indices: <Avx512 as SimdKernel<Bf8>>::IndexVector, mask: <Avx512 as SimdKernel<Bf8>>::Mask, src: <Avx512 as SimdKernel<Bf8>>::Vector, ) -> <Avx512 as SimdKernel<Bf8>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<Bf8>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<Bf8>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<Bf8>>::Vector

Set all lanes to zero.

unsafe fn splat(val: Bf8) -> <Avx512 as SimdKernel<Bf8>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<Bf8>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<Bf8>>::Mask, ) -> <Avx512 as SimdKernel<Bf8>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<Bf8>>::Vector, ) -> <Avx512 as SimdKernel<Bf8>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<Bf16> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 32

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [Bf16; 32]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 32]

Hardware-native mask type.

type IndexVector = [i32; 32]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const Bf16) -> <Avx512 as SimdKernel<Bf16>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned( ptr: *const Bf16, ) -> <Avx512 as SimdKernel<Bf16>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned( ptr: *mut Bf16, val: <Avx512 as SimdKernel<Bf16>>::Vector, )

Store a vector to an aligned pointer.

unsafe fn store_unaligned( ptr: *mut Bf16, val: <Avx512 as SimdKernel<Bf16>>::Vector, )

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<Bf16>>::Vector, b: <Avx512 as SimdKernel<Bf16>>::Vector, ) -> <Avx512 as SimdKernel<Bf16>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<Bf16>>::Vector, b: <Avx512 as SimdKernel<Bf16>>::Vector, ) -> <Avx512 as SimdKernel<Bf16>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<Bf16>>::Vector, b: <Avx512 as SimdKernel<Bf16>>::Vector, ) -> <Avx512 as SimdKernel<Bf16>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<Bf16>>::Vector, ) -> <Avx512 as SimdKernel<Bf16>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<Bf16>>::Vector, b: <Avx512 as SimdKernel<Bf16>>::Vector, c: <Avx512 as SimdKernel<Bf16>>::Vector, ) -> <Avx512 as SimdKernel<Bf16>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<Bf16>>::Vector) -> Bf16

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<Bf16>>::Vector, mask: <Avx512 as SimdKernel<Bf16>>::Mask, ) -> <Avx512 as SimdKernel<Bf16>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<Bf16>>::Vector, mask: <Avx512 as SimdKernel<Bf16>>::Mask, fill: <Avx512 as SimdKernel<Bf16>>::Vector, ) -> <Avx512 as SimdKernel<Bf16>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const Bf16, indices: <Avx512 as SimdKernel<Bf16>>::IndexVector, ) -> <Avx512 as SimdKernel<Bf16>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const Bf16, indices: <Avx512 as SimdKernel<Bf16>>::IndexVector, mask: <Avx512 as SimdKernel<Bf16>>::Mask, src: <Avx512 as SimdKernel<Bf16>>::Vector, ) -> <Avx512 as SimdKernel<Bf16>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<Bf16>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<Bf16>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<Bf16>>::Vector

Set all lanes to zero.

unsafe fn splat(val: Bf16) -> <Avx512 as SimdKernel<Bf16>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<Bf16>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<Bf16>>::Mask, ) -> <Avx512 as SimdKernel<Bf16>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<Bf16>>::Vector, ) -> <Avx512 as SimdKernel<Bf16>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<F4> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 64

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [F4; 64]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 64]

Hardware-native mask type.

type IndexVector = [i32; 64]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const F4) -> <Avx512 as SimdKernel<F4>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const F4) -> <Avx512 as SimdKernel<F4>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut F4, val: <Avx512 as SimdKernel<F4>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned(ptr: *mut F4, val: <Avx512 as SimdKernel<F4>>::Vector)

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<F4>>::Vector, b: <Avx512 as SimdKernel<F4>>::Vector, ) -> <Avx512 as SimdKernel<F4>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<F4>>::Vector, b: <Avx512 as SimdKernel<F4>>::Vector, ) -> <Avx512 as SimdKernel<F4>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<F4>>::Vector, b: <Avx512 as SimdKernel<F4>>::Vector, ) -> <Avx512 as SimdKernel<F4>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<F4>>::Vector, ) -> <Avx512 as SimdKernel<F4>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<F4>>::Vector, b: <Avx512 as SimdKernel<F4>>::Vector, c: <Avx512 as SimdKernel<F4>>::Vector, ) -> <Avx512 as SimdKernel<F4>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<F4>>::Vector) -> F4

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<F4>>::Vector, mask: <Avx512 as SimdKernel<F4>>::Mask, ) -> <Avx512 as SimdKernel<F4>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<F4>>::Vector, mask: <Avx512 as SimdKernel<F4>>::Mask, fill: <Avx512 as SimdKernel<F4>>::Vector, ) -> <Avx512 as SimdKernel<F4>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const F4, indices: <Avx512 as SimdKernel<F4>>::IndexVector, ) -> <Avx512 as SimdKernel<F4>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const F4, indices: <Avx512 as SimdKernel<F4>>::IndexVector, mask: <Avx512 as SimdKernel<F4>>::Mask, src: <Avx512 as SimdKernel<F4>>::Vector, ) -> <Avx512 as SimdKernel<F4>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<F4>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<F4>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<F4>>::Vector

Set all lanes to zero.

unsafe fn splat(val: F4) -> <Avx512 as SimdKernel<F4>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<F4>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<F4>>::Mask, ) -> <Avx512 as SimdKernel<F4>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<F4>>::Vector, ) -> <Avx512 as SimdKernel<F4>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<F8> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 64

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [F8; 64]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 64]

Hardware-native mask type.

type IndexVector = [i32; 64]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const F8) -> <Avx512 as SimdKernel<F8>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const F8) -> <Avx512 as SimdKernel<F8>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut F8, val: <Avx512 as SimdKernel<F8>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned(ptr: *mut F8, val: <Avx512 as SimdKernel<F8>>::Vector)

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<F8>>::Vector, b: <Avx512 as SimdKernel<F8>>::Vector, ) -> <Avx512 as SimdKernel<F8>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<F8>>::Vector, b: <Avx512 as SimdKernel<F8>>::Vector, ) -> <Avx512 as SimdKernel<F8>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<F8>>::Vector, b: <Avx512 as SimdKernel<F8>>::Vector, ) -> <Avx512 as SimdKernel<F8>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<F8>>::Vector, ) -> <Avx512 as SimdKernel<F8>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<F8>>::Vector, b: <Avx512 as SimdKernel<F8>>::Vector, c: <Avx512 as SimdKernel<F8>>::Vector, ) -> <Avx512 as SimdKernel<F8>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<F8>>::Vector) -> F8

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<F8>>::Vector, mask: <Avx512 as SimdKernel<F8>>::Mask, ) -> <Avx512 as SimdKernel<F8>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<F8>>::Vector, mask: <Avx512 as SimdKernel<F8>>::Mask, fill: <Avx512 as SimdKernel<F8>>::Vector, ) -> <Avx512 as SimdKernel<F8>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const F8, indices: <Avx512 as SimdKernel<F8>>::IndexVector, ) -> <Avx512 as SimdKernel<F8>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const F8, indices: <Avx512 as SimdKernel<F8>>::IndexVector, mask: <Avx512 as SimdKernel<F8>>::Mask, src: <Avx512 as SimdKernel<F8>>::Vector, ) -> <Avx512 as SimdKernel<F8>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<F8>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<F8>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<F8>>::Vector

Set all lanes to zero.

unsafe fn splat(val: F8) -> <Avx512 as SimdKernel<F8>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<F8>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<F8>>::Mask, ) -> <Avx512 as SimdKernel<F8>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<F8>>::Vector, ) -> <Avx512 as SimdKernel<F8>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<F16> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 32

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [F16; 32]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 32]

Hardware-native mask type.

type IndexVector = [i32; 32]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const F16) -> <Avx512 as SimdKernel<F16>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const F16) -> <Avx512 as SimdKernel<F16>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut F16, val: <Avx512 as SimdKernel<F16>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned( ptr: *mut F16, val: <Avx512 as SimdKernel<F16>>::Vector, )

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<F16>>::Vector, b: <Avx512 as SimdKernel<F16>>::Vector, ) -> <Avx512 as SimdKernel<F16>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<F16>>::Vector, b: <Avx512 as SimdKernel<F16>>::Vector, ) -> <Avx512 as SimdKernel<F16>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<F16>>::Vector, b: <Avx512 as SimdKernel<F16>>::Vector, ) -> <Avx512 as SimdKernel<F16>>::Vector

Elementwise subtraction: a - b.

unsafe fn fmadd( a: <Avx512 as SimdKernel<F16>>::Vector, b: <Avx512 as SimdKernel<F16>>::Vector, c: <Avx512 as SimdKernel<F16>>::Vector, ) -> <Avx512 as SimdKernel<F16>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<F16>>::Vector) -> F16

Horizontal sum of all lanes.

unsafe fn masked_load_unaligned( ptr: *const F16, mask: <Avx512 as SimdKernel<F16>>::Mask, src: <Avx512 as SimdKernel<F16>>::Vector, ) -> <Avx512 as SimdKernel<F16>>::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut F16, mask: <Avx512 as SimdKernel<F16>>::Mask, val: <Avx512 as SimdKernel<F16>>::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: <Avx512 as SimdKernel<F16>>::Vector, b: <Avx512 as SimdKernel<F16>>::Vector, mask: <Avx512 as SimdKernel<F16>>::Mask, src: <Avx512 as SimdKernel<F16>>::Vector, ) -> <Avx512 as SimdKernel<F16>>::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: <Avx512 as SimdKernel<F16>>::Vector, b: <Avx512 as SimdKernel<F16>>::Vector, mask: <Avx512 as SimdKernel<F16>>::Mask, src: <Avx512 as SimdKernel<F16>>::Vector, ) -> <Avx512 as SimdKernel<F16>>::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: <Avx512 as SimdKernel<F16>>::Vector, b: <Avx512 as SimdKernel<F16>>::Vector, c: <Avx512 as SimdKernel<F16>>::Vector, mask: <Avx512 as SimdKernel<F16>>::Mask, ) -> <Avx512 as SimdKernel<F16>>::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce( v: <Avx512 as SimdKernel<F16>>::Vector, mask: <Avx512 as SimdKernel<F16>>::Mask, ) -> F16

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn compress( src: <Avx512 as SimdKernel<F16>>::Vector, mask: <Avx512 as SimdKernel<F16>>::Mask, ) -> <Avx512 as SimdKernel<F16>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<F16>>::Vector, mask: <Avx512 as SimdKernel<F16>>::Mask, fill: <Avx512 as SimdKernel<F16>>::Vector, ) -> <Avx512 as SimdKernel<F16>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const F16, indices: <Avx512 as SimdKernel<F16>>::IndexVector, ) -> <Avx512 as SimdKernel<F16>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const F16, indices: <Avx512 as SimdKernel<F16>>::IndexVector, mask: <Avx512 as SimdKernel<F16>>::Mask, src: <Avx512 as SimdKernel<F16>>::Vector, ) -> <Avx512 as SimdKernel<F16>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<F16>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<F16>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<F16>>::Vector

Set all lanes to zero.

unsafe fn splat(val: F16) -> <Avx512 as SimdKernel<F16>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<F16>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<F16>>::Mask, ) -> <Avx512 as SimdKernel<F16>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<F16>>::Vector, ) -> <Avx512 as SimdKernel<F16>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn neg(a: Self::Vector) -> Self::Vector

Elementwise negate: -a.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<F32> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 16

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [F32; 16]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 16]

Hardware-native mask type.

type IndexVector = [i32; 16]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const F32) -> <Avx512 as SimdKernel<F32>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const F32) -> <Avx512 as SimdKernel<F32>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut F32, val: <Avx512 as SimdKernel<F32>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned( ptr: *mut F32, val: <Avx512 as SimdKernel<F32>>::Vector, )

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<F32>>::Vector, b: <Avx512 as SimdKernel<F32>>::Vector, ) -> <Avx512 as SimdKernel<F32>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<F32>>::Vector, b: <Avx512 as SimdKernel<F32>>::Vector, ) -> <Avx512 as SimdKernel<F32>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<F32>>::Vector, b: <Avx512 as SimdKernel<F32>>::Vector, ) -> <Avx512 as SimdKernel<F32>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<F32>>::Vector, ) -> <Avx512 as SimdKernel<F32>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<F32>>::Vector, b: <Avx512 as SimdKernel<F32>>::Vector, c: <Avx512 as SimdKernel<F32>>::Vector, ) -> <Avx512 as SimdKernel<F32>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<F32>>::Vector) -> F32

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<F32>>::Vector, mask: <Avx512 as SimdKernel<F32>>::Mask, ) -> <Avx512 as SimdKernel<F32>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<F32>>::Vector, mask: <Avx512 as SimdKernel<F32>>::Mask, fill: <Avx512 as SimdKernel<F32>>::Vector, ) -> <Avx512 as SimdKernel<F32>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const F32, indices: <Avx512 as SimdKernel<F32>>::IndexVector, ) -> <Avx512 as SimdKernel<F32>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const F32, indices: <Avx512 as SimdKernel<F32>>::IndexVector, mask: <Avx512 as SimdKernel<F32>>::Mask, src: <Avx512 as SimdKernel<F32>>::Vector, ) -> <Avx512 as SimdKernel<F32>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<F32>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<F32>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<F32>>::Vector

Set all lanes to zero.

unsafe fn splat(val: F32) -> <Avx512 as SimdKernel<F32>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<F32>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<F32>>::Mask, ) -> <Avx512 as SimdKernel<F32>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<F32>>::Vector, ) -> <Avx512 as SimdKernel<F32>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<F64> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 8

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [F64; 8]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 8]

Hardware-native mask type.

type IndexVector = [i32; 8]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const F64) -> <Avx512 as SimdKernel<F64>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const F64) -> <Avx512 as SimdKernel<F64>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut F64, val: <Avx512 as SimdKernel<F64>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned( ptr: *mut F64, val: <Avx512 as SimdKernel<F64>>::Vector, )

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<F64>>::Vector, b: <Avx512 as SimdKernel<F64>>::Vector, ) -> <Avx512 as SimdKernel<F64>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<F64>>::Vector, b: <Avx512 as SimdKernel<F64>>::Vector, ) -> <Avx512 as SimdKernel<F64>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<F64>>::Vector, b: <Avx512 as SimdKernel<F64>>::Vector, ) -> <Avx512 as SimdKernel<F64>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<F64>>::Vector, ) -> <Avx512 as SimdKernel<F64>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<F64>>::Vector, b: <Avx512 as SimdKernel<F64>>::Vector, c: <Avx512 as SimdKernel<F64>>::Vector, ) -> <Avx512 as SimdKernel<F64>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<F64>>::Vector) -> F64

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<F64>>::Vector, mask: <Avx512 as SimdKernel<F64>>::Mask, ) -> <Avx512 as SimdKernel<F64>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<F64>>::Vector, mask: <Avx512 as SimdKernel<F64>>::Mask, fill: <Avx512 as SimdKernel<F64>>::Vector, ) -> <Avx512 as SimdKernel<F64>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const F64, indices: <Avx512 as SimdKernel<F64>>::IndexVector, ) -> <Avx512 as SimdKernel<F64>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const F64, indices: <Avx512 as SimdKernel<F64>>::IndexVector, mask: <Avx512 as SimdKernel<F64>>::Mask, src: <Avx512 as SimdKernel<F64>>::Vector, ) -> <Avx512 as SimdKernel<F64>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<F64>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<F64>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<F64>>::Vector

Set all lanes to zero.

unsafe fn splat(val: F64) -> <Avx512 as SimdKernel<F64>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<F64>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<F64>>::Mask, ) -> <Avx512 as SimdKernel<F64>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<F64>>::Vector, ) -> <Avx512 as SimdKernel<F64>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<I8> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 64

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [I8; 64]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 64]

Hardware-native mask type.

type IndexVector = [i32; 64]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const I8) -> <Avx512 as SimdKernel<I8>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const I8) -> <Avx512 as SimdKernel<I8>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut I8, val: <Avx512 as SimdKernel<I8>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned(ptr: *mut I8, val: <Avx512 as SimdKernel<I8>>::Vector)

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<I8>>::Vector, b: <Avx512 as SimdKernel<I8>>::Vector, ) -> <Avx512 as SimdKernel<I8>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<I8>>::Vector, b: <Avx512 as SimdKernel<I8>>::Vector, ) -> <Avx512 as SimdKernel<I8>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<I8>>::Vector, b: <Avx512 as SimdKernel<I8>>::Vector, ) -> <Avx512 as SimdKernel<I8>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<I8>>::Vector, ) -> <Avx512 as SimdKernel<I8>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<I8>>::Vector, b: <Avx512 as SimdKernel<I8>>::Vector, c: <Avx512 as SimdKernel<I8>>::Vector, ) -> <Avx512 as SimdKernel<I8>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<I8>>::Vector) -> I8

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<I8>>::Vector, mask: <Avx512 as SimdKernel<I8>>::Mask, ) -> <Avx512 as SimdKernel<I8>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<I8>>::Vector, mask: <Avx512 as SimdKernel<I8>>::Mask, fill: <Avx512 as SimdKernel<I8>>::Vector, ) -> <Avx512 as SimdKernel<I8>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const I8, indices: <Avx512 as SimdKernel<I8>>::IndexVector, ) -> <Avx512 as SimdKernel<I8>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const I8, indices: <Avx512 as SimdKernel<I8>>::IndexVector, mask: <Avx512 as SimdKernel<I8>>::Mask, src: <Avx512 as SimdKernel<I8>>::Vector, ) -> <Avx512 as SimdKernel<I8>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<I8>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<I8>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<I8>>::Vector

Set all lanes to zero.

unsafe fn splat(val: I8) -> <Avx512 as SimdKernel<I8>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<I8>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<I8>>::Mask, ) -> <Avx512 as SimdKernel<I8>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<I8>>::Vector, ) -> <Avx512 as SimdKernel<I8>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<I16> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 32

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [I16; 32]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 32]

Hardware-native mask type.

type IndexVector = [i32; 32]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const I16) -> <Avx512 as SimdKernel<I16>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const I16) -> <Avx512 as SimdKernel<I16>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut I16, val: <Avx512 as SimdKernel<I16>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned( ptr: *mut I16, val: <Avx512 as SimdKernel<I16>>::Vector, )

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<I16>>::Vector, b: <Avx512 as SimdKernel<I16>>::Vector, ) -> <Avx512 as SimdKernel<I16>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<I16>>::Vector, b: <Avx512 as SimdKernel<I16>>::Vector, ) -> <Avx512 as SimdKernel<I16>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<I16>>::Vector, b: <Avx512 as SimdKernel<I16>>::Vector, ) -> <Avx512 as SimdKernel<I16>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<I16>>::Vector, ) -> <Avx512 as SimdKernel<I16>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<I16>>::Vector, b: <Avx512 as SimdKernel<I16>>::Vector, c: <Avx512 as SimdKernel<I16>>::Vector, ) -> <Avx512 as SimdKernel<I16>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<I16>>::Vector) -> I16

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<I16>>::Vector, mask: <Avx512 as SimdKernel<I16>>::Mask, ) -> <Avx512 as SimdKernel<I16>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<I16>>::Vector, mask: <Avx512 as SimdKernel<I16>>::Mask, fill: <Avx512 as SimdKernel<I16>>::Vector, ) -> <Avx512 as SimdKernel<I16>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const I16, indices: <Avx512 as SimdKernel<I16>>::IndexVector, ) -> <Avx512 as SimdKernel<I16>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const I16, indices: <Avx512 as SimdKernel<I16>>::IndexVector, mask: <Avx512 as SimdKernel<I16>>::Mask, src: <Avx512 as SimdKernel<I16>>::Vector, ) -> <Avx512 as SimdKernel<I16>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<I16>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<I16>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<I16>>::Vector

Set all lanes to zero.

unsafe fn splat(val: I16) -> <Avx512 as SimdKernel<I16>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<I16>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<I16>>::Mask, ) -> <Avx512 as SimdKernel<I16>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<I16>>::Vector, ) -> <Avx512 as SimdKernel<I16>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<I32> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 16

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [I32; 16]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 16]

Hardware-native mask type.

type IndexVector = [i32; 16]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const I32) -> <Avx512 as SimdKernel<I32>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const I32) -> <Avx512 as SimdKernel<I32>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut I32, val: <Avx512 as SimdKernel<I32>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned( ptr: *mut I32, val: <Avx512 as SimdKernel<I32>>::Vector, )

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<I32>>::Vector, b: <Avx512 as SimdKernel<I32>>::Vector, ) -> <Avx512 as SimdKernel<I32>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<I32>>::Vector, b: <Avx512 as SimdKernel<I32>>::Vector, ) -> <Avx512 as SimdKernel<I32>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<I32>>::Vector, b: <Avx512 as SimdKernel<I32>>::Vector, ) -> <Avx512 as SimdKernel<I32>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<I32>>::Vector, ) -> <Avx512 as SimdKernel<I32>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<I32>>::Vector, b: <Avx512 as SimdKernel<I32>>::Vector, c: <Avx512 as SimdKernel<I32>>::Vector, ) -> <Avx512 as SimdKernel<I32>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<I32>>::Vector) -> I32

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<I32>>::Vector, mask: <Avx512 as SimdKernel<I32>>::Mask, ) -> <Avx512 as SimdKernel<I32>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<I32>>::Vector, mask: <Avx512 as SimdKernel<I32>>::Mask, fill: <Avx512 as SimdKernel<I32>>::Vector, ) -> <Avx512 as SimdKernel<I32>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const I32, indices: <Avx512 as SimdKernel<I32>>::IndexVector, ) -> <Avx512 as SimdKernel<I32>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const I32, indices: <Avx512 as SimdKernel<I32>>::IndexVector, mask: <Avx512 as SimdKernel<I32>>::Mask, src: <Avx512 as SimdKernel<I32>>::Vector, ) -> <Avx512 as SimdKernel<I32>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<I32>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<I32>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<I32>>::Vector

Set all lanes to zero.

unsafe fn splat(val: I32) -> <Avx512 as SimdKernel<I32>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<I32>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<I32>>::Mask, ) -> <Avx512 as SimdKernel<I32>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<I32>>::Vector, ) -> <Avx512 as SimdKernel<I32>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<f32> for Avx512

Available on x86 or x86-64 only.

type Mask = u16

Native AVX-512 16-bit mask register. Bit i set → lane i active.

type IndexVector = __m512i

16 × i32 index vector for gather (__m512i).

unsafe fn masked_fmadd( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, c: <Avx512 as SimdKernel<f32>>::Vector, mask: <Avx512 as SimdKernel<f32>>::Mask, ) -> <Avx512 as SimdKernel<f32>>::Vector

mask3_fmadd: inactive lanes retain c (the addend / 3rd operand).

const LANE_COUNT: usize = 16

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

const SUPPORTS_NT_STORE: bool = true

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

type Vector = Avx512F32Vec

The underlying raw register/vector type for this architecture and element type.

unsafe fn load_aligned(ptr: *const f32) -> <Avx512 as SimdKernel<f32>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const f32) -> <Avx512 as SimdKernel<f32>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut f32, val: <Avx512 as SimdKernel<f32>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned( ptr: *mut f32, val: <Avx512 as SimdKernel<f32>>::Vector, )

Store a vector to an unaligned pointer.

unsafe fn store_streaming( ptr: *mut f32, val: <Avx512 as SimdKernel<f32>>::Vector, )

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn add( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise subtraction: a - b.

unsafe fn fmadd( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, c: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn swap_adjacent( v: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even( v: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd( v: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, c: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, c: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<f32>>::Vector) -> f32

Horizontal sum of all lanes.

unsafe fn masked_load_unaligned( ptr: *const f32, mask: <Avx512 as SimdKernel<f32>>::Mask, src: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut f32, mask: <Avx512 as SimdKernel<f32>>::Mask, val: <Avx512 as SimdKernel<f32>>::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, mask: <Avx512 as SimdKernel<f32>>::Mask, src: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, mask: <Avx512 as SimdKernel<f32>>::Mask, src: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_sum_reduce( v: <Avx512 as SimdKernel<f32>>::Vector, mask: <Avx512 as SimdKernel<f32>>::Mask, ) -> f32

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn compress( src: <Avx512 as SimdKernel<f32>>::Vector, mask: <Avx512 as SimdKernel<f32>>::Mask, ) -> <Avx512 as SimdKernel<f32>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<f32>>::Vector, mask: <Avx512 as SimdKernel<f32>>::Mask, fill: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const f32, indices: <Avx512 as SimdKernel<f32>>::IndexVector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const f32, indices: <Avx512 as SimdKernel<f32>>::IndexVector, mask: <Avx512 as SimdKernel<f32>>::Mask, src: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<f32>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<f32>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<f32>>::Vector

Set all lanes to zero.

unsafe fn splat(val: f32) -> <Avx512 as SimdKernel<f32>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn div( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise division: a / b.

unsafe fn bitand( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs( a: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise absolute value.

unsafe fn min( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise minimum of a and b.

unsafe fn max( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise maximum of a and b.

unsafe fn sqrt( a: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise square root.

unsafe fn recip_sqrt( a: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn popcount( a: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise population count (number of set bits).

unsafe fn cmp_eq( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise less than: a < b.

unsafe fn cmp_le( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge( a: <Avx512 as SimdKernel<f32>>::Vector, b: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: <Avx512 as SimdKernel<f32>>::Vector, true_val: <Avx512 as SimdKernel<f32>>::Vector, false_val: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<f32>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<f32>>::Mask, ) -> <Avx512 as SimdKernel<f32>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<f32>>::Vector, ) -> <Avx512 as SimdKernel<f32>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn neg(a: Self::Vector) -> Self::Vector

Elementwise negate: -a.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

impl SimdKernel<f64> for Avx512

Available on x86 or x86-64 only.

type Mask = u8

Native AVX-512 8-bit mask register. Bit i set → lane i active.

type IndexVector = __m256i

8 × i32 index vector for gather (__m256i).

unsafe fn masked_fmadd( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, c: <Avx512 as SimdKernel<f64>>::Vector, mask: <Avx512 as SimdKernel<f64>>::Mask, ) -> <Avx512 as SimdKernel<f64>>::Vector

mask3_fmadd: inactive lanes retain c (the addend / 3rd operand).

const LANE_COUNT: usize = 8

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

const SUPPORTS_NT_STORE: bool = true

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

type Vector = Avx512F64Vec

The underlying raw register/vector type for this architecture and element type.

unsafe fn load_aligned(ptr: *const f64) -> <Avx512 as SimdKernel<f64>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const f64) -> <Avx512 as SimdKernel<f64>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut f64, val: <Avx512 as SimdKernel<f64>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned( ptr: *mut f64, val: <Avx512 as SimdKernel<f64>>::Vector, )

Store a vector to an unaligned pointer.

unsafe fn store_streaming( ptr: *mut f64, val: <Avx512 as SimdKernel<f64>>::Vector, )

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn add( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise subtraction: a - b.

unsafe fn fmadd( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, c: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn swap_adjacent( v: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even( v: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd( v: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, c: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, c: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<f64>>::Vector) -> f64

Horizontal sum of all lanes.

unsafe fn masked_load_unaligned( ptr: *const f64, mask: <Avx512 as SimdKernel<f64>>::Mask, src: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut f64, mask: <Avx512 as SimdKernel<f64>>::Mask, val: <Avx512 as SimdKernel<f64>>::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, mask: <Avx512 as SimdKernel<f64>>::Mask, src: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, mask: <Avx512 as SimdKernel<f64>>::Mask, src: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_sum_reduce( v: <Avx512 as SimdKernel<f64>>::Vector, mask: <Avx512 as SimdKernel<f64>>::Mask, ) -> f64

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn compress( src: <Avx512 as SimdKernel<f64>>::Vector, mask: <Avx512 as SimdKernel<f64>>::Mask, ) -> <Avx512 as SimdKernel<f64>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<f64>>::Vector, mask: <Avx512 as SimdKernel<f64>>::Mask, fill: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const f64, indices: <Avx512 as SimdKernel<f64>>::IndexVector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const f64, indices: <Avx512 as SimdKernel<f64>>::IndexVector, mask: <Avx512 as SimdKernel<f64>>::Mask, src: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<f64>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<f64>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<f64>>::Vector

Set all lanes to zero.

unsafe fn splat(val: f64) -> <Avx512 as SimdKernel<f64>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn div( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise division: a / b.

unsafe fn bitand( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs( a: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise absolute value.

unsafe fn min( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise minimum of a and b.

unsafe fn max( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise maximum of a and b.

unsafe fn sqrt( a: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise square root.

unsafe fn recip_sqrt( a: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn popcount( a: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise population count (number of set bits).

unsafe fn cmp_eq( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise less than: a < b.

unsafe fn cmp_le( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge( a: <Avx512 as SimdKernel<f64>>::Vector, b: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: <Avx512 as SimdKernel<f64>>::Vector, true_val: <Avx512 as SimdKernel<f64>>::Vector, false_val: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<f64>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<f64>>::Mask, ) -> <Avx512 as SimdKernel<f64>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<f64>>::Vector, ) -> <Avx512 as SimdKernel<f64>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn neg(a: Self::Vector) -> Self::Vector

Elementwise negate: -a.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

impl SimdKernel<i8> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 64

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [i8; 64]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 64]

Hardware-native mask type.

type IndexVector = [i32; 64]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const i8) -> <Avx512 as SimdKernel<i8>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const i8) -> <Avx512 as SimdKernel<i8>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut i8, val: <Avx512 as SimdKernel<i8>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned(ptr: *mut i8, val: <Avx512 as SimdKernel<i8>>::Vector)

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<i8>>::Vector, b: <Avx512 as SimdKernel<i8>>::Vector, ) -> <Avx512 as SimdKernel<i8>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<i8>>::Vector, b: <Avx512 as SimdKernel<i8>>::Vector, ) -> <Avx512 as SimdKernel<i8>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<i8>>::Vector, b: <Avx512 as SimdKernel<i8>>::Vector, ) -> <Avx512 as SimdKernel<i8>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<i8>>::Vector, ) -> <Avx512 as SimdKernel<i8>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<i8>>::Vector, b: <Avx512 as SimdKernel<i8>>::Vector, c: <Avx512 as SimdKernel<i8>>::Vector, ) -> <Avx512 as SimdKernel<i8>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<i8>>::Vector) -> i8

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<i8>>::Vector, mask: <Avx512 as SimdKernel<i8>>::Mask, ) -> <Avx512 as SimdKernel<i8>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<i8>>::Vector, mask: <Avx512 as SimdKernel<i8>>::Mask, fill: <Avx512 as SimdKernel<i8>>::Vector, ) -> <Avx512 as SimdKernel<i8>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const i8, indices: <Avx512 as SimdKernel<i8>>::IndexVector, ) -> <Avx512 as SimdKernel<i8>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const i8, indices: <Avx512 as SimdKernel<i8>>::IndexVector, mask: <Avx512 as SimdKernel<i8>>::Mask, src: <Avx512 as SimdKernel<i8>>::Vector, ) -> <Avx512 as SimdKernel<i8>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<i8>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<i8>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<i8>>::Vector

Set all lanes to zero.

unsafe fn splat(val: i8) -> <Avx512 as SimdKernel<i8>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<i8>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<i8>>::Mask, ) -> <Avx512 as SimdKernel<i8>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<i8>>::Vector, ) -> <Avx512 as SimdKernel<i8>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<i16> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 32

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [i16; 32]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 32]

Hardware-native mask type.

type IndexVector = [i32; 32]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const i16) -> <Avx512 as SimdKernel<i16>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const i16) -> <Avx512 as SimdKernel<i16>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut i16, val: <Avx512 as SimdKernel<i16>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned( ptr: *mut i16, val: <Avx512 as SimdKernel<i16>>::Vector, )

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<i16>>::Vector, b: <Avx512 as SimdKernel<i16>>::Vector, ) -> <Avx512 as SimdKernel<i16>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<i16>>::Vector, b: <Avx512 as SimdKernel<i16>>::Vector, ) -> <Avx512 as SimdKernel<i16>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<i16>>::Vector, b: <Avx512 as SimdKernel<i16>>::Vector, ) -> <Avx512 as SimdKernel<i16>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<i16>>::Vector, ) -> <Avx512 as SimdKernel<i16>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<i16>>::Vector, b: <Avx512 as SimdKernel<i16>>::Vector, c: <Avx512 as SimdKernel<i16>>::Vector, ) -> <Avx512 as SimdKernel<i16>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<i16>>::Vector) -> i16

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<i16>>::Vector, mask: <Avx512 as SimdKernel<i16>>::Mask, ) -> <Avx512 as SimdKernel<i16>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<i16>>::Vector, mask: <Avx512 as SimdKernel<i16>>::Mask, fill: <Avx512 as SimdKernel<i16>>::Vector, ) -> <Avx512 as SimdKernel<i16>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const i16, indices: <Avx512 as SimdKernel<i16>>::IndexVector, ) -> <Avx512 as SimdKernel<i16>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const i16, indices: <Avx512 as SimdKernel<i16>>::IndexVector, mask: <Avx512 as SimdKernel<i16>>::Mask, src: <Avx512 as SimdKernel<i16>>::Vector, ) -> <Avx512 as SimdKernel<i16>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<i16>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<i16>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<i16>>::Vector

Set all lanes to zero.

unsafe fn splat(val: i16) -> <Avx512 as SimdKernel<i16>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<i16>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<i16>>::Mask, ) -> <Avx512 as SimdKernel<i16>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<i16>>::Vector, ) -> <Avx512 as SimdKernel<i16>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl SimdKernel<i32> for Avx512

Available on x86 or x86-64 only.

const LANE_COUNT: usize = 16

Number of primitive elements of type T in one Vector.

const UNROLL_FACTOR: usize = 4

Loop unrolling register accumulation factor to break loop-carried dependency chains.

type Vector = [i32; 16]

The underlying raw register/vector type for this architecture and element type.

type Mask = [bool; 16]

Hardware-native mask type.

type IndexVector = [i32; 16]

Integer index vector for gather operations.

unsafe fn load_aligned(ptr: *const i32) -> <Avx512 as SimdKernel<i32>>::Vector

Load a vector from an aligned pointer.

unsafe fn load_unaligned(ptr: *const i32) -> <Avx512 as SimdKernel<i32>>::Vector

Load a vector from an unaligned pointer.

unsafe fn store_aligned(ptr: *mut i32, val: <Avx512 as SimdKernel<i32>>::Vector)

Store a vector to an aligned pointer.

unsafe fn store_unaligned( ptr: *mut i32, val: <Avx512 as SimdKernel<i32>>::Vector, )

Store a vector to an unaligned pointer.

unsafe fn add( a: <Avx512 as SimdKernel<i32>>::Vector, b: <Avx512 as SimdKernel<i32>>::Vector, ) -> <Avx512 as SimdKernel<i32>>::Vector

Elementwise addition: a + b.

unsafe fn mul( a: <Avx512 as SimdKernel<i32>>::Vector, b: <Avx512 as SimdKernel<i32>>::Vector, ) -> <Avx512 as SimdKernel<i32>>::Vector

Elementwise multiplication: a * b.

unsafe fn sub( a: <Avx512 as SimdKernel<i32>>::Vector, b: <Avx512 as SimdKernel<i32>>::Vector, ) -> <Avx512 as SimdKernel<i32>>::Vector

Elementwise subtraction: a - b.

unsafe fn neg( a: <Avx512 as SimdKernel<i32>>::Vector, ) -> <Avx512 as SimdKernel<i32>>::Vector

Elementwise negate: -a.

unsafe fn fmadd( a: <Avx512 as SimdKernel<i32>>::Vector, b: <Avx512 as SimdKernel<i32>>::Vector, c: <Avx512 as SimdKernel<i32>>::Vector, ) -> <Avx512 as SimdKernel<i32>>::Vector

Fused multiply-add: (a * b) + c.

unsafe fn sum_reduce(v: <Avx512 as SimdKernel<i32>>::Vector) -> i32

Horizontal sum of all lanes.

unsafe fn compress( src: <Avx512 as SimdKernel<i32>>::Vector, mask: <Avx512 as SimdKernel<i32>>::Mask, ) -> <Avx512 as SimdKernel<i32>>::Vector

Compress: pack selected lanes (where mask[i]=1) into the low lanes of the result.

unsafe fn expand( src: <Avx512 as SimdKernel<i32>>::Vector, mask: <Avx512 as SimdKernel<i32>>::Mask, fill: <Avx512 as SimdKernel<i32>>::Vector, ) -> <Avx512 as SimdKernel<i32>>::Vector

Expand: scatter the low lanes of src into result positions where mask[i]=1.

unsafe fn gather( base: *const i32, indices: <Avx512 as SimdKernel<i32>>::IndexVector, ) -> <Avx512 as SimdKernel<i32>>::Vector

Gather: load LANE_COUNT elements at base + indices[i] for each lane i.

unsafe fn gather_masked( base: *const i32, indices: <Avx512 as SimdKernel<i32>>::IndexVector, mask: <Avx512 as SimdKernel<i32>>::Mask, src: <Avx512 as SimdKernel<i32>>::Vector, ) -> <Avx512 as SimdKernel<i32>>::Vector

Masked gather: gather active lanes; inactive lanes take value from src.

unsafe fn mask_from_bools(bits: &[bool]) -> <Avx512 as SimdKernel<i32>>::Mask

Construct a mask from a slice of booleans (length must equal LANE_COUNT).

unsafe fn leading_k_mask(k: usize) -> <Avx512 as SimdKernel<i32>>::Mask

Construct a mask with the first k lanes active and the rest inactive.

unsafe fn zero() -> <Avx512 as SimdKernel<i32>>::Vector

Set all lanes to zero.

unsafe fn splat(val: i32) -> <Avx512 as SimdKernel<i32>>::Vector

Broadcast a scalar value to all lanes.

unsafe fn mask_to_bitmask(mask: <Avx512 as SimdKernel<i32>>::Mask) -> u64

Convert the native mask back to a raw u64 bitmask.

unsafe fn mask_to_vector( mask: <Avx512 as SimdKernel<i32>>::Mask, ) -> <Avx512 as SimdKernel<i32>>::Vector

Convert the native mask back to a vector register where active lanes are set to T::ALL_ONES and inactive lanes to T::ZERO.

unsafe fn vector_to_mask( v: <Avx512 as SimdKernel<i32>>::Vector, ) -> <Avx512 as SimdKernel<i32>>::Mask

Convert a comparison-result vector into the native mask, the inverse of SimdKernel::mask_to_vector.

const LANE_BOUND_CHECK: () = _

Compile-time guard that LANE_COUNT fits the fixed MAX_SIMD_LANES scalar-fallback stack buffers. Referencing this const in the buffer-using default methods forces the assertion to be evaluated for each concrete backend at monomorphization, turning a would-be silent stack-buffer overflow into a compile error.

const SUPPORTS_NT_STORE: bool = false

Whether this backend provides a non-temporal (cache-bypassing) store via store_streaming. Backends leaving this false keep the regular store default; callers gate the streaming path on this const so it is a compile-time branch, dead-code-eliminated where unsupported.

unsafe fn store_streaming(ptr: *mut T, val: Self::Vector)

Store a vector with a non-temporal (streaming) hint that bypasses the cache, avoiding the read-for-ownership traffic a normal write-allocate pays for write-only data larger than the last-level cache (measured 1.71× on out-of-LLC AVX2 f32 elementwise writes; see streaming_bench).

fn stream_write_barrier()

Fence ordering this backend’s non-temporal stores before subsequent reads. No-op by default (only meaningful where store_streaming is a weakly ordered non-temporal store).

unsafe fn masked_load_unaligned( ptr: *const T, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked load: active lanes loaded from ptr, inactive lanes taken from src.

unsafe fn masked_store_unaligned( ptr: *mut T, mask: Self::Mask, val: Self::Vector, )

Masked store: active lanes written to ptr, inactive lanes left unchanged.

unsafe fn masked_add( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise add: active lanes compute a + b, inactive lanes yield src.

unsafe fn masked_mul( a: Self::Vector, b: Self::Vector, mask: Self::Mask, src: Self::Vector, ) -> Self::Vector

Masked elementwise multiply: active lanes compute a * b, inactive lanes yield src.

unsafe fn masked_fmadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, mask: Self::Mask, ) -> Self::Vector

Masked fused multiply-add: active lanes compute (a * b) + c, inactive lanes retain c.

unsafe fn masked_sum_reduce(v: Self::Vector, mask: Self::Mask) -> T

Masked horizontal sum: only lanes where mask[i]=1 contribute.

unsafe fn mask_from_bitmask(bm: u64) -> Self::Mask

Convert a raw u64 bitmask to the architecture-native mask type.

unsafe fn scan_vector<Op, SMode>(v: Self::Vector, carry: T) -> (Self::Vector, T)

where Op: ScanOp<T>, SMode: ScanMode,

Perform an intra-vector prefix scan (inclusive or exclusive) of the vector, using the specified ScanOp strategy and starting carry value. Returns the scanned vector and the final carry value.

unsafe fn div(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise division: a / b.

unsafe fn bitand(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise AND: a & b.

unsafe fn bitor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise OR: a | b.

unsafe fn bitxor(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise bitwise XOR: a ^ b.

unsafe fn abs(a: Self::Vector) -> Self::Vector

Elementwise absolute value.

unsafe fn min(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise minimum of a and b.

unsafe fn max(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise maximum of a and b.

unsafe fn sqrt(a: Self::Vector) -> Self::Vector

Elementwise square root.

unsafe fn recip_sqrt(a: Self::Vector) -> Self::Vector

Elementwise reciprocal square root, 1/√x, to full T precision (~1 ulp).

unsafe fn cmp_eq(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise equal: a == b.

unsafe fn cmp_ne(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise not equal: a != b.

unsafe fn cmp_lt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than: a < b.

unsafe fn cmp_le(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise less than or equal: a <= b.

unsafe fn cmp_gt(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than: a > b.

unsafe fn cmp_ge(a: Self::Vector, b: Self::Vector) -> Self::Vector

Elementwise greater than or equal: a >= b.

unsafe fn blend( mask: Self::Vector, true_val: Self::Vector, false_val: Self::Vector, ) -> Self::Vector

Elementwise blend: select lanes from true_val where the sign bit of mask is set, and from false_val otherwise.

unsafe fn bitnot(a: Self::Vector) -> Self::Vector

Elementwise bitwise NOT: !a.

unsafe fn min_reduce(v: Self::Vector) -> T

Horizontal minimum across all lanes.

unsafe fn max_reduce(v: Self::Vector) -> T

Horizontal maximum across all lanes.

unsafe fn popcount(a: Self::Vector) -> Self::Vector

Elementwise population count (number of set bits).

unsafe fn horizontal_bitwise_and(v: Self::Vector) -> T

Horizontal bitwise AND across all lanes.

unsafe fn horizontal_bitwise_or(v: Self::Vector) -> T

Horizontal bitwise OR across all lanes.

unsafe fn horizontal_bitwise_xor(v: Self::Vector) -> T

Horizontal bitwise XOR across all lanes.

unsafe fn swap_adjacent(v: Self::Vector) -> Self::Vector

Swap each adjacent lane pair: [a0, a1, a2, a3, ...] -> [a1, a0, a3, a2, ...].

unsafe fn dup_even(v: Self::Vector) -> Self::Vector

Duplicate even lanes into odd lanes: [a0, a1, a2, a3, ...] -> [a0, a0, a2, a2, ...].

unsafe fn dup_odd(v: Self::Vector) -> Self::Vector

Duplicate odd lanes into even lanes: [a0, a1, a2, a3, ...] -> [a1, a1, a3, a3, ...].

unsafe fn fmaddsub( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b - c, odd lanes a*b + c.

unsafe fn fmsubadd( a: Self::Vector, b: Self::Vector, c: Self::Vector, ) -> Self::Vector

Alternating fused multiply: even lanes a*b + c, odd lanes a*b - c.

impl StructuralPartialEq for Avx512

impl TileMatrixMultiply<Bf16, Bf16, F32, Avx512, Avx512, 16, 16, 32> for Avx512

unsafe fn tile_matmul( c: *mut F32, c_stride: usize, a: *const Bf16, a_stride: usize, b: *const Bf16, b_stride: usize, )

Performs tile matrix multiplication: C += A * B

impl TileMatrixMultiply<I8, I8, I32, Avx512, Avx512, 16, 16, 64> for Avx512

unsafe fn tile_matmul( c: *mut I32, c_stride: usize, a: *const I8, a_stride: usize, b: *const I8, b_stride: usize, )

Performs tile matrix multiplication: C += A * B

impl TileMatrixMultiply<i8, i8, i32, Avx512, Avx512, 16, 16, 64> for Avx512

unsafe fn tile_matmul( c: *mut i32, c_stride: usize, a: *const i8, a_stride: usize, b: *const i8, b_stride: usize, )

Performs tile matrix multiplication: C += A * B