opus1 0.3.0

//! Safe loads and stores of vector registers.
//!
//! The intrinsics of `core::arch` that read or write memory (`vld1q_f32`,
//! `_mm_loadu_ps`, `_mm256_storeu_ps`, ...) take raw pointers, so each call
//! needs an `unsafe` block. The functions here make these calls for the
//! kernels. Each takes a reference to an array of exactly as many values as
//! the instruction reads or writes. The reference already guarantees what
//! the intrinsic needs: the memory is valid for that many values, and for a
//! store no other reference reaches it. The instructions are the unaligned
//! forms. The x86 ones need no alignment at all; the NEON ones need only the
//! alignment of one value, which the reference has. So each function is
//! safe, and its one `unsafe` block is the only one a kernel needs for
//! memory.
//!
//! Each function is compiled with the target feature of its intrinsic. A
//! kernel compiled with that feature, or with one that includes it (AVX2
//! includes AVX, SSE2 and SSE), calls it without `unsafe`, just as it calls
//! the intrinsics that only compute on registers (`vaddq_f32`,
//! `_mm256_fmadd_ps`). The `unsafe` block of a kernel is then the call to
//! the kernel itself, from code without the feature, where the caller
//! states why the processor has the feature.
//!
//! The names say what moves between memory and registers:
//! - `load_` and `store_`, then the type and the number of values read or
//!   written: `load_f32x4` reads four `f32` values, `store_i16x4` writes
//!   four `i16` values (on x86 from the lower half of a 128-bit register).
//! - `load2_` and `store2_` read or write two registers interleaved
//!   (`vld2q_f32`, `vst2q_f32`); `load_i8x16x4` reads four registers one
//!   after the other (`vld1q_s8_x4`).
//! - `load_dup_` reads one value into every lane, `load_lane_` and
//!   `store_lane_` read or write one lane. Here the name gives the register
//!   (`f32x4`).
//! - The end of the name says what the array holds when it is not plain
//!   values: `_cells` for `Cell` values, `_uninit` for `MaybeUninit`
//!   values that the store initializes, `_cpx` for complex values
//!   ([`KissCpx`](crate::kiss_fft::KissCpx), two `f32` values each).
//! - `to_array_` copies the lanes of a register into an array value; it
//!   touches no memory the caller can see.
//!
//! How a kernel gets the arrays from a slice without an index check in its
//! loop:
//! - `as_chunks`, `as_chunks_mut`, `as_rchunks` and `chunks_exact` split a
//!   slice into arrays. A loop over them has no index check.
//! - [`array_at`] and [`array_at_mut`] take the array at an index. The
//!   compiler drops their check when it can tell that the range is inside
//!   the slice, for example in a loop over at most 32 values that it
//!   unrolls. In other loops, such as `while i + 4 <= steps`, it may not:
//!   when it cannot rule out that `i + 4` wraps around, a compare stays in
//!   the loop. So check the generated code of a hot loop.
//! - Two ranges of one slice that a loop reads and writes in turn, such as
//!   `x[i..i + 4]` and `x[i + stride..i + stride + 4]`, are views of the
//!   same `&[Cell<f32>]` (`Cell::from_mut(x).as_slice_of_cells()`), split
//!   with `as_chunks`. The `_cells` functions load and store them.
//! - [`array_at_unchecked`] takes the array at an index without a check.
//!   It is an `unsafe fn`, for reads at positions that come from data the
//!   kernel checked before, such as the index table of a sparse layer of
//!   the neural networks.
//!
//! The functions are `#[inline]`, not `#[inline(always)]`, which Rust does
//! not allow together with `#[target_feature]`. Each is one instruction, so
//! the compiler always inlines it.

/// The `N` values `s[i..i + N]` as an array.
///
/// Panics when `s` holds fewer than `i + N` values.
// Only the NEON and x86 kernels use this and `array_at_mut`.
#[cfg_attr(
    not(any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")),
    allow(dead_code)
)]
#[inline(always)]
pub(crate) fn array_at<T, const N: usize>(s: &[T], i: usize) -> &[T; N] {
    s[i..i + N].try_into().unwrap()
}

/// The `N` values `s[i..i + N]` as a mutable array.
///
/// Panics when `s` holds fewer than `i + N` values.
#[cfg_attr(
    not(any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")),
    allow(dead_code)
)]
#[inline(always)]
pub(crate) fn array_at_mut<T, const N: usize>(s: &mut [T], i: usize) -> &mut [T; N] {
    (&mut s[i..i + N]).try_into().unwrap()
}

/// The `N` values `s[i..i + N]` as an array, without an index check.
///
/// For reads at positions that come from data the caller has checked
/// before, such as the index table of a sparse layer, where a check at
/// every read would cost speed.
///
/// # Safety
///
/// `i + N <= s.len()`.
// Only the neural network kernels use this.
#[cfg_attr(not(feature = "dnn"), allow(dead_code))]
#[inline(always)]
pub(crate) unsafe fn array_at_unchecked<T, const N: usize>(s: &[T], i: usize) -> &[T; N] {
    // SAFETY: the caller guarantees that `s[i..i + N]` lies inside `s`, so
    // the pointer is valid for reading `N` values of `T`, and `[T; N]` has
    // the alignment of `T`.
    unsafe { &*s.as_ptr().add(i).cast::<[T; N]>() }
}

/// Loads and stores for NEON (aarch64).
#[cfg(target_arch = "aarch64")]
pub(crate) mod neon {
    use crate::kiss_fft::KissCpx;
    use core::arch::aarch64::*;
    use core::cell::Cell;
    use core::mem::MaybeUninit;

    // `f32` values.

    /// Loads two `f32` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_f32x2(a: &[f32; 2]) -> float32x2_t {
        // SAFETY: `a` is valid for reading two `f32` values, which is
        // what `vld1_f32` reads. It needs only the alignment of `f32`.
        unsafe { vld1_f32(a.as_ptr()) }
    }

    /// Loads four `f32` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_f32x4(a: &[f32; 4]) -> float32x4_t {
        // SAFETY: `a` is valid for reading four `f32` values, which is
        // what `vld1q_f32` reads. It needs only the alignment of `f32`.
        unsafe { vld1q_f32(a.as_ptr()) }
    }

    /// Stores four `f32` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_f32x4(a: &mut [f32; 4], v: float32x4_t) {
        // SAFETY: `a` is valid for writing four `f32` values, which is
        // what `vst1q_f32` writes, and no other reference reaches them. It
        // needs only the alignment of `f32`.
        unsafe { vst1q_f32(a.as_mut_ptr(), v) }
    }

    /// Stores four `f32` values into memory that may not hold values yet.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_f32x4_uninit(a: &mut [MaybeUninit<f32>; 4], v: float32x4_t) {
        // SAFETY: `MaybeUninit<f32>` has the layout of `f32`, so `a` is
        // valid for writing four `f32` values, which is what `vst1q_f32`
        // writes, and no other reference reaches them. It needs only the
        // alignment of `f32`.
        unsafe { vst1q_f32(a.as_mut_ptr().cast(), v) }
    }

    /// Loads one `f32` value into all four lanes.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_dup_f32x4(a: &f32) -> float32x4_t {
        // SAFETY: `a` is valid for reading one `f32` value, which is what
        // `vld1q_dup_f32` reads.
        unsafe { vld1q_dup_f32(a) }
    }

    /// Loads one `f32` value into lane `LANE` of `v`. The other lanes keep
    /// their values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_lane_f32x4<const LANE: i32>(a: &f32, v: float32x4_t) -> float32x4_t {
        // SAFETY: `a` is valid for reading one `f32` value, which is what
        // `vld1q_lane_f32` reads.
        unsafe { vld1q_lane_f32::<LANE>(a, v) }
    }

    /// Stores lane `LANE` of `v`.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_lane_f32x4<const LANE: i32>(a: &mut f32, v: float32x4_t) {
        // SAFETY: `a` is valid for writing one `f32` value, which is what
        // `vst1q_lane_f32` writes, and no other reference reaches it.
        unsafe { vst1q_lane_f32::<LANE>(a, v) }
    }

    /// Loads eight `f32` values as two vectors: `.0` holds the values at
    /// the even places (0, 2, 4, 6), `.1` those at the odd places.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load2_f32x4(a: &[f32; 8]) -> float32x4x2_t {
        // SAFETY: `a` is valid for reading eight `f32` values, which is
        // what `vld2q_f32` reads. It needs only the alignment of `f32`.
        unsafe { vld2q_f32(a.as_ptr()) }
    }

    /// Stores two vectors as eight `f32` values, interleaved: lane `k` of
    /// `v.0` at place `2 * k`, lane `k` of `v.1` at place `2 * k + 1`.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store2_f32x4(a: &mut [f32; 8], v: float32x4x2_t) {
        // SAFETY: `a` is valid for writing eight `f32` values, which is
        // what `vst2q_f32` writes, and no other reference reaches them. It
        // needs only the alignment of `f32`.
        unsafe { vst2q_f32(a.as_mut_ptr(), v) }
    }

    /// Loads four `f32` values from cells.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_f32x4_cells(a: &[Cell<f32>; 4]) -> float32x4_t {
        // SAFETY: `Cell<f32>` has the layout of `f32`, so `a` is valid for
        // reading four `f32` values. Nothing writes them during the load:
        // a `Cell` lends no reference to its value and stays in one
        // thread.
        unsafe { vld1q_f32(a.as_ptr().cast()) }
    }

    /// Stores four `f32` values into cells.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_f32x4_cells(a: &[Cell<f32>; 4], v: float32x4_t) {
        // SAFETY: `Cell<f32>` has the layout of `f32` and may be written
        // through a shared reference, so `a` is valid for writing four `f32`
        // values. Nothing else reads or writes them during the store: a
        // `Cell` lends no reference to its value and stays in one thread.
        unsafe { vst1q_f32(a.as_ptr().cast_mut().cast(), v) }
    }

    /// Stores two vectors as eight `f32` values into cells, interleaved as
    /// `store2_f32x4` does.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store2_f32x4_cells(a: &[Cell<f32>; 8], v: float32x4x2_t) {
        // SAFETY: `Cell<f32>` has the layout of `f32` and may be written
        // through a shared reference, so `a` is valid for writing eight
        // `f32` values. Nothing else reads or writes them during the store:
        // a `Cell` lends no reference to its value and stays in one thread.
        unsafe { vst2q_f32(a.as_ptr().cast_mut().cast(), v) }
    }

    /// The four lanes of `v` as an array.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn to_array_f32x4(v: float32x4_t) -> [f32; 4] {
        // SAFETY: `float32x4_t` and `[f32; 4]` have the same size, and
        // every bit pattern is a valid value of both.
        unsafe { core::mem::transmute::<float32x4_t, [f32; 4]>(v) }
    }

    // Complex values (`KissCpx`: the real part, then the imaginary part,
    // as two `f32` values without padding).

    /// Loads one complex value as two `f32` values: lane 0 is the real
    /// part, lane 1 the imaginary part.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_f32x2_cpx(a: &KissCpx) -> float32x2_t {
        // SAFETY: `KissCpx` is two `f32` values without padding
        // (`#[repr(C)]`), so `a` is valid for reading two `f32` values,
        // which is what `vld1_f32` reads. It needs only the alignment of
        // `f32`.
        unsafe { vld1_f32((a as *const KissCpx).cast()) }
    }

    /// Stores two `f32` values as one complex value: lane 0 as the real
    /// part, lane 1 as the imaginary part.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_f32x2_cpx(a: &mut KissCpx, v: float32x2_t) {
        // SAFETY: `KissCpx` is two `f32` values without padding
        // (`#[repr(C)]`), so `a` is valid for writing two `f32` values,
        // which is what `vst1_f32` writes, and no other reference reaches
        // them. It needs only the alignment of `f32`.
        unsafe { vst1_f32((a as *mut KissCpx).cast(), v) }
    }

    /// Stores two `f32` values as one complex value, as `store_f32x2_cpx`,
    /// into memory that may not hold a value yet.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_f32x2_cpx_uninit(a: &mut MaybeUninit<KissCpx>, v: float32x2_t) {
        // SAFETY: `MaybeUninit<KissCpx>` has the layout of `KissCpx`, two
        // `f32` values without padding, so `a` is valid for writing two
        // `f32` values, which is what `vst1_f32` writes, and no other
        // reference reaches them. It needs only the alignment of `f32`.
        // After the store the value is initialized.
        unsafe { vst1_f32(a.as_mut_ptr().cast(), v) }
    }

    /// Loads two complex values as four `f32` values: the real and the
    /// imaginary part of the first, then those of the second.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_f32x4_cpx(a: &[KissCpx; 2]) -> float32x4_t {
        // SAFETY: `KissCpx` is two `f32` values without padding
        // (`#[repr(C)]`), so `a` is valid for reading four `f32` values,
        // which is what `vld1q_f32` reads. It needs only the alignment of
        // `f32`.
        unsafe { vld1q_f32(a.as_ptr().cast()) }
    }

    /// Stores four `f32` values as two complex values, in the order of
    /// `load_f32x4_cpx`.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_f32x4_cpx(a: &mut [KissCpx; 2], v: float32x4_t) {
        // SAFETY: `KissCpx` is two `f32` values without padding
        // (`#[repr(C)]`), so `a` is valid for writing four `f32` values,
        // which is what `vst1q_f32` writes, and no other reference reaches
        // them. It needs only the alignment of `f32`.
        unsafe { vst1q_f32(a.as_mut_ptr().cast(), v) }
    }

    /// Loads four complex values as two vectors: `.0` holds the real
    /// parts, `.1` the imaginary parts.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load2_f32x4_cpx(a: &[KissCpx; 4]) -> float32x4x2_t {
        // SAFETY: `KissCpx` is two `f32` values without padding
        // (`#[repr(C)]`), so `a` is valid for reading eight `f32` values,
        // which is what `vld2q_f32` reads. It needs only the alignment of
        // `f32`.
        unsafe { vld2q_f32(a.as_ptr().cast()) }
    }

    // `i32` and `u32` values.

    /// Loads two `i32` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_i32x2(a: &[i32; 2]) -> int32x2_t {
        // SAFETY: `a` is valid for reading two `i32` values, which is what
        // `vld1_s32` reads. It needs only the alignment of `i32`.
        unsafe { vld1_s32(a.as_ptr()) }
    }

    /// Stores two `i32` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_i32x2(a: &mut [i32; 2], v: int32x2_t) {
        // SAFETY: `a` is valid for writing two `i32` values, which is what
        // `vst1_s32` writes, and no other reference reaches them. It needs
        // only the alignment of `i32`.
        unsafe { vst1_s32(a.as_mut_ptr(), v) }
    }

    /// Loads four `i32` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_i32x4(a: &[i32; 4]) -> int32x4_t {
        // SAFETY: `a` is valid for reading four `i32` values, which is
        // what `vld1q_s32` reads. It needs only the alignment of `i32`.
        unsafe { vld1q_s32(a.as_ptr()) }
    }

    /// Stores four `i32` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_i32x4(a: &mut [i32; 4], v: int32x4_t) {
        // SAFETY: `a` is valid for writing four `i32` values, which is
        // what `vst1q_s32` writes, and no other reference reaches them. It
        // needs only the alignment of `i32`.
        unsafe { vst1q_s32(a.as_mut_ptr(), v) }
    }

    /// Stores lane `LANE` of `v`.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_lane_i32x4<const LANE: i32>(a: &mut i32, v: int32x4_t) {
        // SAFETY: `a` is valid for writing one `i32` value, which is what
        // `vst1q_lane_s32` writes, and no other reference reaches it.
        unsafe { vst1q_lane_s32::<LANE>(a, v) }
    }

    /// Loads four `u32` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_u32x4(a: &[u32; 4]) -> uint32x4_t {
        // SAFETY: `a` is valid for reading four `u32` values, which is
        // what `vld1q_u32` reads. It needs only the alignment of `u32`.
        unsafe { vld1q_u32(a.as_ptr()) }
    }

    // `i16` values.

    /// Loads four `i16` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_i16x4(a: &[i16; 4]) -> int16x4_t {
        // SAFETY: `a` is valid for reading four `i16` values, which is
        // what `vld1_s16` reads. It needs only the alignment of `i16`.
        unsafe { vld1_s16(a.as_ptr()) }
    }

    /// Stores four `i16` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_i16x4(a: &mut [i16; 4], v: int16x4_t) {
        // SAFETY: `a` is valid for writing four `i16` values, which is
        // what `vst1_s16` writes, and no other reference reaches them. It
        // needs only the alignment of `i16`.
        unsafe { vst1_s16(a.as_mut_ptr(), v) }
    }

    /// Stores four `i16` values into memory that may not hold values yet.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_i16x4_uninit(a: &mut [MaybeUninit<i16>; 4], v: int16x4_t) {
        // SAFETY: `MaybeUninit<i16>` has the layout of `i16`, so `a` is
        // valid for writing four `i16` values, which is what `vst1_s16`
        // writes, and no other reference reaches them. It needs only the
        // alignment of `i16`.
        unsafe { vst1_s16(a.as_mut_ptr().cast(), v) }
    }

    /// Loads eight `i16` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_i16x8(a: &[i16; 8]) -> int16x8_t {
        // SAFETY: `a` is valid for reading eight `i16` values, which is
        // what `vld1q_s16` reads. It needs only the alignment of `i16`.
        unsafe { vld1q_s16(a.as_ptr()) }
    }

    /// Stores eight `i16` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_i16x8(a: &mut [i16; 8], v: int16x8_t) {
        // SAFETY: `a` is valid for writing eight `i16` values, which is
        // what `vst1q_s16` writes, and no other reference reaches them. It
        // needs only the alignment of `i16`.
        unsafe { vst1q_s16(a.as_mut_ptr(), v) }
    }

    /// Loads sixteen `i16` values as two vectors: `.0` holds the values at
    /// the even places, `.1` those at the odd places.
    // Only the analysis of the encoder (`crate::analysis`) uses this.
    #[cfg_attr(not(feature = "analysis"), allow(dead_code))]
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load2_i16x8(a: &[i16; 16]) -> int16x8x2_t {
        // SAFETY: `a` is valid for reading sixteen `i16` values, which is
        // what `vld2q_s16` reads. It needs only the alignment of `i16`.
        unsafe { vld2q_s16(a.as_ptr()) }
    }

    // `u8` and `i8` values.

    /// Loads eight `u8` values.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_u8x8(a: &[u8; 8]) -> uint8x8_t {
        // SAFETY: `a` is valid for reading eight `u8` values, which is
        // what `vld1_u8` reads. It needs no alignment.
        unsafe { vld1_u8(a.as_ptr()) }
    }

    /// Loads sixteen `u8` values, such as a byte table for `vqtbl1q_u8`.
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_u8x16(a: &[u8; 16]) -> uint8x16_t {
        // SAFETY: `a` is valid for reading sixteen `u8` values, which is
        // what `vld1q_u8` reads. It needs no alignment.
        unsafe { vld1q_u8(a.as_ptr()) }
    }

    /// Loads sixteen `i8` values.
    // Only the neural network kernels (`crate::dnn`) use this.
    #[cfg_attr(not(feature = "dnn"), allow(dead_code))]
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_i8x16(a: &[i8; 16]) -> int8x16_t {
        // SAFETY: `a` is valid for reading sixteen `i8` values, which is
        // what `vld1q_s8` reads. It needs no alignment.
        unsafe { vld1q_s8(a.as_ptr()) }
    }

    /// Loads 64 `i8` values into four registers (`vld1q_s8_x4`): register
    /// `k` gets `a[16·k..16·k + 16]`.
    // Only the neural network kernels (`crate::dnn`) use this.
    #[cfg_attr(not(feature = "dnn"), allow(dead_code))]
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn load_i8x16x4(a: &[i8; 64]) -> int8x16x4_t {
        // SAFETY: `a` is valid for reading 64 `i8` values, which is what
        // `vld1q_s8_x4` reads. It needs no alignment.
        unsafe { vld1q_s8_x4(a.as_ptr()) }
    }

    /// Stores the low four `i8` values of `v` into memory that may not
    /// hold values yet.
    // Only the neural network kernels (`crate::dnn`) use this.
    #[cfg_attr(not(feature = "dnn"), allow(dead_code))]
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_i8x4_uninit(a: &mut [MaybeUninit<i8>; 4], v: int8x8_t) {
        // The four bytes as one `i32`, written without alignment: `a` has
        // the alignment of `i8` only.
        let x = vget_lane_s32::<0>(vreinterpret_s32_s8(v));
        // SAFETY: `a` is valid for writing four bytes, which is what
        // `write_unaligned` of an `i32` writes, and no other reference
        // reaches them. It needs no alignment. Writing values into
        // `MaybeUninit` is always allowed.
        unsafe { a.as_mut_ptr().cast::<i32>().write_unaligned(x) }
    }

    /// Stores sixteen `i8` values into memory that may not hold values yet.
    // Only the neural network kernels (`crate::dnn`) use this.
    #[cfg_attr(not(feature = "dnn"), allow(dead_code))]
    #[target_feature(enable = "neon")]
    #[inline]
    pub(crate) fn store_i8x16_uninit(a: &mut [MaybeUninit<i8>; 16], v: int8x16_t) {
        // SAFETY: `a` is valid for writing sixteen bytes, which is what
        // `vst1q_s8` writes, and no other reference reaches them. It needs
        // no alignment. Writing values into `MaybeUninit` is always
        // allowed.
        unsafe { vst1q_s8(a.as_mut_ptr().cast(), v) }
    }
}

/// Loads and stores for SSE, SSE2, AVX and AVX-512 (x86 and x86-64).
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
pub(crate) mod x86 {
    use crate::kiss_fft::KissCpx;
    #[cfg(target_arch = "x86")]
    use core::arch::x86::*;
    #[cfg(target_arch = "x86_64")]
    use core::arch::x86_64::*;
    use core::cell::Cell;
    use core::mem::MaybeUninit;

    // `f32` values.

    /// Loads four `f32` values (SSE).
    #[target_feature(enable = "sse")]
    #[inline]
    pub(crate) fn load_f32x4(a: &[f32; 4]) -> __m128 {
        // SAFETY: `a` is valid for reading four `f32` values, which is
        // what `_mm_loadu_ps` reads. It needs no alignment.
        unsafe { _mm_loadu_ps(a.as_ptr()) }
    }

    /// Stores four `f32` values (SSE).
    #[target_feature(enable = "sse")]
    #[inline]
    pub(crate) fn store_f32x4(a: &mut [f32; 4], v: __m128) {
        // SAFETY: `a` is valid for writing four `f32` values, which is
        // what `_mm_storeu_ps` writes, and no other reference reaches
        // them. It needs no alignment.
        unsafe { _mm_storeu_ps(a.as_mut_ptr(), v) }
    }

    /// Loads four `f32` values from cells (SSE).
    #[target_feature(enable = "sse")]
    #[inline]
    pub(crate) fn load_f32x4_cells(a: &[Cell<f32>; 4]) -> __m128 {
        // SAFETY: `Cell<f32>` has the layout of `f32`, so `a` is valid for
        // reading four `f32` values. Nothing writes them during the load:
        // a `Cell` lends no reference to its value and stays in one
        // thread.
        unsafe { _mm_loadu_ps(a.as_ptr().cast()) }
    }

    /// Stores four `f32` values into cells (SSE).
    #[target_feature(enable = "sse")]
    #[inline]
    pub(crate) fn store_f32x4_cells(a: &[Cell<f32>; 4], v: __m128) {
        // SAFETY: `Cell<f32>` has the layout of `f32` and may be written
        // through a shared reference, so `a` is valid for writing four `f32`
        // values. Nothing else reads or writes them during the store: a
        // `Cell` lends no reference to its value and stays in one thread.
        unsafe { _mm_storeu_ps(a.as_ptr().cast_mut().cast(), v) }
    }

    /// The four lanes of `v` as an array (SSE).
    #[target_feature(enable = "sse")]
    #[inline]
    pub(crate) fn to_array_f32x4(v: __m128) -> [f32; 4] {
        // SAFETY: `__m128` and `[f32; 4]` have the same size, and every
        // bit pattern is a valid value of both.
        unsafe { core::mem::transmute::<__m128, [f32; 4]>(v) }
    }

    /// Loads eight `f32` values (AVX).
    #[target_feature(enable = "avx")]
    #[inline]
    pub(crate) fn load_f32x8(a: &[f32; 8]) -> __m256 {
        // SAFETY: `a` is valid for reading eight `f32` values, which is
        // what `_mm256_loadu_ps` reads. It needs no alignment.
        unsafe { _mm256_loadu_ps(a.as_ptr()) }
    }

    /// Stores eight `f32` values (AVX).
    #[target_feature(enable = "avx")]
    #[inline]
    pub(crate) fn store_f32x8(a: &mut [f32; 8], v: __m256) {
        // SAFETY: `a` is valid for writing eight `f32` values, which is
        // what `_mm256_storeu_ps` writes, and no other reference reaches
        // them. It needs no alignment.
        unsafe { _mm256_storeu_ps(a.as_mut_ptr(), v) }
    }

    // Complex values (`KissCpx`: the real part, then the imaginary part,
    // as two `f32` values without padding).

    /// Stores the lower two lanes of `v` as one complex value: lane 0 as
    /// the real part, lane 1 as the imaginary part (SSE2).
    // Only the analysis of the encoder (`crate::analysis`) uses this.
    #[cfg_attr(not(feature = "analysis"), allow(dead_code))]
    #[target_feature(enable = "sse2")]
    #[inline]
    pub(crate) fn store_f32x2_cpx(a: &mut KissCpx, v: __m128) {
        // SAFETY: `KissCpx` is two `f32` values without padding
        // (`#[repr(C)]`), so `a` is valid for writing two `f32` values
        // (8 bytes), which is what `_mm_storel_epi64` writes, and no other
        // reference reaches them. It needs no alignment.
        unsafe { _mm_storel_epi64((a as *mut KissCpx).cast(), _mm_castps_si128(v)) }
    }

    /// Loads two complex values as four `f32` values: the real and the
    /// imaginary part of the first, then those of the second (SSE).
    // Only the analysis of the encoder (`crate::analysis`) uses this.
    #[cfg_attr(not(feature = "analysis"), allow(dead_code))]
    #[target_feature(enable = "sse")]
    #[inline]
    pub(crate) fn load_f32x4_cpx(a: &[KissCpx; 2]) -> __m128 {
        // SAFETY: `KissCpx` is two `f32` values without padding
        // (`#[repr(C)]`), so `a` is valid for reading four `f32` values,
        // which is what `_mm_loadu_ps` reads. It needs no alignment.
        unsafe { _mm_loadu_ps(a.as_ptr().cast()) }
    }

    // `i32` values.

    /// Loads four `i32` values (SSE2).
    #[target_feature(enable = "sse2")]
    #[inline]
    pub(crate) fn load_i32x4(a: &[i32; 4]) -> __m128i {
        // SAFETY: `a` is valid for reading four `i32` values (16 bytes),
        // which is what `_mm_loadu_si128` reads. It needs no alignment.
        unsafe { _mm_loadu_si128(a.as_ptr().cast()) }
    }

    /// Stores four `i32` values (SSE2).
    #[target_feature(enable = "sse2")]
    #[inline]
    pub(crate) fn store_i32x4(a: &mut [i32; 4], v: __m128i) {
        // SAFETY: `a` is valid for writing four `i32` values (16 bytes),
        // which is what `_mm_storeu_si128` writes, and no other reference
        // reaches them. It needs no alignment.
        unsafe { _mm_storeu_si128(a.as_mut_ptr().cast(), v) }
    }

    /// Stores four `i32` values into memory that may not hold values yet
    /// (SSE2).
    #[target_feature(enable = "sse2")]
    #[inline]
    pub(crate) fn store_i32x4_uninit(a: &mut [MaybeUninit<i32>; 4], v: __m128i) {
        // SAFETY: `MaybeUninit<i32>` has the layout of `i32`, so `a` is
        // valid for writing four `i32` values (16 bytes), which is what
        // `_mm_storeu_si128` writes, and no other reference reaches them.
        // It needs no alignment.
        unsafe { _mm_storeu_si128(a.as_mut_ptr().cast(), v) }
    }

    /// Loads eight `i32` values (AVX).
    #[target_feature(enable = "avx")]
    #[inline]
    pub(crate) fn load_i32x8(a: &[i32; 8]) -> __m256i {
        // SAFETY: `a` is valid for reading eight `i32` values (32 bytes),
        // which is what `_mm256_loadu_si256` reads. It needs no alignment.
        unsafe { _mm256_loadu_si256(a.as_ptr().cast()) }
    }

    /// Stores eight `i32` values (AVX).
    #[target_feature(enable = "avx")]
    #[inline]
    pub(crate) fn store_i32x8(a: &mut [i32; 8], v: __m256i) {
        // SAFETY: `a` is valid for writing eight `i32` values (32 bytes),
        // which is what `_mm256_storeu_si256` writes, and no other
        // reference reaches them. It needs no alignment.
        unsafe { _mm256_storeu_si256(a.as_mut_ptr().cast(), v) }
    }

    // `i16` values.

    /// Loads four `i16` values into the lower half (SSE2).
    #[target_feature(enable = "sse2")]
    #[inline]
    pub(crate) fn load_i16x4(a: &[i16; 4]) -> __m128i {
        // SAFETY: `a` is valid for reading four `i16` values (8 bytes),
        // which is what `_mm_loadl_epi64` reads. It needs no alignment.
        unsafe { _mm_loadl_epi64(a.as_ptr().cast()) }
    }

    /// Stores the four `i16` values of the lower half (SSE2).
    #[target_feature(enable = "sse2")]
    #[inline]
    pub(crate) fn store_i16x4(a: &mut [i16; 4], v: __m128i) {
        // SAFETY: `a` is valid for writing four `i16` values (8 bytes),
        // which is what `_mm_storel_epi64` writes, and no other reference
        // reaches them. It needs no alignment.
        unsafe { _mm_storel_epi64(a.as_mut_ptr().cast(), v) }
    }

    /// Stores the four `i16` values of the lower half into memory that may
    /// not hold values yet (SSE2).
    #[target_feature(enable = "sse2")]
    #[inline]
    pub(crate) fn store_i16x4_uninit(a: &mut [MaybeUninit<i16>; 4], v: __m128i) {
        // SAFETY: `MaybeUninit<i16>` has the layout of `i16`, so `a` is
        // valid for writing four `i16` values (8 bytes), which is what
        // `_mm_storel_epi64` writes, and no other reference reaches them.
        // It needs no alignment.
        unsafe { _mm_storel_epi64(a.as_mut_ptr().cast(), v) }
    }

    /// Loads eight `i16` values (SSE2).
    #[target_feature(enable = "sse2")]
    #[inline]
    pub(crate) fn load_i16x8(a: &[i16; 8]) -> __m128i {
        // SAFETY: `a` is valid for reading eight `i16` values (16 bytes),
        // which is what `_mm_loadu_si128` reads. It needs no alignment.
        unsafe { _mm_loadu_si128(a.as_ptr().cast()) }
    }

    /// Stores eight `i16` values (SSE2).
    #[target_feature(enable = "sse2")]
    #[inline]
    pub(crate) fn store_i16x8(a: &mut [i16; 8], v: __m128i) {
        // SAFETY: `a` is valid for writing eight `i16` values (16 bytes),
        // which is what `_mm_storeu_si128` writes, and no other reference
        // reaches them. It needs no alignment.
        unsafe { _mm_storeu_si128(a.as_mut_ptr().cast(), v) }
    }

    /// Loads sixteen `i16` values (AVX).
    #[target_feature(enable = "avx")]
    #[inline]
    pub(crate) fn load_i16x16(a: &[i16; 16]) -> __m256i {
        // SAFETY: `a` is valid for reading sixteen `i16` values (32 bytes),
        // which is what `_mm256_loadu_si256` reads. It needs no alignment.
        unsafe { _mm256_loadu_si256(a.as_ptr().cast()) }
    }

    // `u8` and `i8` values.

    /// Loads sixteen `u8` values, such as a byte table for
    /// `_mm_shuffle_epi8` (SSE2).
    #[target_feature(enable = "sse2")]
    #[inline]
    pub(crate) fn load_u8x16(a: &[u8; 16]) -> __m128i {
        // SAFETY: `a` is valid for reading sixteen `u8` values (16 bytes),
        // which is what `_mm_loadu_si128` reads. It needs no alignment.
        unsafe { _mm_loadu_si128(a.as_ptr().cast()) }
    }

    /// Loads 32 `i8` values (AVX).
    // Only the neural network kernels (`crate::dnn`) use this.
    #[cfg_attr(not(feature = "dnn"), allow(dead_code))]
    #[target_feature(enable = "avx")]
    #[inline]
    pub(crate) fn load_i8x32(a: &[i8; 32]) -> __m256i {
        // SAFETY: `a` is valid for reading 32 bytes, which is what
        // `_mm256_loadu_si256` reads. It needs no alignment.
        unsafe { _mm256_loadu_si256(a.as_ptr().cast()) }
    }

    /// Loads 64 `i8` values (AVX-512 F).
    // Only the neural network kernels (`crate::dnn`) use this.
    #[cfg_attr(not(feature = "dnn"), allow(dead_code))]
    #[target_feature(enable = "avx512f")]
    #[inline]
    pub(crate) fn load_i8x64(a: &[i8; 64]) -> __m512i {
        // SAFETY: `a` is valid for reading 64 bytes, which is what
        // `_mm512_loadu_si512` reads. It needs no alignment.
        unsafe { _mm512_loadu_si512(a.as_ptr().cast()) }
    }
}