npsimd 0.3.0

//! Intrinsics for SSE2.

use core::ptr;

use super::*;

/// SSE2 (Streaming SIMD Extensions 2).
///
/// SSE2 is the second extension in the SSE generation, released in 2000 with
/// the Intel Pentium 4.  It is available on practically every x86 CPU you can
/// find nowadays.  While it is quite old, it provides a solid foundation of
/// integer vector operations that is sufficient for most algorithms.  If you
/// are designing a vectorized algorithm by hand, SSE2 is mandatory.
///
/// While SSE2 has the basic operations one would expect -- addition,
/// subtraction, bitwise AND/OR/XOR, and some data shuffling -- it also has a
/// few specialized instructions, like [`psadbw`].
///
/// [`psadbw`]: Use::sum_of_abs_diff_u8x16
///
/// # Instructions
///
/// These instructions are implemented by [`Use`].
///
/// Converting between general-purpose and vector registers:
///
/// - [x] `movd`
///   - [`Use::set_first_u32x4`]
///   - [`Use::set_first_s32x4`]
///   - [`Use::get_first_u32x4`]
///   - [`Use::get_first_s32x4`]
///
/// - [x] `movq`
///   - [`Use::set_first_u64x2`]
///   - [`Use::set_first_s64x2`]
///   - [`Use::get_first_u64x2`]
///   - [`Use::get_first_s64x2`]
///
/// Element insertion and extraction:
///
/// - [x] `pextrw`
///   - [`Use::get_u16x8`]
///   - [`Use::get_s16x8`]
///
/// - [x] `pinsrw`
///   - [`Use::put_u16x8`]
///   - [`Use::put_s16x8`]
///
/// Loading from and storing to memory:
///
/// - [x] `movdqa`
///   - [`Use::load_aligned_u8x16`]
///   - [`Use::load_aligned_u16x8`]
///   - [`Use::load_aligned_u32x4`]
///   - [`Use::load_aligned_u64x2`]
///   - [`Use::load_aligned_s8x16`]
///   - [`Use::load_aligned_s16x8`]
///   - [`Use::load_aligned_s32x4`]
///   - [`Use::load_aligned_s64x2`]
///
/// - [x] `movdqu`
///   - [`Use::load_u8x16`]
///   - [`Use::load_u16x8`]
///   - [`Use::load_u32x4`]
///   - [`Use::load_u64x2`]
///   - [`Use::load_s8x16`]
///   - [`Use::load_s16x8`]
///   - [`Use::load_s32x4`]
///   - [`Use::load_s64x2`]
///
/// Basic integer operations:
///
/// - [x] `paddb`
///   - [`Use::add_u8x16`]
///   - [`Use::add_s8x16`]
/// - [x] `paddw`
///   - [`Use::add_u16x8`]
///   - [`Use::add_s16x8`]
/// - [x] `paddd`
///   - [`Use::add_u32x4`]
///   - [`Use::add_s32x4`]
/// - [x] `paddq`
///   - [`Use::add_u64x2`]
///   - [`Use::add_s64x2`]
///
/// - [x] `psubb`
///   - [`Use::sub_u8x16`]
///   - [`Use::sub_s8x16`]
/// - [x] `psubw`
///   - [`Use::sub_u16x8`]
///   - [`Use::sub_s16x8`]
/// - [x] `psubd`
///   - [`Use::sub_u32x4`]
///   - [`Use::sub_s32x4`]
/// - [x] `psubq`
///   - [`Use::sub_u64x2`]
///   - [`Use::sub_s64x2`]
///
/// - [x] `paddsb`
///   - [`Use::saturating_add_s8x16`]
/// - [x] `paddsw`
///   - [`Use::saturating_add_s16x8`]
///
/// - [x] `psubsb`
///   - [`Use::saturating_sub_s8x16`]
/// - [x] `psubsw`
///   - [`Use::saturating_sub_s16x8`]
///
/// - [x] `pmulhuw`
///   - [`Use::mul_hi_u16x8`]
/// - [x] `pmulhw`
///   - [`Use::mul_hi_s16x8`]
/// - [x] `pmullw`
///   - [`Use::mul_lo_u16x8`]
///   - [`Use::mul_lo_s16x8`]
/// - [x] `pmuludq`
///   - [`Use::mul_u32_u64x2`]
///
/// - [x] `pmaxub`
///   - [`Use::max_u8x16`]
/// - [x] `pmaxsw`
///   - [`Use::max_s16x8`]
///
/// - [x] `pminub`
///   - [`Use::min_u8x16`]
/// - [x] `pminsw`
///   - [`Use::min_s16x8`]
///
/// - [x] `pavgb`
///   - [`Use::avg_u8x16`]
/// - [x] `pavgw`
///   - [`Use::avg_u16x8`]
///
/// Bitwise operations:
///
/// - [x] `pand`
///   - [`Use::and_u8x16`]
///   - [`Use::and_u16x8`]
///   - [`Use::and_u32x4`]
///   - [`Use::and_u64x2`]
///   - [`Use::and_s8x16`]
///   - [`Use::and_s16x8`]
///   - [`Use::and_s32x4`]
///   - [`Use::and_s64x2`]
///
/// - [x] `pandn`
///   - [`Use::and_not_u8x16`]
///   - [`Use::and_not_u16x8`]
///   - [`Use::and_not_u32x4`]
///   - [`Use::and_not_u64x2`]
///   - [`Use::and_not_s8x16`]
///   - [`Use::and_not_s16x8`]
///   - [`Use::and_not_s32x4`]
///   - [`Use::and_not_s64x2`]
///
/// - [x] `por`
///   - [`Use::ior_u8x16`]
///   - [`Use::ior_u16x8`]
///   - [`Use::ior_u32x4`]
///   - [`Use::ior_u64x2`]
///   - [`Use::ior_s8x16`]
///   - [`Use::ior_s16x8`]
///   - [`Use::ior_s32x4`]
///   - [`Use::ior_s64x2`]
///
/// - [x] `pxor`
///   - [`Use::xor_u8x16`]
///   - [`Use::xor_u16x8`]
///   - [`Use::xor_u32x4`]
///   - [`Use::xor_u64x2`]
///   - [`Use::xor_s8x16`]
///   - [`Use::xor_s16x8`]
///   - [`Use::xor_s32x4`]
///   - [`Use::xor_s64x2`]
///
/// - [x] `psllw`
///   - [`Use::shl_all_u16x8`]
///   - [`Use::shl_all_by_u16x8`]
///   - [`Use::shl_all_s16x8`]
///   - [`Use::shl_all_by_s16x8`]
///
/// - [x] `pslld`
///   - [`Use::shl_all_u32x4`]
///   - [`Use::shl_all_by_u32x4`]
///   - [`Use::shl_all_s32x4`]
///   - [`Use::shl_all_by_s32x4`]
///
/// - [x] `psllq`
///   - [`Use::shl_all_u64x2`]
///   - [`Use::shl_all_by_u64x2`]
///   - [`Use::shl_all_s64x2`]
///   - [`Use::shl_all_by_s64x2`]
///
/// - [x] `psrlw`
///   - [`Use::shr_all_u16x8`]
///   - [`Use::shr_all_by_u16x8`]
/// - [x] `psraw`
///   - [`Use::shr_all_s16x8`]
///   - [`Use::shr_all_by_s16x8`]
///
/// - [x] `psrld`
///   - [`Use::shr_all_u32x4`]
///   - [`Use::shr_all_by_u32x4`]
/// - [x] `psrad`
///   - [`Use::shr_all_s32x4`]
///   - [`Use::shr_all_by_s32x4`]
///
/// - [x] `psrlq`
///   - [`Use::shr_all_u64x2`]
///   - [`Use::shr_all_by_u64x2`]
///
/// Specialized integer operations:
///
/// - [x] `psadbw`
///   - [`Use::sum_of_abs_diff_u8x16`]
/// - [x] `pmaddwd`
///   - [`Use::sum_of_prod_s16x2x4`]
///
/// Integer comparisons:
///
/// - [x] `pcmpeqb`
///   - [`Use::cmp_eq_u8x16`]
///   - [`Use::cmp_eq_s8x16`]
/// - [x] `pcmpeqd`
///   - [`Use::cmp_eq_u16x8`]
///   - [`Use::cmp_eq_s16x8`]
/// - [x] `pcmpeqw`
///   - [`Use::cmp_eq_u32x4`]
///   - [`Use::cmp_eq_s32x4`]
///
/// - [x] `pcmpgtb`
///   - [`Use::cmp_gt_s8x16`]
/// - [x] `pcmpgtw`
///   - [`Use::cmp_gt_s16x8`]
/// - [x] `pcmpgtd`
///   - [`Use::cmp_gt_s32x4`]
///
/// Data shuffling:
///
/// - [x] `pshufd`
///   - [`Use::shuffle_by_u32x4`]
///   - [`Use::shuffle_by_s32x4`]
///
/// - [ ] `pshuflw`
/// - [ ] `pshufhw`
///
/// - [x] `pslldq`
///   - [`Use::move_l_by_u8x16`]
///   - [`Use::move_l_by_s8x16`]
/// - [x] `psrldq`
///   - [`Use::move_r_by_u8x16`]
///   - [`Use::move_r_by_s8x16`]
///
/// - [x] `packuswb`
///   - [`Use::concat_and_saturate_u8_s16x8`]
/// - [x] `packssdw`
///   - [`Use::concat_and_saturate_s8_s16x8`]
/// - [x] `packsswb`
///   - [`Use::concat_and_saturate_s16_s32x4`]
///
/// - [x] `punpcklbw`
///   - [`Use::interleave_lo_u8x16`]
///   - [`Use::interleave_lo_s8x16`]
/// - [x] `punpcklwd`
///   - [`Use::interleave_lo_u16x8`]
///   - [`Use::interleave_lo_s16x8`]
/// - [x] `punpckldq`
///   - [`Use::interleave_lo_u32x4`]
///   - [`Use::interleave_lo_s32x4`]
/// - [x] `punpcklqdq`
///   - [`Use::interleave_lo_u64x2`]
///   - [`Use::interleave_lo_s64x2`]
///
/// - [x] `punpckhbw`
///   - [`Use::interleave_hi_u8x16`]
///   - [`Use::interleave_hi_s8x16`]
/// - [x] `punpckhwd`
///   - [`Use::interleave_hi_u16x8`]
///   - [`Use::interleave_hi_s16x8`]
/// - [x] `punpckhdq`
///   - [`Use::interleave_hi_u32x4`]
///   - [`Use::interleave_hi_s32x4`]
/// - [x] `punpckhqdq`
///   - [`Use::interleave_hi_u64x2`]
///   - [`Use::interleave_hi_s64x2`]
///
/// Bitmasks:
///
/// - [x] `pmovmskb`
///   - [`Use::bitmask_u8x16`]
///   - [`Use::bitmask_s8x16`]
pub struct SSE2(());

impl Feature<FeatureGroup> for SSE2 {
    fn get_support(runtime: &RuntimeSupport) -> Option<Self> {
        runtime.sse2().then_some(Self(()))
    }
}

/// SSE2 intrinsics.
///
/// See [`SSE2`] for information about this extension and which instructions are
/// supported here.
impl<FS> Use<FS>
where FS: HasFeature<FeatureGroup, SSE2> {
    // Construct a 16-byte integer vector from a primitive first element.
    defn_simd_shared!("sse2", { simd_set_first(value) }, {
        /// Create a vector with a given first element and all else zero.
        #[intrinsic_for("movd")]
        #[intel_equivalents("_mm_cvtsi32_si128")]
        pub fn set_first_u32x4(value: u32) -> u32x4;

        /// Create a vector with a given first element and all else zero.
        #[intrinsic_for("movd")]
        #[intel_equivalents("_mm_cvtsi32_si128")]
        pub fn set_first_s32x4(value: i32) -> s32x4;

        /// Create a vector with a given first element and all else zero.
        #[intrinsic_for("movq")]
        #[intel_equivalents("_mm_cvtsi64_si128", "_mm_cvtsi64x_si128")]
        #[cfg(target_arch = "x86_64")]
        pub fn set_first_u64x2(value: u64) -> u64x2;

        /// Create a vector with a given first element and all else zero.
        #[intrinsic_for("movq")]
        #[intel_equivalents("_mm_cvtsi64_si128", "_mm_cvtsi64x_si128")]
        #[cfg(target_arch = "x86_64")]
        pub fn set_first_s64x2(value: i64) -> s64x2;
    });

    // Extract the first primitive element in a 16-byte vector.
    defn_simd_shared!("sse2", { simd_extract(x, 0) }, {
        /// Extract the first 32-bit integer in the vector.
        #[intrinsic_for("movd")]
        #[intel_equivalents("_mm_cvtsi128_si32")]
        pub fn get_first_u32x4(x: u32x4) -> u32;

        /// Extract the first 64-bit integer in the vector.
        #[intrinsic_for("movq")]
        #[intel_equivalents("_mm_cvtsi128_si64", "_mm_cvtsi128_si64x")]
        pub fn get_first_u64x2(x: u64x2) -> u64;

        /// Extract the first 32-bit integer in the vector.
        #[intrinsic_for("movd")]
        #[intel_equivalents("_mm_cvtsi128_si32")]
        pub fn get_first_s32x4(x: s32x4) -> i32;

        /// Extract the first 64-bit integer in the vector.
        #[intrinsic_for("movq")]
        #[intel_equivalents("_mm_cvtsi128_si64", "_mm_cvtsi128_si64x")]
        pub fn get_first_s64x2(x: s64x2) -> i64;
    });

    // Insert a primitive element into a 16-byte vector.
    defn_simd_shared!("sse2", fn(T, E) -> R {
        const_assert!(INDEX < T::LEN as u8);
        simd_insert(x, const { INDEX as u32 }, e)
    }, {
        /// Replace an unsigned 16-bit integer at a constant index.
        #[intrinsic_for("pinsrw")]
        #[intel_equivalents("_mm_insert_epi16")]
        pub fn put_u16x8<INDEX: u8>(x: u16x8, e: u16) -> u16x8;

        /// Replace a signed 16-bit integer at a constant index.
        #[intrinsic_for("pinsrw")]
        #[intel_equivalents("_mm_insert_epi16")]
        pub fn put_s16x8<INDEX: u8>(x: s16x8, e: i16) -> s16x8;
    });

    // Extract a primitive element from a 16-byte vector.
    defn_simd_shared!("sse2", fn(T) -> E {
        const_assert!(INDEX < T::LEN as u8);
        simd_extract(x, const { INDEX as u32 })
    }, {
        /// Extract an unsigned 16-bit integer from a constant index.
        #[intrinsic_for("pextrw")]
        #[intel_equivalents("_mm_extract_epi16")]
        pub fn get_u16x8<INDEX: u8>(x: u16x8) -> u16;

        /// Extract a signed 16-bit integer from a constant index.
        #[intrinsic_for("pextrw")]
        #[intel_equivalents("_mm_extract_epi16")]
        pub fn get_s16x8<INDEX: u8>(x: s16x8) -> i16;
    });

    // Load a 16-byte integer vector from unaligned memory.
    defn_simd_shared!("sse2", {
        ptr::read_unaligned(ptr as *const _ as *const _)
    }, {
        /// Load 16 unsigned 8-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_loadu_si128")]
        pub fn load_u8x16(ptr: &[u8; 16]) -> u8x16;

        /// Load 8 unsigned 16-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_loadu_si128")]
        pub fn load_u16x8(ptr: &[u16; 8]) -> u16x8;

        /// Load 4 unsigned 32-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_loadu_si128")]
        pub fn load_u32x4(ptr: &[u32; 4]) -> u32x4;

        /// Load 2 unsigned 64-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_loadu_si128")]
        pub fn load_u64x2(ptr: &[u64; 2]) -> u64x2;

        /// Load 16 signed 8-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_loadu_si128")]
        pub fn load_s8x16(ptr: &[i8; 16]) -> s8x16;

        /// Load 8 signed 16-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_loadu_si128")]
        pub fn load_s16x8(ptr: &[i16; 8]) -> s16x8;

        /// Load 4 signed 32-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_loadu_si128")]
        pub fn load_s32x4(ptr: &[i32; 4]) -> s32x4;

        /// Load 2 signed 64-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_loadu_si128")]
        pub fn load_s64x2(ptr: &[i64; 2]) -> s64x2;
    });

    // Load a 16-byte integer vector from aligned memory.
    defn_simd_shared!("sse2", { *ptr }, {
        /// Load 16 unsigned 8-bit integers from aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_load_si128")]
        pub fn load_aligned_u8x16(ptr: &u8x16) -> u8x16;

        /// Load 8 unsigned 16-bit integers from aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_load_si128")]
        pub fn load_aligned_u16x8(ptr: &u16x8) -> u16x8;

        /// Load 4 unsigned 32-bit integers from aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_load_si128")]
        pub fn load_aligned_u32x4(ptr: &u32x4) -> u32x4;

        /// Load 2 unsigned 64-bit integers from aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_load_si128")]
        pub fn load_aligned_u64x2(ptr: &u64x2) -> u64x2;

        /// Load 16 signed 8-bit integers from aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_load_si128")]
        pub fn load_aligned_s8x16(ptr: &s8x16) -> s8x16;

        /// Load 8 signed 16-bit integers from aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_load_si128")]
        pub fn load_aligned_s16x8(ptr: &s16x8) -> s16x8;

        /// Load 4 signed 32-bit integers from aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_load_si128")]
        pub fn load_aligned_s32x4(ptr: &s32x4) -> s32x4;

        /// Load 2 signed 64-bit integers from aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_load_si128")]
        pub fn load_aligned_s64x2(ptr: &s64x2) -> s64x2;
    });

    // Store a 16-byte integer vector to unaligned memory.
    defn_simd_shared!("sse2", {
        ptr::write_unaligned(ptr as *mut _ as *mut _, x)
    }, {
        /// Store 16 unsigned 8-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_storeu_si128")]
        pub fn store_u8x16(x: u8x16, ptr: &mut [u8; 16]);

        /// Store 8 unsigned 16-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_storeu_si128")]
        pub fn store_u16x8(x: u16x8, ptr: &mut [u16; 8]);

        /// Store 4 unsigned 32-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_storeu_si128")]
        pub fn store_u32x4(x: u32x4, ptr: &mut [u32; 4]);

        /// Store 2 unsigned 64-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_storeu_si128")]
        pub fn store_u64x2(x: u64x2, ptr: &mut [u64; 2]);

        /// Store 16 signed 8-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_storeu_si128")]
        pub fn store_s8x16(x: s8x16, ptr: &mut [i8; 16]);

        /// Store 8 signed 16-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_storeu_si128")]
        pub fn store_s16x8(x: s16x8, ptr: &mut [i16; 8]);

        /// Store 4 signed 32-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_storeu_si128")]
        pub fn store_s32x4(x: s32x4, ptr: &mut [i32; 4]);

        /// Store 2 signed 64-bit integers.
        #[intrinsic_for("movdqu")]
        #[intel_equivalents("_mm_storeu_si128")]
        pub fn store_s64x2(x: s64x2, ptr: &mut [i64; 2]);
    });

    // Store a 16-byte integer vector to aligned memory.
    defn_simd_shared!("sse2", { *ptr = x }, {
        /// Store 16 unsigned 8-bit integers to aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_store_si128")]
        pub fn store_aligned_u8x16(x: u8x16, ptr: &mut u8x16);

        /// Store 8 unsigned 16-bit integers to aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_store_si128")]
        pub fn store_aligned_u16x8(x: u16x8, ptr: &mut u16x8);

        /// Store 4 unsigned 32-bit integers to aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_store_si128")]
        pub fn store_aligned_u32x4(x: u32x4, ptr: &mut u32x4);

        /// Store 2 unsigned 64-bit integers to aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_store_si128")]
        pub fn store_aligned_u64x2(x: u64x2, ptr: &mut u64x2);

        /// Store 16 signed 8-bit integers to aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_store_si128")]
        pub fn store_aligned_s8x16(x: s8x16, ptr: &mut s8x16);

        /// Store 8 signed 16-bit integers to aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_store_si128")]
        pub fn store_aligned_s16x8(x: s16x8, ptr: &mut s16x8);

        /// Store 4 signed 32-bit integers to aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_store_si128")]
        pub fn store_aligned_s32x4(x: s32x4, ptr: &mut s32x4);

        /// Store 2 signed 64-bit integers to aligned memory.
        #[intrinsic_for("movdqa")]
        #[intel_equivalents("_mm_store_si128")]
        pub fn store_aligned_s64x2(x: s64x2, ptr: &mut s64x2);
    });

    // Add integer elements in two 16-byte vectors.
    defn_simd_shared!("sse2", { simd_add(a, b) }, {
        /// Add unsigned 8-bit integers.
        #[intrinsic_for("paddb")]
        #[intel_equivalents("_mm_add_epi8")]
        pub fn add_u8x16(a: u8x16, b: u8x16) -> u8x16;

        /// Add unsigned 16-bit integers.
        #[intrinsic_for("paddw")]
        #[intel_equivalents("_mm_add_epi16")]
        pub fn add_u16x8(a: u16x8, b: u16x8) -> u16x8;

        /// Add unsigned 32-bit integers.
        #[intrinsic_for("paddd")]
        #[intel_equivalents("_mm_add_epi32")]
        pub fn add_u32x4(a: u32x4, b: u32x4) -> u32x4;

        /// Add unsigned 64-bit integers.
        #[intrinsic_for("paddq")]
        #[intel_equivalents("_mm_add_epi64")]
        pub fn add_u64x2(a: u64x2, b: u64x2) -> u64x2;

        /// Add signed 8-bit integers.
        #[intrinsic_for("paddb")]
        #[intel_equivalents("_mm_add_epi8")]
        pub fn add_s8x16(a: s8x16, b: s8x16) -> s8x16;

        /// Add signed 16-bit integers.
        #[intrinsic_for("paddw")]
        #[intel_equivalents("_mm_add_epi16")]
        pub fn add_s16x8(a: s16x8, b: s16x8) -> s16x8;

        /// Add signed 32-bit integers.
        #[intrinsic_for("paddd")]
        #[intel_equivalents("_mm_add_epi32")]
        pub fn add_s32x4(a: s32x4, b: s32x4) -> s32x4;

        /// Add signed 64-bit integers.
        #[intrinsic_for("paddq")]
        #[intel_equivalents("_mm_add_epi64")]
        pub fn add_s64x2(a: s64x2, b: s64x2) -> s64x2;
    });

    // Subtract integer elements in two 16-byte vectors.
    defn_simd_shared!("sse2", { simd_sub(a, b) }, {
        /// Subtract unsigned 8-bit integers.
        #[intrinsic_for("psubb")]
        #[intel_equivalents("_mm_sub_epi8")]
        pub fn sub_u8x16(a: u8x16, b: u8x16) -> u8x16;

        /// Subtract unsigned 16-bit integers.
        #[intrinsic_for("psubw")]
        #[intel_equivalents("_mm_sub_epi16")]
        pub fn sub_u16x8(a: u16x8, b: u16x8) -> u16x8;

        /// Subtract unsigned 32-bit integers.
        #[intrinsic_for("psubd")]
        #[intel_equivalents("_mm_sub_epi32")]
        pub fn sub_u32x4(a: u32x4, b: u32x4) -> u32x4;

        /// Subtract unsigned 64-bit integers.
        #[intrinsic_for("psubq")]
        #[intel_equivalents("_mm_sub_epi64")]
        pub fn sub_u64x2(a: u64x2, b: u64x2) -> u64x2;

        /// Subtract signed 8-bit integers.
        #[intrinsic_for("psubb")]
        #[intel_equivalents("_mm_sub_epi8")]
        pub fn sub_s8x16(a: s8x16, b: s8x16) -> s8x16;

        /// Subtract signed 16-bit integers.
        #[intrinsic_for("psubw")]
        #[intel_equivalents("_mm_sub_epi16")]
        pub fn sub_s16x8(a: s16x8, b: s16x8) -> s16x8;

        /// Subtract signed 32-bit integers.
        #[intrinsic_for("psubd")]
        #[intel_equivalents("_mm_sub_epi32")]
        pub fn sub_s32x4(a: s32x4, b: s32x4) -> s32x4;

        /// Subtract signed 64-bit integers.
        #[intrinsic_for("psubq")]
        #[intel_equivalents("_mm_sub_epi64")]
        pub fn sub_s64x2(a: s64x2, b: s64x2) -> s64x2;
    });

    // Add integer elements in two 16-byte vectors with saturation.
    defn_simd_shared!("sse2", { simd_saturating_add(a, b) }, {
        /// Add signed 8-bit integers with saturation.
        #[intrinsic_for("paddsb")]
        #[intel_equivalents("_mm_adds_epi8")]
        pub fn saturating_add_s8x16(a: s8x16, b: s8x16) -> s8x16;

        /// Add signed 16-bit integers with saturation.
        #[intrinsic_for("paddsw")]
        #[intel_equivalents("_mm_adds_epi16")]
        pub fn saturating_add_s16x8(a: s16x8, b: s16x8) -> s16x8;
    });

    // Subtract integer elements in two 16-byte vectors with saturation.
    defn_simd_shared!("sse2", { simd_saturating_sub(a, b) }, {
        /// Subtract signed 8-bit integers with saturation.
        #[intrinsic_for("psubsb")]
        #[intel_equivalents("_mm_subs_epi8")]
        pub fn saturating_sub_s8x16(a: s8x16, b: s8x16) -> s8x16;

        /// Subtract signed 16-bit integers with saturation.
        #[intrinsic_for("psubsw")]
        #[intel_equivalents("_mm_subs_epi16")]
        pub fn saturating_sub_s16x8(a: s16x8, b: s16x8) -> s16x8;
    });

    // Multiply integer elements in 16-byte vectors for low words.
    defn_simd_shared!("sse2", { simd_mul(a, b) }, {
        /// Multiply unsigned 16-bit integers for the low words.
        #[intrinsic_for("pmullw")]
        #[intel_equivalents("_mm_mullo_epi16")]
        pub fn mul_lo_u16x8(a: u16x8, b: u16x8) -> u16x8;

        /// Multiply signed 16-bit integers for the low words.
        #[intrinsic_for("pmullw")]
        #[intel_equivalents("_mm_mullo_epi16")]
        pub fn mul_lo_s16x8(a: s16x8, b: s16x8) -> s16x8;
    });

    // Multiply integer elements in 16-byte vectors for high words.
    defn_simd_manual!("sse2", {
        /// Multiply unsigned 16-bit integers and return the higher half.
        #[intrinsic_for("pmulhuw")]
        #[intel_equivalents("_mm_mulhi_epu16")]
        pub fn mul_hi_u16x8(a: u16x8, b: u16x8) -> u16x8 {
            let prod = simd_mul(simd_cast(a), simd_cast(b));
            simd_cast(simd_shr(prod, u32x8::splat(16)))
        }

        /// Multiply signed 16-bit integers and return the higher half.
        #[intrinsic_for("pmulhw")]
        #[intel_equivalents("_mm_mulhi_epi16")]
        pub fn mul_hi_s16x8(a: s16x8, b: s16x8) -> s16x8 {
            let prod = simd_mul(simd_cast(a), simd_cast(b));
            simd_cast(simd_shr(prod, s32x8::splat(16)))
        }
    });

    // Multiply low words in integer elements in 16-byte vectors.
    defn_simd_manual!("sse2", {
        /// Multiply the low 32 bits in unsigned 64-bit integers.
        #[intrinsic_for("pmuludq")]
        #[intel_equivalents("_mm_mul_epu32")]
        pub fn mul_u32_u64x2(a: u64x2, b: u64x2) -> u64x2 {
            let [a, b]: [u32x2; 2] = [simd_cast(a), simd_cast(b)];
            simd_mul(simd_cast(a), simd_cast(b))
        }
    });

    // Multiply integer elements in two 16-byte vectors and horizontally add
    // pairs of full-width products.
    defn_simd_llvm!("sse2", {
        /// Multiply signed 16-bit integers and horizontally add pairs of 32-bit
        /// results.
        #[intrinsic_for("pmaddwd")]
        #[intel_equivalents("_mm_madd_epi16")]
        pub fn sum_of_prod_s16x2x4
            (x: s16x8, y: s16x8) -> s32x4
            = "llvm.x86.sse2.pmadd.wd";
    });

    // Compare integer elements in two 16-byte vectors for equality.
    defn_simd_shared!("sse2", { simd_eq(a, b) }, {
        /// Compare unsigned 8-bit integers for equality.
        #[intrinsic_for("pcmpeqb")]
        #[intel_equivalents("_mm_cmpeq_epi8")]
        pub fn cmp_eq_u8x16(a: u8x16, b: u8x16) -> u8x16;

        /// Compare unsigned 16-bit integers for equality.
        #[intrinsic_for("pcmpeqw")]
        #[intel_equivalents("_mm_cmpeq_epi16")]
        pub fn cmp_eq_u16x8(a: u16x8, b: u16x8) -> u16x8;

        /// Compare unsigned 32-bit integers for equality.
        #[intrinsic_for("pcmpeqd")]
        #[intel_equivalents("_mm_cmpeq_epi32")]
        pub fn cmp_eq_u32x4(a: u32x4, b: u32x4) -> u32x4;

        /// Compare signed 8-bit integers for equality.
        #[intrinsic_for("pcmpeqb")]
        #[intel_equivalents("_mm_cmpeq_epi8")]
        pub fn cmp_eq_s8x16(a: s8x16, b: s8x16) -> s8x16;

        /// Compare signed 16-bit integers for equality.
        #[intrinsic_for("pcmpeqw")]
        #[intel_equivalents("_mm_cmpeq_epi16")]
        pub fn cmp_eq_s16x8(a: s16x8, b: s16x8) -> s16x8;

        /// Compare signed 32-bit integers for equality.
        #[intrinsic_for("pcmpeqd")]
        #[intel_equivalents("_mm_cmpeq_epi32")]
        pub fn cmp_eq_s32x4(a: s32x4, b: s32x4) -> s32x4;
    });

    // Compare integer elements in two 16-byte vectors for greater-than.
    defn_simd_shared!("sse2", { simd_gt(a, b) }, {
        /// Compare unsigned 8-bit integers for greater-than
        #[intrinsic_for("pcmpgtb")]
        #[intel_equivalents("_mm_cmpgt_epi8", "_mm_cmplt_epi8")]
        pub fn cmp_gt_s8x16(a: s8x16, b: s8x16) -> s8x16;

        /// Compare unsigned 16-bit integers for greater-than.
        #[intrinsic_for("pcmpgtw")]
        #[intel_equivalents("_mm_cmpgt_epi16", "_mm_cmplt_epi16")]
        pub fn cmp_gt_s16x8(a: s16x8, b: s16x8) -> s16x8;

        /// Compare unsigned 32-bit integers for greater-than.
        #[intrinsic_for("pcmpgtd")]
        #[intel_equivalents("_mm_cmpgt_epi32", "_mm_cmplt_epi32")]
        pub fn cmp_gt_s32x4(a: s32x4, b: s32x4) -> s32x4;
    });

    // Bitwise AND two 16-byte vectors.
    defn_simd_shared!("sse2", { simd_and(a, b) }, {
        /// Bitwise AND unsigned 8-bit integers.
        #[intrinsic_for("pand")]
        #[intel_equivalents("_mm_and_si128")]
        pub fn and_u8x16(a: u8x16, b: u8x16) -> u8x16;

        /// Bitwise AND unsigned 16-bit integers.
        #[intrinsic_for("pand")]
        #[intel_equivalents("_mm_and_si128")]
        pub fn and_u16x8(a: u16x8, b: u16x8) -> u16x8;

        /// Bitwise AND unsigned 32-bit integers.
        #[intrinsic_for("pand")]
        #[intel_equivalents("_mm_and_si128")]
        pub fn and_u32x4(a: u32x4, b: u32x4) -> u32x4;

        /// Bitwise AND unsigned 64-bit integers.
        #[intrinsic_for("pand")]
        #[intel_equivalents("_mm_and_si128")]
        pub fn and_u64x2(a: u64x2, b: u64x2) -> u64x2;

        /// Bitwise AND signed 8-bit integers.
        #[intrinsic_for("pand")]
        #[intel_equivalents("_mm_and_si128")]
        pub fn and_s8x16(a: s8x16, b: s8x16) -> s8x16;

        /// Bitwise AND signed 16-bit integers.
        #[intrinsic_for("pand")]
        #[intel_equivalents("_mm_and_si128")]
        pub fn and_s16x8(a: s16x8, b: s16x8) -> s16x8;

        /// Bitwise AND signed 32-bit integers.
        #[intrinsic_for("pand")]
        #[intel_equivalents("_mm_and_si128")]
        pub fn and_s32x4(a: s32x4, b: s32x4) -> s32x4;

        /// Bitwise AND signed 64-bit integers.
        #[intrinsic_for("pand")]
        #[intel_equivalents("_mm_and_si128")]
        pub fn and_s64x2(a: s64x2, b: s64x2) -> s64x2;
    });

    // Bitwise inclusive OR two 16-byte vectors.
    defn_simd_shared!("sse2", { simd_or(a, b) }, {
        /// Bitwise inclusive OR unsigned 8-bit integers.
        #[intrinsic_for("por")]
        #[intel_equivalents("_mm_or_si128")]
        pub fn ior_u8x16(a: u8x16, b: u8x16) -> u8x16;

        /// Bitwise inclusive OR unsigned 16-bit integers.
        #[intrinsic_for("por")]
        #[intel_equivalents("_mm_or_si128")]
        pub fn ior_u16x8(a: u16x8, b: u16x8) -> u16x8;

        /// Bitwise inclusive OR unsigned 32-bit integers.
        #[intrinsic_for("por")]
        #[intel_equivalents("_mm_or_si128")]
        pub fn ior_u32x4(a: u32x4, b: u32x4) -> u32x4;

        /// Bitwise inclusive OR unsigned 64-bit integers.
        #[intrinsic_for("por")]
        #[intel_equivalents("_mm_or_si128")]
        pub fn ior_u64x2(a: u64x2, b: u64x2) -> u64x2;

        /// Bitwise inclusive OR signed 8-bit integers.
        #[intrinsic_for("por")]
        #[intel_equivalents("_mm_or_si128")]
        pub fn ior_s8x16(a: s8x16, b: s8x16) -> s8x16;

        /// Bitwise inclusive OR signed 16-bit integers.
        #[intrinsic_for("por")]
        #[intel_equivalents("_mm_or_si128")]
        pub fn ior_s16x8(a: s16x8, b: s16x8) -> s16x8;

        /// Bitwise inclusive OR signed 32-bit integers.
        #[intrinsic_for("por")]
        #[intel_equivalents("_mm_or_si128")]
        pub fn ior_s32x4(a: s32x4, b: s32x4) -> s32x4;

        /// Bitwise inclusive OR signed 64-bit integers.
        #[intrinsic_for("por")]
        #[intel_equivalents("_mm_or_si128")]
        pub fn ior_s64x2(a: s64x2, b: s64x2) -> s64x2;
    });

    // Bitwise exclusive OR two 16-byte vectors.
    defn_simd_shared!("sse2", { simd_xor(a, b) }, {
        /// Bitwise exclusive OR unsigned 8-bit integers.
        #[intrinsic_for("pxor")]
        #[intel_equivalents("_mm_xor_si128")]
        pub fn xor_u8x16(a: u8x16, b: u8x16) -> u8x16;

        /// Bitwise exclusive OR unsigned 16-bit integers.
        #[intrinsic_for("pxor")]
        #[intel_equivalents("_mm_xor_si128")]
        pub fn xor_u16x8(a: u16x8, b: u16x8) -> u16x8;

        /// Bitwise exclusive OR unsigned 32-bit integers.
        #[intrinsic_for("pxor")]
        #[intel_equivalents("_mm_xor_si128")]
        pub fn xor_u32x4(a: u32x4, b: u32x4) -> u32x4;

        /// Bitwise exclusive OR unsigned 64-bit integers.
        #[intrinsic_for("pxor")]
        #[intel_equivalents("_mm_xor_si128")]
        pub fn xor_u64x2(a: u64x2, b: u64x2) -> u64x2;

        /// Bitwise exclusive OR signed 8-bit integers.
        #[intrinsic_for("pxor")]
        #[intel_equivalents("_mm_xor_si128")]
        pub fn xor_s8x16(a: s8x16, b: s8x16) -> s8x16;

        /// Bitwise exclusive OR signed 16-bit integers.
        #[intrinsic_for("pxor")]
        #[intel_equivalents("_mm_xor_si128")]
        pub fn xor_s16x8(a: s16x8, b: s16x8) -> s16x8;

        /// Bitwise exclusive OR signed 32-bit integers.
        #[intrinsic_for("pxor")]
        #[intel_equivalents("_mm_xor_si128")]
        pub fn xor_s32x4(a: s32x4, b: s32x4) -> s32x4;

        /// Bitwise exclusive OR signed 64-bit integers.
        #[intrinsic_for("pxor")]
        #[intel_equivalents("_mm_xor_si128")]
        pub fn xor_s64x2(a: s64x2, b: s64x2) -> s64x2;
    });

    // Bitwise AND NOT two 16-byte vectors.
    defn_simd_shared!("sse2", { simd_andnot(a, b) }, {
        /// Bitwise AND NOT unsigned 8-bit integers.
        #[intrinsic_for("pandn")]
        #[intel_equivalents("_mm_andnot_si128")]
        pub fn and_not_u8x16(a: u8x16, b: u8x16) -> u8x16;

        /// Bitwise AND NOT unsigned 16-bit integers.
        #[intrinsic_for("pandn")]
        #[intel_equivalents("_mm_andnot_si128")]
        pub fn and_not_u16x8(a: u16x8, b: u16x8) -> u16x8;

        /// Bitwise AND NOT unsigned 32-bit integers.
        #[intrinsic_for("pandn")]
        #[intel_equivalents("_mm_andnot_si128")]
        pub fn and_not_u32x4(a: u32x4, b: u32x4) -> u32x4;

        /// Bitwise AND NOT unsigned 64-bit integers.
        #[intrinsic_for("pandn")]
        #[intel_equivalents("_mm_andnot_si128")]
        pub fn and_not_u64x2(a: u64x2, b: u64x2) -> u64x2;

        /// Bitwise AND NOT signed 8-bit integers.
        #[intrinsic_for("pandn")]
        #[intel_equivalents("_mm_andnot_si128")]
        pub fn and_not_s8x16(a: s8x16, b: s8x16) -> s8x16;

        /// Bitwise AND NOT signed 16-bit integers.
        #[intrinsic_for("pandn")]
        #[intel_equivalents("_mm_andnot_si128")]
        pub fn and_not_s16x8(a: s16x8, b: s16x8) -> s16x8;

        /// Bitwise AND NOT signed 32-bit integers.
        #[intrinsic_for("pandn")]
        #[intel_equivalents("_mm_andnot_si128")]
        pub fn and_not_s32x4(a: s32x4, b: s32x4) -> s32x4;

        /// Bitwise AND NOT signed 64-bit integers.
        #[intrinsic_for("pandn")]
        #[intel_equivalents("_mm_andnot_si128")]
        pub fn and_not_s64x2(a: s64x2, b: s64x2) -> s64x2;
    });

    // Average of integer elements in 16-byte vectors.
    defn_simd_manual!("sse2", {
        /// Average unsigned 8-bit integers.
        #[intrinsic_for("pavgb")]
        #[intel_equivalents("_mm_avg_epu8")]
        pub fn avg_u8x16(a: u8x16, b: u8x16) -> u8x16
            = simd_avg::<u8x16, u16x16>;

        /// Average unsigned 16-bit integers.
        #[intrinsic_for("pavgw")]
        #[intel_equivalents("_mm_avg_epu16")]
        pub fn avg_u16x8(a: u16x8, b: u16x8) -> u16x8
            = simd_avg::<u16x8, u32x8>;
    });

    // Maximum of integer elements in 16-byte vectors.
    defn_simd_shared!("sse2", { simd_max(a, b) }, {
        /// Maximum of unsigned 8-bit integers.
        #[intrinsic_for("pmaxub")]
        #[intel_equivalents("_mm_max_epu8")]
        pub fn max_u8x16(a: u8x16, b: u8x16) -> u8x16;

        /// Maximum of signed 16-bit integers.
        #[intrinsic_for("pmaxsw")]
        #[intel_equivalents("_mm_max_epi16")]
        pub fn max_s16x8(a: s16x8, b: s16x8) -> s16x8;
    });

    // Minimum of integer elements in 16-byte vectors.
    defn_simd_shared!("sse2", { simd_min(a, b) }, {
        /// Minimum of unsigned 8-bit integers.
        #[intrinsic_for("pminub")]
        #[intel_equivalents("_mm_min_epu8")]
        pub fn min_u8x16(a: u8x16, b: u8x16) -> u8x16;

        /// Minimum of signed 16-bit integers.
        #[intrinsic_for("pminsw")]
        #[intel_equivalents("_mm_min_epi16")]
        pub fn min_s16x8(a: s16x8, b: s16x8) -> s16x8;
    });

    // Left-shift integer elements in a 16-byte vector by a variable.
    defn_simd_llvm!("sse2", {
        /// Left-shift every unsigned 16-bit integer by the first value in the
        /// second argument.
        #[intrinsic_for("psllw")]
        #[intel_equivalents("_mm_sll_epi16")]
        pub fn shl_all_u16x8(x: u16x8, s: u64x2) -> u16x8
            = "llvm.x86.sse2.psll.w";

        /// Left-shift every unsigned 32-bit integer by the first value in the
        /// second argument.
        #[intrinsic_for("pslld")]
        #[intel_equivalents("_mm_sll_epi32")]
        pub fn shl_all_u32x4(x: u32x4, s: u64x2) -> u32x4
            = "llvm.x86.sse2.psll.d";

        /// Left-shift every unsigned 64-bit integer by the first value in the
        /// second argument.
        #[intrinsic_for("psllq")]
        #[intel_equivalents("_mm_sll_epi64")]
        pub fn shl_all_u64x2(x: u64x2, s: u64x2) -> u64x2
            = "llvm.x86.sse2.psll.q";

        /// Left-shift every signed 16-bit integer by the first value in the
        /// second argument.
        #[intrinsic_for("psllw")]
        #[intel_equivalents("_mm_sll_epi16")]
        pub fn shl_all_s16x8(x: s16x8, s: u64x2) -> s16x8
            = "llvm.x86.sse2.psll.w";

        /// Left-shift every signed 32-bit integer by the first value in the
        /// second argument.
        #[intrinsic_for("pslld")]
        #[intel_equivalents("_mm_sll_epi32")]
        pub fn shl_all_s32x4(x: s32x4, s: u64x2) -> s32x4
            = "llvm.x86.sse2.psll.d";

        /// Left-shift every signed 64-bit integer by the first value in the
        /// second argument.
        #[intrinsic_for("psllq")]
        #[intel_equivalents("_mm_sll_epi64")]
        pub fn shl_all_s64x2(x: s64x2, s: u64x2) -> s64x2
            = "llvm.x86.sse2.psll.q";
    });

    // Left-shift integer elements in a 16-byte vector by a constant.
    defn_simd_shared!("sse2", { simd_shl_all::<_, BITS>(x) }, {
        /// Left-shift every unsigned 16-bit integer by a constant.
        #[intrinsic_for("psllw")]
        #[intel_equivalents("_mm_slli_epi16")]
        pub fn shl_all_by_u16x8<BITS: u8>(x: u16x8) -> u16x8;

        /// Left-shift every unsigned 32-bit integer by a constant.
        #[intrinsic_for("pslld")]
        #[intel_equivalents("_mm_slli_epi32")]
        pub fn shl_all_by_u32x4<BITS: u8>(x: u32x4) -> u32x4;

        /// Left-shift every unsigned 64-bit integer by a constant.
        #[intrinsic_for("psllq")]
        #[intel_equivalents("_mm_slli_epi64")]
        pub fn shl_all_by_u64x2<BITS: u8>(x: u64x2) -> u64x2;

        /// Left-shift every signed 16-bit integer by a constant.
        #[intrinsic_for("psllw")]
        #[intel_equivalents("_mm_slli_epi16")]
        pub fn shl_all_by_s16x8<BITS: u8>(x: s16x8) -> s16x8;

        /// Left-shift every signed 32-bit integer by a constant.
        #[intrinsic_for("pslld")]
        #[intel_equivalents("_mm_slli_epi32")]
        pub fn shl_all_by_s32x4<BITS: u8>(x: s32x4) -> s32x4;

        /// Left-shift every signed 64-bit integer by a constant.
        #[intrinsic_for("psllq")]
        #[intel_equivalents("_mm_slli_epi64")]
        pub fn shl_all_by_s64x2<BITS: u8>(x: s64x2) -> s64x2;
    });

    // Right-shift integer elements in a 16-byte vector by a variable.
    defn_simd_llvm!("sse2", {
        /// Right-shift every unsigned 16-bit integer by the first value in the
        /// second argument.
        #[intrinsic_for("psrlw")]
        #[intel_equivalents("_mm_srl_epi16")]
        pub fn shr_all_u16x8(x: u16x8, s: u64x2) -> u16x8
            = "llvm.x86.sse2.psrl.w";

        /// Right-shift every unsigned 32-bit integer by the first value in the
        /// second argument.
        #[intrinsic_for("psrld")]
        #[intel_equivalents("_mm_srl_epi32")]
        pub fn shr_all_u32x4(x: u32x4, s: u64x2) -> u32x4
            = "llvm.x86.sse2.psrl.d";

        /// Right-shift every unsigned 64-bit integer by the first value in the
        /// second argument.
        #[intrinsic_for("psrlq")]
        #[intel_equivalents("_mm_srl_epi64")]
        pub fn shr_all_u64x2(x: u64x2, s: u64x2) -> u64x2
            = "llvm.x86.sse2.psrl.q";

        /// Right-shift every signed 16-bit integer by the first value in the
        /// second argument.
        #[intrinsic_for("psraw")]
        #[intel_equivalents("_mm_sra_epi16")]
        pub fn shr_all_s16x8(x: s16x8, s: u64x2) -> s16x8
            = "llvm.x86.sse2.psra.w";

        /// Right-shift every signed 32-bit integer by the first value in the
        /// second argument.
        #[intrinsic_for("psrad")]
        #[intel_equivalents("_mm_sra_epi32")]
        pub fn shr_all_s32x4(x: s32x4, s: u64x2) -> s32x4
            = "llvm.x86.sse2.psra.d";
    });

    // Right-shift integer elements in a 16-byte vector by a constant.
    defn_simd_shared!("sse2", fn(T) -> R { simd_shr_all::<_, BITS>(x) }, {
        /// Right-shift every unsigned 16-bit integer by a constant.
        #[intrinsic_for("psrlw")]
        #[intel_equivalents("_mm_srli_epi16")]
        pub fn shr_all_by_u16x8<BITS: u8>(x: u16x8) -> u16x8;

        /// Right-shift every unsigned 32-bit integer by a constant.
        #[intrinsic_for("psrld")]
        #[intel_equivalents("_mm_srli_epi32")]
        pub fn shr_all_by_u32x4<BITS: u8>(x: u32x4) -> u32x4;

        /// Right-shift every unsigned 64-bit integer by a constant.
        #[intrinsic_for("psrlq")]
        #[intel_equivalents("_mm_srli_epi64")]
        pub fn shr_all_by_u64x2<BITS: u8>(x: u64x2) -> u64x2;

        /// Right-shift every signed 16-bit integer by a constant.
        #[intrinsic_for("psraw")]
        #[intel_equivalents("_mm_srai_epi16")]
        pub fn shr_all_by_s16x8<BITS: u8>(x: s16x8) -> s16x8;

        /// Right-shift every signed 32-bit integer by a constant.
        #[intrinsic_for("psrad")]
        #[intel_equivalents("_mm_srai_epi32")]
        pub fn shr_all_by_s32x4<BITS: u8>(x: s32x4) -> s32x4;
    });

    // Saturate and pack integer elements from two 16-byte vectors.
    defn_simd_llvm!("sse2", {
        /// Saturate two vectors of signed 16-bit integers to unsigned 8 bits.
        #[intrinsic_for("packuswb")]
        #[intel_equivalents("_mm_packus_epi16")]
        pub fn concat_and_saturate_u8_s16x8(a: s16x8, b: s16x8) -> u8x16
            = "llvm.x86.sse2.packuswb.128";

        /// Saturate two vectors of signed 16-bit integers to signed 8 bits.
        #[intrinsic_for("packsswb")]
        #[intel_equivalents("_mm_packs_epi16")]
        pub fn concat_and_saturate_s8_s16x8(a: s16x8, b: s16x8) -> s8x16
            = "llvm.x86.sse2.packsswb.128";

        /// Saturate two vectors of signed 32-bit integers to signed 16 bits.
        #[intrinsic_for("packssdw")]
        #[intel_equivalents("_mm_packs_epi32")]
        pub fn concat_and_saturate_s16_s32x4(a: s32x4, b: s32x4) -> s16x8
            = "llvm.x86.sse2.packssdw.128";
    });

    // Shuffle integer elements in a 16-byte vector.
    defn_simd_shared!("sse2", fn(T) -> R {
        const fn indices(idxs: [u8; T::LEN]) -> [i32; T::LEN] {
            let mut indices = [0i32; T::LEN];
            let mut index = 0usize;
            while index < idxs.len() {
                assert!((idxs[index] as usize) < T::LEN);
                indices[index] = idxs[index] as i32;
                index += 1;
            }
            indices
        }

        simd_shuffle(x, T::splat(0), const { indices(Indices::VAL) })
    }, {
        /// Shuffle unsigned 32-bit integers using a constant index list.
        #[intrinsic_for("pshufd")]
        #[intel_equivalents("_mm_shuffle_epi32")]
        pub fn shuffle_by_u32x4[Indices: [u8; 4]](x: u32x4) -> u32x4;

        /// Shuffle signed 32-bit integers using a constant index list.
        #[intrinsic_for("pshufd")]
        #[intel_equivalents("_mm_shuffle_epi32")]
        pub fn shuffle_by_s32x4[Indices: [u8; 4]](x: s32x4) -> s32x4;
    });

    // Interleave integer elements from the low halves of two 16-byte vectors.
    defn_simd_shared!("sse2", fn(T, U) -> R {
        simd_shuffle(a, b, const { simd_unpack_indices::<R>(0) })
    }, {
        /// Interleave unsigned 8-bit integers from the low halves of vectors.
        #[intrinsic_for("punpcklbw")]
        #[intel_equivalents("_mm_unpacklo_epi8")]
        pub fn interleave_lo_u8x16(a: u8x16, b: u8x16) -> u8x16;

        /// Interleave unsigned 16-bit integers from the low halves of vectors.
        #[intrinsic_for("punpcklwd")]
        #[intel_equivalents("_mm_unpacklo_epi16")]
        pub fn interleave_lo_u16x8(a: u16x8, b: u16x8) -> u16x8;

        /// Interleave unsigned 32-bit integers from the low halves of vectors.
        #[intrinsic_for("punpckldq")]
        #[intel_equivalents("_mm_unpacklo_epi32")]
        pub fn interleave_lo_u32x4(a: u32x4, b: u32x4) -> u32x4;

        /// Interleave unsigned 64-bit integers from the low halves of vectors.
        #[intrinsic_for("punpcklqdq")]
        #[intel_equivalents("_mm_unpacklo_epi64")]
        pub fn interleave_lo_u64x2(a: u64x2, b: u64x2) -> u64x2;

        /// Interleave signed 8-bit integers from the low halves of vectors.
        #[intrinsic_for("punpcklbw")]
        #[intel_equivalents("_mm_unpacklo_epi8")]
        pub fn interleave_lo_s8x16(a: s8x16, b: s8x16) -> s8x16;

        /// Interleave signed 16-bit integers from the low halves of vectors.
        #[intrinsic_for("punpcklwd")]
        #[intel_equivalents("_mm_unpacklo_epi16")]
        pub fn interleave_lo_s16x8(a: s16x8, b: s16x8) -> s16x8;

        /// Interleave signed 32-bit integers from the low halves of vectors.
        #[intrinsic_for("punpckldq")]
        #[intel_equivalents("_mm_unpacklo_epi32")]
        pub fn interleave_lo_s32x4(a: s32x4, b: s32x4) -> s32x4;

        /// Interleave signed 64-bit integers from the low halves of vectors.
        #[intrinsic_for("punpcklqdq")]
        #[intel_equivalents("_mm_unpacklo_epi64")]
        pub fn interleave_lo_s64x2(a: s64x2, b: s64x2) -> s64x2;
    });

    // Interleave integer elements from the high halves of two 16-byte vectors.
    defn_simd_shared!("sse2", fn(T, U) -> R {
        simd_shuffle(a, b, const { simd_unpack_indices::<R>(T::LEN / 2) })
    }, {
        /// Interleave unsigned 8-bit integers from the high halves of vectors.
        #[intrinsic_for("punpckhbw")]
        #[intel_equivalents("_mm_unpackhi_epi8")]
        pub fn interleave_hi_u8x16(a: u8x16, b: u8x16) -> u8x16;

        /// Interleave unsigned 16-bit integers from the high halves of vectors.
        #[intrinsic_for("punpckhwd")]
        #[intel_equivalents("_mm_unpackhi_epi16")]
        pub fn interleave_hi_u16x8(a: u16x8, b: u16x8) -> u16x8;

        /// Interleave unsigned 32-bit integers from the high halves of vectors.
        #[intrinsic_for("punpckhdq")]
        #[intel_equivalents("_mm_unpackhi_epi32")]
        pub fn interleave_hi_u32x4(a: u32x4, b: u32x4) -> u32x4;

        /// Interleave unsigned 64-bit integers from the high halves of vectors.
        #[intrinsic_for("punpckhqdq")]
        #[intel_equivalents("_mm_unpackhi_epi64")]
        pub fn interleave_hi_u64x2(a: u64x2, b: u64x2) -> u64x2;

        /// Interleave signed 8-bit integers from the high halves of vectors.
        #[intrinsic_for("punpckhbw")]
        #[intel_equivalents("_mm_unpackhi_epi8")]
        pub fn interleave_hi_s8x16(a: s8x16, b: s8x16) -> s8x16;

        /// Interleave signed 16-bit integers from the high halves of vectors.
        #[intrinsic_for("punpckhwd")]
        #[intel_equivalents("_mm_unpackhi_epi16")]
        pub fn interleave_hi_s16x8(a: s16x8, b: s16x8) -> s16x8;

        /// Interleave signed 32-bit integers from the high halves of vectors.
        #[intrinsic_for("punpckhdq")]
        #[intel_equivalents("_mm_unpackhi_epi32")]
        pub fn interleave_hi_s32x4(a: s32x4, b: s32x4) -> s32x4;

        /// Interleave signed 64-bit integers from the high halves of vectors.
        #[intrinsic_for("punpckhqdq")]
        #[intel_equivalents("_mm_unpackhi_epi64")]
        pub fn interleave_hi_s64x2(a: s64x2, b: s64x2) -> s64x2;
    });

    // Compute absolute differences between integer elements in 16-byte vectors
    // and horizontally add them.
    defn_simd_llvm!("sse2", {
        /// Sum of 8 absolute differences of unsigned 8-bit integers.
        ///
        /// Compute the absolute differences between corresponding 8-bit
        /// unsigned integers, then horizontally add groups of unsigned 8-bit
        /// sums into 16-bit sums, and zero-extend them to 64-bit integers.
        #[intrinsic_for("psadbw")]
        #[intel_equivalents("_mm_sad_epu8")]
        pub fn sum_of_abs_diff_u8x16(a: u8x16, b: u8x16) -> u64x2
            = "llvm.x86.sse2.psad.bw";
    });

    // Move byte elements in a 16-byte vector left by a constant.
    defn_simd_shared!("sse2", fn(T) -> R {
        simd_shuffle(T::splat(0), x, const {
            simd_slice_indices::<T>(T::LEN.saturating_sub(ELEMS as usize))
        })
    }, {
        /// Move unsigned 8-bit integer elements to the left.
        #[intrinsic_for("pslldq")]
        #[intel_equivalents("_mm_bslli_si128", "_mm_slli_si128")]
        pub fn move_l_by_u8x16<ELEMS: u8>(x: u8x16) -> u8x16;

        /// Move signed 8-bit integer elements to the left.
        #[intrinsic_for("pslldq")]
        #[intel_equivalents("_mm_bslli_si128", "_mm_slli_si128")]
        pub fn move_l_by_s8x16<ELEMS: u8>(x: s8x16) -> s8x16;
    });

    // Move byte elements in a 16-byte vector right by a constant.
    defn_simd_shared!("sse2", fn(T) -> R {
        simd_shuffle(T::splat(0), x, const {
            simd_slice_indices::<T>(ELEMS as usize)
        })
    }, {
        /// Move unsigned 8-bit integer elements to the right.
        #[intrinsic_for("psrldq")]
        #[intel_equivalents("_mm_bsrli_si128", "_mm_srli_si128")]
        pub fn move_r_by_u8x16<ELEMS: u8>(x: u8x16) -> u8x16;

        /// Move signed 8-bit integer elements to the right.
        #[intrinsic_for("psrldq")]
        #[intel_equivalents("_mm_bsrli_si128", "_mm_srli_si128")]
        pub fn move_r_by_s8x16<ELEMS: u8>(x: s8x16) -> s8x16;
    });

    // Mask bit extraction.
    defn_simd_manual!("sse2", {
        /// Extract the most significant bit from unsigned 8-bit integers.
        #[intrinsic_for("pmovmskb")]
        #[intel_equivalents("_mm_movemask_epi8")]
        pub fn bitmask_u8x16(x: u8x16) -> u16 {
            simd_bitmask(simd_ge::<_, u8x16>(x, u8x16::splat(0x80)))
        }

        /// Extract the sign bit from signed 8-bit integers.
        #[intrinsic_for("pmovmskb")]
        #[intel_equivalents("_mm_movemask_epi8")]
        pub fn bitmask_s8x16(x: s8x16) -> u16 {
            simd_bitmask(simd_lt::<_, s8x16>(x, s8x16::splat(0)))
        }
    });
}