npsimd 0.3.0

//! Intrinsics for SSE2.

use core::ops::*;

use super::*;

/// SSE2 (Streaming SIMD Extensions 2).
///
/// SSE2 is the second extension in the SSE generation, released in 2000 with
/// the Intel Pentium 4.  It is available on practically every x86 CPU you can
/// find nowadays.  While it is quite old, it provides a solid foundation of
/// integer vector operations that is sufficient for most algorithms.  If you
/// are designing a vectorized algorithm by hand, SSE2 is the bare minimum.
///
/// # Instructions
///
/// These instructions are implemented by [`Vector`].
///
/// Converting between general-purpose and vector registers:
///
/// - [ ] `movd`
/// - [ ] `movq`
///
/// Element insertion and extraction:
///
/// - [ ] `pextrw`
/// - [ ] `pinsrw`
///
/// Loading from and storing to memory:
///
/// - [x] `movd`
/// - [x] `movq`
/// - [x] `movdqa`
/// - [x] `movdqu`
///
/// Basic integer operations:
///
/// - [x] `paddb`
/// - [ ] `paddw`
/// - [ ] `paddd`
/// - [ ] `paddq`
///
/// - [x] `psubb`
/// - [ ] `psubw`
/// - [ ] `psubd`
/// - [ ] `psubq`
///
/// - [ ] `paddsb`
/// - [ ] `paddsw`
///
/// - [ ] `psubsb`
/// - [ ] `psubsw`
///
/// - [ ] `pmulhuw`
/// - [ ] `pmulhw`
/// - [ ] `pmullw`
/// - [ ] `pmuludq`
///
/// - [x] `pmaxub`
/// - [x] `pmaxsw`
///
/// - [x] `pminub`
/// - [x] `pminsw`
///
/// - [ ] `pavgb`
/// - [ ] `pavgw`
///
/// Bitwise operations:
///
/// - [x] `pand`
/// - [ ] `pandn`
/// - [ ] `por`
/// - [ ] `pxor`
///
/// - [ ] `psllw`
/// - [ ] `pslld`
/// - [ ] `psllq`
///
/// - [x] `psrlw`
/// - [x] `psraw`
/// - [x] `psrld`
/// - [x] `psrad`
/// - [x] `psrlq`
///
/// Specialized integer operations:
///
/// - [ ] `psadbw`
/// - [ ] `pmaddwd`
///
/// Integer comparisons:
///
/// - [x] `pcmpeqb`
/// - [x] `pcmpeqw`
/// - [x] `pcmpeqd`
///
/// - [x] `pcmpgtb`
/// - [x] `pcmpgtw`
/// - [x] `pcmpgtd`
///
/// Data shuffling:
///
/// - [ ] `pshufd`
/// - [ ] `pshuflw`
/// - [ ] `pshufhw`
///
/// - [x] `pslldq`
/// - [x] `psrldq`
///
/// - [ ] `packuswb`
/// - [ ] `packssdw`
/// - [ ] `packsswb`
///
/// - [ ] `punpcklbw`
/// - [ ] `punpcklwd`
/// - [ ] `punpckldq`
/// - [ ] `punpcklqdq`
///
/// - [ ] `punpckhbw`
/// - [ ] `punpckhwd`
/// - [ ] `punpckhdq`
/// - [ ] `punpckhqdq`
///
/// Bitmasks:
///
/// - [ ] `pmovmskb`
#[derive(Copy, Clone)]
pub struct SSE2(());

impl SSE2 {
    /// Construct a new [`SSE2`].
    #[cfg(any(doc, target_feature = "sse2"))]
    pub const fn new() -> Self {
        Self(())
    }
}

#[cfg(any(doc, target_feature = "sse2"))]
impl Default for SSE2 {
    fn default() -> Self {
        Self::new()
    }
}

impl Features<SSE> for SSE2 {
    fn query(support: &RuntimeSupport) -> Option<Self> {
        support.sse2().then_some(Self(()))
    }
}

impl Feature<SSE> for SSE2 {}

// Memory loads and stores.

macro_rules! impl_simd_movable {
    ([$elem:ty; $size:literal]) => {
        unsafe impl Movable<SSE, $size> for $elem {
            type Feature = SSE2;

            #[inline]
            #[target_feature(enable = "sse2")]
            unsafe fn load(ptr: &[Self; $size]) -> Self::Primitive {
                core::ptr::read_unaligned(ptr as *const _ as *const _)
            }

            #[inline]
            #[target_feature(enable = "sse2")]
            unsafe fn load_aligned(
                ptr: &Vector<[Self; $size]>
            ) -> Self::Primitive {
                (*ptr).primitive
            }

            #[inline]
            #[target_feature(enable = "sse2")]
            unsafe fn store(this: Self::Primitive, ptr: &mut [Self; $size]) {
                core::ptr::write_unaligned(ptr as *mut _ as *mut _, this)
            }

            #[inline]
            #[target_feature(enable = "sse2")]
            unsafe fn store_aligned(
                this: Self::Primitive,
                ptr: &mut Vector<[Self; $size]>,
            ) {
                *ptr = Vector::from_primitive(this)
            }
        }
    };
}

// Loading and storing the low 4 bytes of a vector from and into memory.

impl_simd_movable!([u8; 4]);
impl_simd_movable!([u16; 2]);
impl_simd_movable!([u32; 1]);

impl_simd_movable!([i8; 4]);
impl_simd_movable!([i16; 2]);
impl_simd_movable!([i32; 1]);

impl_simd_movable!([Mask8; 4]);
impl_simd_movable!([Mask16; 2]);
impl_simd_movable!([Mask32; 1]);

// Loading and storing the low 8 bytes of a vector from and into memory.

impl_simd_movable!([u8; 8]);
impl_simd_movable!([u16; 4]);
impl_simd_movable!([u32; 2]);
impl_simd_movable!([u64; 1]);

impl_simd_movable!([i8; 8]);
impl_simd_movable!([i16; 4]);
impl_simd_movable!([i32; 2]);
impl_simd_movable!([i64; 1]);

impl_simd_movable!([Mask8; 8]);
impl_simd_movable!([Mask16; 4]);
impl_simd_movable!([Mask32; 2]);
impl_simd_movable!([Mask64; 1]);

// Loading and storing a 16-byte vector from and to memory.

impl_simd_movable!([u8; 16]);
impl_simd_movable!([u16; 8]);
impl_simd_movable!([u32; 4]);
impl_simd_movable!([u64; 2]);

impl_simd_movable!([i8; 16]);
impl_simd_movable!([i16; 8]);
impl_simd_movable!([i32; 4]);
impl_simd_movable!([i64; 2]);

impl_simd_movable!([Mask8; 16]);
impl_simd_movable!([Mask16; 8]);
impl_simd_movable!([Mask32; 4]);
impl_simd_movable!([Mask64; 2]);

// Add corresponding elements of 16-byte vectors.

impl_simd!(SSE[SSE2: "sse2"], [u8; 16]: Add {
    type Output = Self;

    /// Add corresponding unsigned 8-bit integers.
    #[intrinsic_for("paddb")]
    #[intel_equivalents("_mm_add_epi8")]
    fn add(self => this, that: Self) -> Self {
        simd_add(this.primitive, that.primitive)
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i8; 16]: Add {
    type Output = Self;

    /// Add corresponding signed 8-bit integers.
    #[intrinsic_for("paddb")]
    #[intel_equivalents("_mm_add_epi8")]
    fn add(self => this, that: Self) -> Self {
        simd_add(this.primitive, that.primitive)
    }
});

// Add a constant to elements of a 16-byte vector.

impl_simd!(SSE[SSE2: "sse2"]<Const: Imm<u8>>, [u8; 16]: Add<Const> {
    type Output = Self;

    /// Add a constant to unsigned 8-bit integers.
    #[intrinsic_for("paddb")]
    #[intel_equivalents("_mm_add_epi8")]
    fn add(self => this, _that: Const) -> Self {
        simd_add(this.primitive, vector![Const::VAL; 16].primitive)
    }
});

impl_simd!(SSE[SSE2: "sse2"]<Const: Imm<i8>>, [i8; 16]: Add<Const> {
    type Output = Self;

    /// Add a constant to unsigned 8-bit integers.
    #[intrinsic_for("paddb")]
    #[intel_equivalents("_mm_add_epi8")]
    fn add(self => this, _that: Const) -> Self {
        simd_add(this.primitive, vector![Const::VAL; 16].primitive)
    }
});

// Subtract corresponding elements of 16-byte vectors.

impl_simd!(SSE[SSE2: "sse2"], [u8; 16]: Sub {
    type Output = Self;

    /// Subtract corresponding unsigned 8-bit integers.
    #[intrinsic_for("psubb")]
    #[intel_equivalents("_mm_sub_epi8")]
    fn sub(self => this, that: Self) -> Self {
        simd_sub(this.primitive, that.primitive)
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i8; 16]: Sub {
    type Output = Self;

    /// Subtract corresponding signed 8-bit integers.
    #[intrinsic_for("psubb")]
    #[intel_equivalents("_mm_sub_epi8")]
    fn sub(self => this, that: Self) -> Self {
        simd_sub(this.primitive, that.primitive)
    }
});

// Subtract a constant from elements of a 16-byte vector.

impl_simd!(SSE[SSE2: "sse2"]<Const: Imm<u8>>, [u8; 16]: Sub<Const> {
    type Output = Self;

    /// Subtract a constant from unsigned 8-bit integers.
    #[intrinsic_for("psubb")]
    #[intel_equivalents("_mm_sub_epi8")]
    fn sub(self => this, _that: Const) -> Self {
        simd_sub(this.primitive, vector![Const::VAL; 16].primitive)
    }
});

impl_simd!(SSE[SSE2: "sse2"]<Const: Imm<i8>>, [i8; 16]: Sub<Const> {
    type Output = Self;

    /// Subtract a constant from unsigned 8-bit integers.
    #[intrinsic_for("psubb")]
    #[intel_equivalents("_mm_sub_epi8")]
    fn sub(self => this, _that: Const) -> Self {
        simd_sub(this.primitive, vector![Const::VAL; 16].primitive)
    }
});

// Negate elements of a 16-byte vector.

impl_simd!(SSE[SSE2: "sse2"], [u8; 16]: Neg {
    type Output = Self;

    /// Negate unsigned 8-bit integers.
    ///
    /// This evaluates to a `PSUBB`, a subtraction from 0 (i.e. 256).
    fn neg(self => this) -> Self {
        this.cast::<i8, 16>().neg().cast::<u8, 16>().primitive
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i8; 16]: Neg {
    type Output = Self;

    /// Negate signed 8-bit integers.
    ///
    /// This evaluates to a `PSUBB`, a subtraction from 0.
    fn neg(self => this) -> Self {
        simd_neg(this.primitive)
    }
});

// Bitwise AND corresponding elements of 16-byte vectors.

impl_simd!(SSE[SSE2: "sse2"], [u8; 16]: BitAnd {
    type Output = Self;

    /// Bitwise AND corresponding unsigned 8-bit integers.
    #[intrinsic_for("pand")]
    #[intel_equivalents("_mm_and_si128")]
    fn bitand(self => this, that: Self) -> Self {
        simd_and(this.primitive, that.primitive)
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i8; 16]: BitAnd {
    type Output = Self;

    /// Bitwise AND corresponding signed 8-bit integers.
    #[intrinsic_for("pand")]
    #[intel_equivalents("_mm_and_si128")]
    fn bitand(self => this, that: Self) -> Self {
        simd_and(this.primitive, that.primitive)
    }
});

impl_simd!(SSE[SSE2: "sse2"], [Mask8; 16]: BitAnd {
    type Output = Self;

    /// Bitwise AND corresponding 8-bit masks.
    #[intrinsic_for("pand")]
    #[intel_equivalents("_mm_and_si128")]
    fn bitand(self => this, that: Self) -> Self {
        simd_and(this.primitive, that.primitive)
    }
});

impl_simd!(SSE[SSE2: "sse2"], [u8; 16]: BitAnd<Vector<[Mask8; 16], SSE, FS>> {
    type Output = Self;

    /// Zero out unsigned 8-bit integers for which the mask is inactive.
    #[intrinsic_for("pand")]
    #[intel_equivalents("_mm_and_si128")]
    fn bitand(
        self => this,
        mask: Vector<[Mask8; 16], SSE, FS>,
    ) -> Self {
        simd_and(this.primitive, mask.cast::<u8, 16>().primitive)
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i8; 16]: BitAnd<Vector<[Mask8; 16], SSE, FS>> {
    type Output = Self;

    /// Zero out unsigned 8-bit integers for which the mask is inactive.
    #[intrinsic_for("pand")]
    #[intel_equivalents("_mm_and_si128")]
    fn bitand(
        self => this,
        mask: Vector<[Mask8; 16], SSE, FS>,
    ) -> Self {
        simd_and(this.primitive, mask.cast::<i8, 16>().primitive)
    }
});

// Bitwise AND a constant from elements of a 16-byte vector.

impl_simd!(SSE[SSE2: "sse2"]<Const: Imm<u8>>, [u8; 16]: BitAnd<Const> {
    type Output = Self;

    /// Bitwise AND a constant with unsigned 8-bit integers.
    #[intrinsic_for("pand")]
    #[intel_equivalents("_mm_and_si128")]
    fn bitand(self => this, _that: Const) -> Self {
        simd_and(this.primitive, vector![Const::VAL; 16].primitive)
    }
});

impl_simd!(SSE[SSE2: "sse2"]<Const: Imm<i8>>, [i8; 16]: BitAnd<Const> {
    type Output = Self;

    /// Bitwise AND a constant with unsigned 8-bit integers.
    #[intrinsic_for("pand")]
    #[intel_equivalents("_mm_and_si128")]
    fn bitand(self => this, _that: Const) -> Self {
        simd_and(this.primitive, vector![Const::VAL; 16].primitive)
    }
});

// Right-shift elements of a 16-byte vector by a constant.

impl_simd!(SSE[SSE2: "sse2"]<Const: Imm<u8>>, [u8; 16]: Shr<Const> {
    type Output = Self;

    /// Right-shift unsigned 8-bit integers by a constant.
    ///
    /// Officially, there is no `PSRLB` instruction, which would perform a right
    /// shift within bytes.  This instruction performs `PSRLD` -- a right shift
    /// over 32-bit integers -- then bitwise ANDs the result using `PAND`.
    ///
    /// The shift value must not be greater than 8.  If it is 8, a zero vector
    /// is returned directly.
    fn shr(self => this, that: Const) -> Self {
        const_assert!(Const::VAL <= 8);
        if Const::VAL < 8 {
            imm! {
                type Mask<Const: Imm<u8>>: u8 = {
                    if Const::VAL < 8 { 0xFF >> Const::VAL } else { 0 }
                };
            }

            this.cast::<u32, 4>()
                .shr(that)
                .cast::<u8, 16>()
                .bitand(Mask::<Const>::new())
                .primitive
        } else {
            vector![0u8; 16].primitive
        }
    }
});

impl_simd!(SSE[SSE2: "sse2"]<Const: Imm<u8>>, [u16; 8]: Shr<Const> {
    type Output = Self;

    /// Right-shift unsigned 16-bit integers by a constant.
    ///
    /// The shift value must not be greater than 16.  If it is 16, a zero vector
    /// is returned directly.
    #[intrinsic_for("psrlw")]
    #[intel_equivalents("_mm_srli_epi16")]
    fn shr(self => this, _that: Const) -> Self {
        const_assert!(Const::VAL <= 16);
        if Const::VAL < 16 {
            simd_shr(this.primitive, vector![Const::VAL as u16; 8].primitive)
        } else {
            vector![0u16; 8].primitive
        }
    }
});

impl_simd!(SSE[SSE2: "sse2"]<Const: Imm<u8>>, [u32; 4]: Shr<Const> {
    type Output = Self;

    /// Right-shift unsigned 32-bit integers by a constant.
    ///
    /// The shift value must not be greater than 32.  If it is 32, a zero vector
    /// is returned directly.
    #[intrinsic_for("psrld")]
    #[intel_equivalents("_mm_srli_epi32")]
    fn shr(self => this, _that: Const) -> Self {
        const_assert!(Const::VAL <= 32);
        if Const::VAL < 32 {
            simd_shr(this.primitive, vector![Const::VAL as u32; 4].primitive)
        } else {
            vector![0u32; 4].primitive
        }
    }
});

impl_simd!(SSE[SSE2: "sse2"]<Const: Imm<u8>>, [u64; 2]: Shr<Const> {
    type Output = Self;

    /// Right-shift unsigned 64-bit integers by a constant.
    ///
    /// The shift value must not be greater than 64.  If it is 64, a zero vector
    /// is returned directly.
    #[intrinsic_for("psrld")]
    #[intel_equivalents("_mm_srli_epi64")]
    fn shr(self => this, _that: Const) -> Self {
        const_assert!(Const::VAL <= 64);
        if Const::VAL < 64 {
            simd_shr(this.primitive, vector![Const::VAL as u64; 2].primitive)
        } else {
            vector![0u64; 2].primitive
        }
    }
});

impl_simd!(SSE[SSE2: "sse2"]<Const: Imm<u8>>, [i16; 8]: Shr<Const> {
    type Output = Self;

    /// Right-shift signed 16-bit integers by a constant.
    ///
    /// The shift value must not be greater than 16.
    #[intrinsic_for("psraw")]
    #[intel_equivalents("_mm_srai_epi16")]
    fn shr(self => this, _that: Const) -> Self {
        const_assert!(Const::VAL <= 16);
        if Const::VAL < 16 {
            simd_shr(this.primitive, vector![Const::VAL as i16; 8].primitive)
        } else {
            simd_lt(this.primitive, vector![0i16; 8].primitive)
        }
    }
});

impl_simd!(SSE[SSE2: "sse2"]<Const: Imm<u8>>, [i32; 4]: Shr<Const> {
    type Output = Self;

    /// Right-shift unsigned 32-bit integers by a constant.
    ///
    /// The shift value must not be greater than 32.
    #[intrinsic_for("psrad")]
    #[intel_equivalents("_mm_srai_epi32")]
    fn shr(self => this, _that: Const) -> Self {
        const_assert!(Const::VAL <= 32);
        if Const::VAL < 32 {
            simd_shr(this.primitive, vector![Const::VAL as i32; 4].primitive)
        } else {
            simd_lt(this.primitive, vector![0i32; 4].primitive)
        }
    }
});

// Compare corresponding elements of 16-byte vectors.

impl_simd!(SSE[SSE2: "sse2"], [u8; 16] {
    /// Test that corresponding unsigned 8-bit integers are equal.
    #[intrinsic_for("pcmpeqb")]
    #[intel_equivalents("_mm_cmpeq_epi8")]
    pub fn test_eq(self => this, that: Self) -> Vector<[Mask8; 16], SSE, FS> {
        Vector::from_primitive({
            simd_eq::<_, i8x16>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding unsigned 8-bit integers are not equal.
    ///
    /// This evaluates to `PCMPEQB` + `PXOR`.
    pub fn test_ne(self => this, that: Self) -> Vector<[Mask8; 16], SSE, FS> {
        Vector::from_primitive({
            simd_ne::<_, i8x16>(this.primitive, that.primitive)
        }).with_features(this.features)
    }
});

impl_simd!(SSE[SSE2: "sse2"], [u16; 8] {
    /// Test that corresponding unsigned 16-bit integers are equal.
    #[intrinsic_for("pcmpeqw")]
    #[intel_equivalents("_mm_cmpeq_epi16")]
    pub fn test_eq(self => this, that: Self) -> Vector<[Mask16; 8], SSE, FS> {
        Vector::from_primitive({
            simd_eq::<_, i16x8>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding unsigned 16-bit integers are not equal.
    ///
    /// This evaluates to `PCMPEQW` + `PXOR`.
    pub fn test_ne(self => this, that: Self) -> Vector<[Mask16; 8], SSE, FS> {
        Vector::from_primitive({
            simd_ne::<_, i16x8>(this.primitive, that.primitive)
        }).with_features(this.features)
    }
});

impl_simd!(SSE[SSE2: "sse2"], [u32; 4] {
    /// Test that corresponding unsigned 32-bit integers are equal.
    #[intrinsic_for("pcmpeqd")]
    #[intel_equivalents("_mm_cmpeq_epi32")]
    pub fn test_eq(self => this, that: Self) -> Vector<[Mask32; 4], SSE, FS> {
        Vector::from_primitive({
            simd_eq::<_, i32x4>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding unsigned 32-bit integers are not equal.
    ///
    /// This evaluates to `PCMPEQD` + `PXOR`.
    pub fn test_ne(self => this, that: Self) -> Vector<[Mask32; 4], SSE, FS> {
        Vector::from_primitive({
            simd_ne::<_, i32x4>(this.primitive, that.primitive)
        }).with_features(this.features)
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i8; 16] {
    /// Test that corresponding signed 8-bit integers are equal.
    #[intrinsic_for("pcmpeqb")]
    #[intel_equivalents("_mm_cmpeq_epi8")]
    pub fn test_eq(self => this, that: Self) -> Vector<[Mask8; 16], SSE, FS> {
        Vector::from_primitive({
            simd_eq::<_, i8x16>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 8-bit integers are not equal.
    ///
    /// This evaluates to `PCMPEQB` + `PXOR`.
    pub fn test_ne(self => this, that: Self) -> Vector<[Mask8; 16], SSE, FS> {
        Vector::from_primitive({
            simd_ne::<_, i8x16>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 8-bit integers are greater.
    #[intrinsic_for("pcmpgtb")]
    #[intel_equivalents("_mm_cmpgt_epi8")]
    pub fn test_gt(self => this, that: Self) -> Vector<[Mask8; 16], SSE, FS> {
        Vector::from_primitive({
            simd_gt::<_, i8x16>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 8-bit integers are greater or equal.
    ///
    /// This evaluates to `PCMPGTB` + `PXOR`.
    pub fn test_ge(self => this, that: Self) -> Vector<[Mask8; 16], SSE, FS> {
        Vector::from_primitive({
            simd_ge::<_, i8x16>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 8-bit integers are lesser.
    #[intrinsic_for("pcmpgtb")]
    #[intel_equivalents("_mm_cmpgt_epi8")]
    pub fn test_lt(self => this, that: Self) -> Vector<[Mask8; 16], SSE, FS> {
        Vector::from_primitive({
            simd_lt::<_, i8x16>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 8-bit integers are lesser or equal.
    ///
    /// This evaluates to `PCMPGTB` + `PXOR`.
    pub fn test_le(self => this, that: Self) -> Vector<[Mask8; 16], SSE, FS> {
        Vector::from_primitive({
            simd_le::<_, i8x16>(this.primitive, that.primitive)
        }).with_features(this.features)
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i16; 8] {
    /// Test that corresponding signed 16-bit integers are equal.
    #[intrinsic_for("pcmpeqw")]
    #[intel_equivalents("_mm_cmpeq_epi16")]
    pub fn test_eq(self => this, that: Self) -> Vector<[Mask16; 8], SSE, FS> {
        Vector::from_primitive({
            simd_eq::<_, i16x8>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test tha corresponding signed 16-bit integers are not equal.
    ///
    /// This evaluates to `PCMPEQW` + `PXOR`.
    pub fn test_ne(self => this, that: Self) -> Vector<[Mask16; 8], SSE, FS> {
        Vector::from_primitive({
            simd_ne::<_, i16x8>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 16-bit integers are greater.
    #[intrinsic_for("pcmpgtw")]
    #[intel_equivalents("_mm_cmpgt_epi16")]
    pub fn test_gt(self => this, that: Self) -> Vector<[Mask16; 8], SSE, FS> {
        Vector::from_primitive({
            simd_gt::<_, i16x8>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 16-bit integers are greater or equal.
    ///
    /// This evaluates to `PCMPGTW` + `PXOR`.
    pub fn test_ge(self => this, that: Self) -> Vector<[Mask16; 8], SSE, FS> {
        Vector::from_primitive({
            simd_ge::<_, i16x8>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 16-bit integers are lesser.
    #[intrinsic_for("pcmpgtw")]
    #[intel_equivalents("_mm_cmpgt_epi16")]
    pub fn test_lt(self => this, that: Self) -> Vector<[Mask16; 8], SSE, FS> {
        Vector::from_primitive({
            simd_lt::<_, i16x8>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 16-bit integers are lesser or equal.
    ///
    /// This evaluates to `PCMPGTW` + `PXOR`.
    pub fn test_le(self => this, that: Self) -> Vector<[Mask16; 8], SSE, FS> {
        Vector::from_primitive({
            simd_le::<_, i16x8>(this.primitive, that.primitive)
        }).with_features(this.features)
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i32; 4] {
    /// Test that corresponding signed 32-bit integers are equal.
    #[intrinsic_for("pcmpeqd")]
    #[intel_equivalents("_mm_cmpeq_epi32")]
    pub fn test_eq(self => this, that: Self) -> Vector<[Mask32; 4], SSE, FS> {
        Vector::from_primitive({
            simd_eq::<_, i32x4>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 32-bit integers are not equal.
    ///
    /// This evaluates to `PCMPEQD` + `PXOR`.
    pub fn test_ne(self => this, that: Self) -> Vector<[Mask32; 4], SSE, FS> {
        Vector::from_primitive({
            simd_ne::<_, i32x4>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 32-bit integers are greater.
    #[intrinsic_for("pcmpgtd")]
    #[intel_equivalents("_mm_cmpgt_epi32")]
    pub fn test_gt(self => this, that: Self) -> Vector<[Mask32; 4], SSE, FS> {
        Vector::from_primitive({
            simd_gt::<_, i32x4>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 32-bit integers are greater or equal.
    ///
    /// This evaluates to `PCMPGTD` + `PXOR`.
    pub fn test_ge(self => this, that: Self) -> Vector<[Mask32; 4], SSE, FS> {
        Vector::from_primitive({
            simd_ge::<_, i32x4>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 32-bit integers are lesser.
    #[intrinsic_for("pcmpgtd")]
    #[intel_equivalents("_mm_cmpgt_epi32")]
    pub fn test_lt(self => this, that: Self) -> Vector<[Mask32; 4], SSE, FS> {
        Vector::from_primitive({
            simd_lt::<_, i32x4>(this.primitive, that.primitive)
        }).with_features(this.features)
    }

    /// Test that corresponding signed 32-bit integers are lesser or equal.
    ///
    /// This evaluates to `PCMPGTD` + `PXOR`.
    pub fn test_le(self => this, that: Self) -> Vector<[Mask32; 4], SSE, FS> {
        Vector::from_primitive({
            simd_le::<_, i32x4>(this.primitive, that.primitive)
        }).with_features(this.features)
    }
});

// Compare elements of a 16-byte vector to a constant.

impl_simd!(SSE[SSE2: "sse2"], [u8; 16] {
    /// Test that unsigned 8-bit integers are equal to a constant.
    #[intrinsic_for("pcmpeqb")]
    #[intel_equivalents("_mm_cmpeq_epi8")]
    pub fn test_eq_to<Const: Imm<u8>>(
        self => this, _that: Const,
    ) -> Vector<[Mask8; 16], SSE, FS> {
        this.test_eq(vector![Const::VAL; 16].with_features(this.features))
    }

    /// Test that unsigned 8-bit integers are not equal to a constant.
    ///
    /// This evaluates to `PCMPEQB` + `PXOR`.
    pub fn test_ne_to<Const: Imm<u8>>(
        self => this, _that: Const,
    ) -> Vector<[Mask8; 16], SSE, FS> {
        this.test_ne(vector![Const::VAL; 16].with_features(this.features))
    }
});

impl_simd!(SSE[SSE2: "sse2"], [u16; 8] {
    /// Test that unsigned 16-bit integers are equal to a constant.
    #[intrinsic_for("pcmpeqw")]
    #[intel_equivalents("_mm_cmpeq_epi16")]
    pub fn test_eq_to<Const: Imm<u16>>(
        self => this, _that: Const,
    ) -> Vector<[Mask16; 8], SSE, FS> {
        this.test_eq(vector![Const::VAL; 8].with_features(this.features))
    }

    /// Test that unsigned 16-bit integers are not equal to a constant.
    ///
    /// This evaluates to `PCMPEQW` + `PXOR`.
    pub fn test_ne_to<Const: Imm<u16>>(
        self => this, _that: Const,
    ) -> Vector<[Mask16; 8], SSE, FS> {
        this.test_ne(vector![Const::VAL; 8].with_features(this.features))
    }
});

impl_simd!(SSE[SSE2: "sse2"], [u32; 4] {
    /// Test that unsigned 32-bit integers are equal to a constant.
    #[intrinsic_for("pcmpeqd")]
    #[intel_equivalents("_mm_cmpeq_epi32")]
    pub fn test_eq_to<Const: Imm<u32>>(
        self => this, _that: Const,
    ) -> Vector<[Mask32; 4], SSE, FS> {
        this.test_eq(vector![Const::VAL; 4].with_features(this.features))
    }

    /// Test that unsigned 32-bit integers are not equal to a constant.
    ///
    /// This evaluates to `PCMPEQD` + `PXOR`.
    pub fn test_ne_to<Const: Imm<u32>>(
        self => this, _that: Const,
    ) -> Vector<[Mask32; 4], SSE, FS> {
        this.test_ne(vector![Const::VAL; 4].with_features(this.features))
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i8; 16] {
    /// Test that signed 8-bit integers are equal to a constant.
    #[intrinsic_for("pcmpeqb")]
    #[intel_equivalents("_mm_cmpeq_epi8")]
    pub fn test_eq_to<Const: Imm<i8>>(
        self => this, _that: Const,
    ) -> Vector<[Mask8; 16], SSE, FS> {
        this.test_eq(vector![Const::VAL; 16].with_features(this.features))
    }

    /// Test that signed 8-bit integers are not equal to a constant.
    ///
    /// This evaluates to `PCMPEQB` + `PXOR`.
    pub fn test_ne_to<Const: Imm<i8>>(
        self => this, _that: Const,
    ) -> Vector<[Mask8; 16], SSE, FS> {
        this.test_ne(vector![Const::VAL; 16].with_features(this.features))
    }

    /// Test that signed 8-bit integers are greater than a constant.
    #[intrinsic_for("pcmpgtb")]
    #[intel_equivalents("_mm_cmpgt_epi8")]
    pub fn test_gt_to<Const: Imm<i8>>(
        self => this, _that: Const,
    ) -> Vector<[Mask8; 16], SSE, FS> {
        this.test_gt(vector![Const::VAL; 16].with_features(this.features))
    }

    /// Test that signed 8-bit integers are greater than or equal to a constant.
    ///
    /// This evaluates to `PCMPGTB` + `PXOR`.
    pub fn test_ge_to<Const: Imm<i8>>(
        self => this, _that: Const,
    ) -> Vector<[Mask8; 16], SSE, FS> {
        this.test_ge(vector![Const::VAL; 16].with_features(this.features))
    }

    /// Test that signed 8-bit integers are lesser than or equal to a constant.
    #[intrinsic_for("pcmpgtb")]
    #[intel_equivalents("_mm_cmpgt_epi8")]
    pub fn test_lt_to<Const: Imm<i8>>(
        self => this, _that: Const,
    ) -> Vector<[Mask8; 16], SSE, FS> {
        this.test_lt(vector![Const::VAL; 16].with_features(this.features))
    }

    /// Test that signed 8-bit integers are lesser than or equal to a constant.
    ///
    /// This evaluates to `PCMPGTB` + `PXOR`.
    pub fn test_le_to<Const: Imm<i8>>(
        self => this, _that: Const,
    ) -> Vector<[Mask8; 16], SSE, FS> {
        this.test_le(vector![Const::VAL; 16].with_features(this.features))
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i16; 8] {
    /// Test that signed 16-bit integers are equal to a constant.
    #[intrinsic_for("pcmpeqw")]
    #[intel_equivalents("_mm_cmpeq_epi16")]
    pub fn test_eq_to<Const: Imm<i16>>(
        self => this, _that: Const,
    ) -> Vector<[Mask16; 8], SSE, FS> {
        this.test_eq(vector![Const::VAL; 8].with_features(this.features))
    }

    /// Test that signed 16-bit integers are not equal to a constant.
    ///
    /// This evaluates to `PCMPEQW` + `PXOR`.
    pub fn test_ne_to<Const: Imm<i16>>(
        self => this, _that: Const,
    ) -> Vector<[Mask16; 8], SSE, FS> {
        this.test_ne(vector![Const::VAL; 8].with_features(this.features))
    }

    /// Test that signed 16-bit integers are greater than a constant.
    #[intrinsic_for("pcmpgtw")]
    #[intel_equivalents("_mm_cmpgt_epi16")]
    pub fn test_gt_to<Const: Imm<i16>>(
        self => this, _that: Const,
    ) -> Vector<[Mask16; 8], SSE, FS> {
        this.test_gt(vector![Const::VAL; 8].with_features(this.features))
    }

    /// Test that signed 16-bit integers are greater than or equal to a
    /// constant.
    ///
    /// This evaluates to `PCMPGTW` + `PXOR`.
    pub fn test_ge_to<Const: Imm<i16>>(
        self => this, _that: Const,
    ) -> Vector<[Mask16; 8], SSE, FS> {
        this.test_ge(vector![Const::VAL; 8].with_features(this.features))
    }

    /// Test that signed 16-bit integers are lesser than a constant.
    #[intrinsic_for("pcmpgtw")]
    #[intel_equivalents("_mm_cmpgt_epi16")]
    pub fn test_lt_to<Const: Imm<i16>>(
        self => this, _that: Const,
    ) -> Vector<[Mask16; 8], SSE, FS> {
        this.test_lt(vector![Const::VAL; 8].with_features(this.features))
    }

    /// Test that signed 16-bit integers are lesser than or equal to a constant.
    ///
    /// This evaluates to `PCMPGTW` + `PXOR`.
    pub fn test_le_to<Const: Imm<i16>>(
        self => this, _that: Const,
    ) -> Vector<[Mask16; 8], SSE, FS> {
        this.test_le(vector![Const::VAL; 8].with_features(this.features))
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i32; 4] {
    /// Test that signed 32-bit integers are equal to a constant.
    #[intrinsic_for("pcmpeqd")]
    #[intel_equivalents("_mm_cmpeq_epi32")]
    pub fn test_eq_to<Const: Imm<i32>>(
        self => this, _that: Const,
    ) -> Vector<[Mask32; 4], SSE, FS> {
        this.test_eq(vector![Const::VAL; 4].with_features(this.features))
    }

    /// Test that signed 32-bit integers are not equal to a constant.
    ///
    /// This evaluates to `PCMPEQD` + `PXOR`.
    pub fn test_ne_to<Const: Imm<i32>>(
        self => this, _that: Const,
    ) -> Vector<[Mask32; 4], SSE, FS> {
        this.test_ne(vector![Const::VAL; 4].with_features(this.features))
    }

    /// Test that signed 32-bit integers are greater than a constant.
    #[intrinsic_for("pcmpgtd")]
    #[intel_equivalents("_mm_cmpgt_epi32")]
    pub fn test_gt_to<Const: Imm<i32>>(
        self => this, _that: Const,
    ) -> Vector<[Mask32; 4], SSE, FS> {
        this.test_gt(vector![Const::VAL; 4].with_features(this.features))
    }

    /// Test that signed 32-bit integers are greater than or equal to a
    /// constant.
    ///
    /// This evaluates to `PCMPGTD` + `PXOR`.
    pub fn test_ge_to<Const: Imm<i32>>(
        self => this, _that: Const,
    ) -> Vector<[Mask32; 4], SSE, FS> {
        this.test_ge(vector![Const::VAL; 4].with_features(this.features))
    }

    /// Test that signed 32-bit integers are lesser than a constant.
    #[intrinsic_for("pcmpgtd")]
    #[intel_equivalents("_mm_cmpgt_epi32")]
    pub fn test_lt_to<Const: Imm<i32>>(
        self => this, _that: Const,
    ) -> Vector<[Mask32; 4], SSE, FS> {
        this.test_lt(vector![Const::VAL; 4].with_features(this.features))
    }

    /// Test that signed 32-bit integers are lesser than or equal to a constant.
    ///
    /// This evaluates to `PCMPGTD` + `PXOR`.
    pub fn test_le_to<Const: Imm<i32>>(
        self => this, _that: Const,
    ) -> Vector<[Mask32; 4], SSE, FS> {
        this.test_le(vector![Const::VAL; 4].with_features(this.features))
    }
});

// Test the elements of 16-byte vectors for belonging within ranges.

impl_simd!(SSE[SSE2: "sse2"], [u8; 16] {
    /// Test that unsigned 8-bit integers fit within an inclusive range.
    ///
    /// This evaluates to `PADDB` + `PCMPGTB`.
    pub fn test_within
    <Min: Imm<u8>, Max: Imm<u8>>
    (self => this, _min: Min, _max: Max)
    -> Vector<[Mask8; 16], SSE, FS> {
        const_assert!(Min::VAL < Max::VAL);
        const_assert!(
            <Min: Imm<u8>, Max: Imm<u8>>
            Min::VAL != u8::MIN || Max::VAL != u8::MAX);

        imm! {
            type Off<Max: Imm<u8>>: i8 = {
                i8::MAX.wrapping_sub(Max::VAL as i8)
            };

            type Bar<Min: Imm<u8>, Max: Imm<u8>>: i8 = {
                i8::MAX.wrapping_sub((1 + Max::VAL - Min::VAL) as i8)
            };
        }

        this.cast::<i8, 16>()
            .add(Off::<Max>::new())
            .test_gt_to(Bar::<Min, Max>::new())
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i8; 16] {
    /// Test that signed 8-bit integers fit within an inclusive range.
    ///
    /// This evaluates to `PADDB` + `PCMPGTB`.
    pub fn test_within
    <Min: Imm<i8>, Max: Imm<i8>>
    (self => this, _min: Min, _max: Max)
    -> Vector<[Mask8; 16], SSE, FS> {
        const_assert!(Min::VAL < Max::VAL);
        const_assert!(
            <Min: Imm<i8>, Max: Imm<i8>>
            Min::VAL != i8::MIN || Max::VAL != i8::MAX);

        imm! {
            type Off<Max: Imm<i8>>: i8 = {
                i8::MAX.wrapping_sub(Max::VAL)
            };

            type Bar<Min: Imm<i8>, Max: Imm<i8>>: i8 = {
                i8::MAX.wrapping_sub(
                    1i8.wrapping_add(Max::VAL)
                       .wrapping_sub(Min::VAL))
            };
        }

        this.add(Off::<Max>::new())
            .test_gt_to(Bar::<Min, Max>::new())
    }
});

// Select the maximum/minimum of corresponding elements of 16-byte vectors.

impl_simd!(SSE[SSE2: "sse2"], [u8; 16] {
    /// Select the maximum of corresponding unsigned 8-bit integers.
    #[intrinsic_for("pmaxub")]
    #[intel_equivalents("_mm_max_epu8")]
    pub fn max(self => this, that: Self) -> Self {
        simd_max::<_, i8x16>(this.primitive, that.primitive)
    }

    /// Select the minimum of corresponding unsigned 8-bit integers.
    #[intrinsic_for("pminub")]
    #[intel_equivalents("_mm_min_epu8")]
    pub fn min(self => this, that: Self) -> Self {
        simd_min::<_, i8x16>(this.primitive, that.primitive)
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i16; 8] {
    /// Select the maximum of corresponding signed 16-bit integers.
    #[intrinsic_for("pmaxsw")]
    #[intel_equivalents("_mm_max_epi16")]
    pub fn max(self => this, that: Self) -> Self {
        simd_max::<_, i16x8>(this.primitive, that.primitive)
    }

    /// Select the minimum of corresponding signed 16-bit integers.
    #[intrinsic_for("pminsw")]
    #[intel_equivalents("_mm_min_epi16")]
    pub fn min(self => this, that: Self) -> Self {
        simd_min::<_, i16x8>(this.primitive, that.primitive)
    }
});

// Select the maximum/minimum of elements of 16-byte vectors with a constant.

impl_simd!(SSE[SSE2: "sse2"], [u8; 16] {
    /// Select the maximum of each unsigned 8-bit integer with a constant.
    #[intrinsic_for("pmaxub")]
    #[intel_equivalents("_mm_max_epu8")]
    pub fn max_with<Const: Imm<u8>>(self => this, _that: Const) -> Self {
        this.max(vector![Const::VAL; 16].with_features(this.features)).primitive
    }

    /// Select the minimum of each unsigned 8-bit integer with a constant.
    #[intrinsic_for("pminub")]
    #[intel_equivalents("_mm_min_epu8")]
    pub fn min_with<Const: Imm<u8>>(self => this, _that: Const) -> Self {
        this.min(vector![Const::VAL; 16].with_features(this.features)).primitive
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i16; 8] {
    /// Select the maximum of each signed 16-bit integer with a constant.
    #[intrinsic_for("pmaxsw")]
    #[intel_equivalents("_mm_max_epi16")]
    pub fn max_with<Const: Imm<i16>>(self => this, _that: Const) -> Self {
        this.max(vector![Const::VAL; 8].with_features(this.features)).primitive
    }

    /// Select the minimum of each signed 16-bit integer with a constant.
    #[intrinsic_for("pminsw")]
    #[intel_equivalents("_mm_min_epi16")]
    pub fn min_with<Const: Imm<i16>>(self => this, _that: Const) -> Self {
        this.min(vector![Const::VAL; 8].with_features(this.features)).primitive
    }
});

// Move elements in 16-byte vectors left or right by a constant offsets.

impl_simd!(SSE[SSE2: "sse2"], [u8; 16] {
    /// Move unsigned 8-bit integers left by a constant amount.
    #[intrinsic_for("pslldq")]
    #[intel_equivalents("_mm_slli_si128", "_mm_bslli_si128")]
    pub fn move_l_by<Const: Imm<u8>>(self => this, _num: Const) -> Self {
        const_assert!(Const::VAL <= 16);
        if Const::VAL == 0 { return this.primitive; }
        if Const::VAL == 16 { return vector![0u8; 16].primitive; }
        simd_shuffle(vector![0u8; 16].primitive, this.primitive, const {
            simd_slice_indices::<16>(16 - Const::VAL as usize)
        })
    }

    /// Move unsigned 8-bit integers right by a constant amount.
    #[intrinsic_for("psrldq")]
    #[intel_equivalents("_mm_srli_si128", "_mm_bsrli_si128")]
    pub fn move_r_by<Const: Imm<u8>>(self => this, _num: Const) -> Self {
        const_assert!(Const::VAL <= 16);
        if Const::VAL == 0 { return this.primitive; }
        if Const::VAL == 16 { return vector![0u8; 16].primitive; }
        simd_shuffle(this.primitive, vector![0u8; 16].primitive, const {
            simd_slice_indices::<16>(Const::VAL as usize)
        })
    }
});

impl_simd!(SSE[SSE2: "sse2"], [i8; 16] {
    /// Move unsigned 8-bit integers left by a constant amount.
    #[intrinsic_for("pslldq")]
    #[intel_equivalents("_mm_slli_si128", "_mm_bslli_si128")]
    pub fn move_l_by<Const: Imm<u8>>(self => this, _num: Const) -> Self {
        const_assert!(Const::VAL <= 16);
        if Const::VAL == 0 { return this.primitive; }
        if Const::VAL == 16 { return vector![0i8; 16].primitive; }
        simd_shuffle(vector![0i8; 16].primitive, this.primitive, const {
            simd_slice_indices::<16>(16 - Const::VAL as usize)
        })
    }

    /// Move unsigned 8-bit integers right by a constant amount.
    #[intrinsic_for("psrldq")]
    #[intel_equivalents("_mm_srli_si128", "_mm_bsrli_si128")]
    pub fn move_r_by<Const: Imm<u8>>(self => this, _num: Const) -> Self {
        const_assert!(Const::VAL <= 16);
        if Const::VAL == 0 { return this.primitive; }
        if Const::VAL == 16 { return vector![0i8; 16].primitive; }
        simd_shuffle(this.primitive, vector![0i8; 16].primitive, const {
            simd_slice_indices::<16>(Const::VAL as usize)
        })
    }
});


#[cfg(test)]
#[cfg(target_feature = "sse2")]
mod tests {
    use core::array;
    use core::fmt::Debug;

    use super::*;

    use proptest::prelude::*;
    use proptest::bits::{BitSetLike, BitSetStrategy};
    use proptest::test_runner::TestCaseResult;

    /// The feature set type for these tests.
    type Feats = features!(SSE2);

    /// The feature set for these tests.
    const FEATS: FeatureSet<SSE, Feats> =
        FeatureSet::new((SSE2::new(), ()));

    /// The specialized vector type for these tests.
    type Vector<T> = super::Vector<T, SSE, Feats>;

    /// Construct a vector from an array.
    fn make<E, const LEN: usize>(a: [E; LEN]) -> Vector<[E; LEN]>
    where E: Element<LEN> + Movable<SSE, LEN> {
        super::Vector::load(&a, FEATS)
    }

    /// A strategy for generating vectors.
    fn vector<E, const LEN: usize>()
        -> impl Strategy<Value = Vector<[E; LEN]>>
    where E: Element<LEN> + Movable<SSE, LEN> + Arbitrary {
        proptest::array::uniform(E::arbitrary()).prop_map(make)
    }

    /// A strategy for generating vectors of bit-sets.
    fn any_vector_bits<E, const LEN: usize>()
        -> impl Strategy<Value = [E; LEN]>
    where E: Element<LEN> + BitSetLike {
        let max = core::mem::size_of::<E>() * 8;
        let elements = BitSetStrategy::new(0, max);
        proptest::array::uniform(elements)
    }

    /// Test that a unary function maps over vectors correctly.
    fn test_una_map<T, R, const LEN: usize>(
        v: [T; LEN],
        vop: impl FnOnce(Vector<[T; LEN]>) -> Vector<[R; LEN]>,
        uop: impl Fn(T) -> R
    ) -> TestCaseResult
    where T: Element<LEN> + Movable<SSE, LEN>,
          R: Element<LEN> + PartialEq + Debug {
        let x = (vop)(make(v));
        let a = v.map(uop);
        prop_assert_eq!(x.as_array(), &a);
        Ok(())
    }

    /// Test that a binary function maps over vectors correctly.
    fn test_bin_map<T, R, const LEN: usize>(
        l: [T; LEN], r: [T; LEN],
        vop: impl FnOnce(Vector<[T; LEN]>, Vector<[T; LEN]>) -> Vector<[R; LEN]>,
        bop: impl Fn(T, T) -> R
    ) -> TestCaseResult
    where T: Element<LEN> + Movable<SSE, LEN>,
          R: Element<LEN> + PartialEq + Debug {
        let x = (vop)(make(l), make(r));
        let a = array::from_fn(|i| (bop)(l[i], r[i]));
        prop_assert_eq!(x.as_array(), &a);
        Ok(())
    }

    /// Test that a binary function taking a constant maps correctly.
    fn test_bin_map_by<T, U, C: Imm<U>, const LEN: usize>(
        l: [T; LEN], r: C,
        vop: impl FnOnce(Vector<[T; LEN]>, C) -> Vector<[T; LEN]>,
        bop: impl Fn(T, U) -> T,
    ) -> TestCaseResult
    where T: Element<LEN> + Movable<SSE, LEN> + PartialEq + Debug {
        let x = (vop)(make(l), r);
        let a = array::from_fn(|i| (bop)(l[i], C::VAL));
        prop_assert_eq!(x.as_array(), &a);
        Ok(())
    }

    // Test moving 16-byte vectors to and from unaligned memory.
    //
    // We generate an overly large array and offset into it, virtually
    // guaranteeing that we will test an unaligned offset at some point.
    macro_rules! test_load_store {
        ([$elem:ty; $size:literal]: {
            fn $load:ident;
            fn $load_aligned:ident;
            fn $store:ident;
            fn $store_aligned:ident;
        }) => {
            proptest! {
                #[test]
                fn $load(mem: [$elem; 2 * $size], off in 0usize .. $size) {
                    let mem: &[$elem; $size] = (&mem[off ..][.. $size]).try_into().unwrap();
                    assert_eq!(Vector::load(mem, FEATS).as_array(), mem);
                }

                #[test]
                fn $load_aligned(mem: super::Vector<[$elem; $size]>) {
                    let v = Vector::load_aligned(&mem, FEATS);
                    prop_assert_eq!(v.as_array(), mem.as_array());
                }

                #[test]
                fn $store(vec in vector::<$elem, $size>(), off in 0usize .. $size) {
                    let mut mem: [$elem; 2 * $size] = [0; 2 * $size];
                    let mem: &mut [$elem; $size] =
                        (&mut mem[off ..][.. $size]).try_into().unwrap();
                    vec.store(mem);
                    prop_assert_eq!(vec.as_array(), mem);
                }

                #[test]
                fn $store_aligned(vec: super::Vector<[$elem; $size]>) {
                    let mut mem = vector![0; $size];
                    vec.with_features(FEATS).store_aligned(&mut mem);
                    prop_assert_eq!(vec.as_array(), mem.as_array());
                }
            }
        };
    }

    test_load_store!([u8; 4]: {
        fn load_u8x4;
        fn load_aligned_u8x4;
        fn store_u8x4;
        fn store_aligned_u8x4;
    });

    test_load_store!([u8; 8]: {
        fn load_u8x8;
        fn load_aligned_u8x8;
        fn store_u8x8;
        fn store_aligned_u8x8;
    });

    test_load_store!([u8; 16]: {
        fn load_u8x16;
        fn load_aligned_u8x16;
        fn store_u8x16;
        fn store_aligned_u8x16;
    });

    // Test adding and subtracting corresponding elements of 16-byte vectors.
    proptest! {
        #[test]
        fn add_u8(l: [u8; 16], r: [u8; 16]) {
            test_bin_map(l, r, |l, r| l + r, |l, r| l.wrapping_add(r))?;
        }

        #[test]
        fn sub_u8(l: [u8; 16], r: [u8; 16]) {
            test_bin_map(l, r, |l, r| l - r, |l, r| l.wrapping_sub(r))?;
        }

        #[test]
        fn neg_u8(v: [u8; 16]) {
            test_una_map(v, |v| -v, |v| v.wrapping_neg())?;
        }

        #[test]
        fn add_i8(l: [i8; 16], r: [i8; 16]) {
            test_bin_map(l, r, |l, r| l + r, |l, r| l.wrapping_add(r))?;
        }

        #[test]
        fn sub_i8(l: [i8; 16], r: [i8; 16]) {
            test_bin_map(l, r, |l, r| l - r, |l, r| l.wrapping_sub(r))?;
        }

        #[test]
        fn neg_i8(v: [i8; 16]) {
            test_una_map(v, |v| -v, |v| v.wrapping_neg())?;
        }
    }

    // Test adding or subtracting a constant to/from elements of 16-byte vectors.
    //
    // We test the usual suspects: maximum/minimum and the most significant bit.
    proptest! {
        #[test]
        fn add_by_u8(x: [u8; 16]) {
            test_bin_map_by(x, imm!(0, u8),
                |l, r| l + r, |l, r| l.wrapping_add(r))?;
            test_bin_map_by(x, imm!(1, u8),
                |l, r| l + r, |l, r| l.wrapping_add(r))?;
            test_bin_map_by(x, imm!(127, u8),
                |l, r| l + r, |l, r| l.wrapping_add(r))?;
            test_bin_map_by(x, imm!(128, u8),
                |l, r| l + r, |l, r| l.wrapping_add(r))?;
            test_bin_map_by(x, imm!(255, u8),
                |l, r| l + r, |l, r| l.wrapping_add(r))?;
        }

        #[test]
        fn sub_by_u8(x: [u8; 16]) {
            test_bin_map_by(x, imm!(0, u8),
                |l, r| l - r, |l, r| l.wrapping_sub(r))?;
            test_bin_map_by(x, imm!(1, u8),
                |l, r| l - r, |l, r| l.wrapping_sub(r))?;
            test_bin_map_by(x, imm!(127, u8),
                |l, r| l - r, |l, r| l.wrapping_sub(r))?;
            test_bin_map_by(x, imm!(128, u8),
                |l, r| l - r, |l, r| l.wrapping_sub(r))?;
            test_bin_map_by(x, imm!(255, u8),
                |l, r| l - r, |l, r| l.wrapping_sub(r))?;
        }

        #[test]
        fn add_by_i8(x: [i8; 16]) {
            test_bin_map_by(x, imm!(0, i8),
                |l, r| l + r, |l, r| l.wrapping_add(r))?;
            test_bin_map_by(x, imm!(1, i8),
                |l, r| l + r, |l, r| l.wrapping_add(r))?;
            test_bin_map_by(x, imm!(-1, i8),
                |l, r| l + r, |l, r| l.wrapping_add(r))?;
            test_bin_map_by(x, imm!(127, i8),
                |l, r| l + r, |l, r| l.wrapping_add(r))?;
            test_bin_map_by(x, imm!(-128, i8),
                |l, r| l + r, |l, r| l.wrapping_add(r))?;
        }

        #[test]
        fn sub_by_i8(x: [i8; 16]) {
            test_bin_map_by(x, imm!(0, i8),
                |l, r| l - r, |l, r| l.wrapping_sub(r))?;
            test_bin_map_by(x, imm!(1, i8),
                |l, r| l - r, |l, r| l.wrapping_sub(r))?;
            test_bin_map_by(x, imm!(-1, i8),
                |l, r| l - r, |l, r| l.wrapping_sub(r))?;
            test_bin_map_by(x, imm!(127, i8),
                |l, r| l - r, |l, r| l.wrapping_sub(r))?;
            test_bin_map_by(x, imm!(-128, i8),
                |l, r| l - r, |l, r| l.wrapping_sub(r))?;
        }
    }

    // Test binary AND between 16-byte vectors.
    proptest! {
        #[test]
        fn and_u8(
            l in any_vector_bits::<u8, 16>(),
            r in any_vector_bits::<u8, 16>(),
        ) {
            test_bin_map(l, r, |l, r| l & r, |l, r| l & r)?;
        }

        #[test]
        fn and_i8(
            l in any_vector_bits::<i8, 16>(),
            r in any_vector_bits::<i8, 16>(),
        ) {
            test_bin_map(l, r, |l, r| l & r, |l, r| l & r)?;
        }
    }

    // Test right shifting a 16-byte vector.
    proptest! {
        #[test]
        fn shr_u8(x in any_vector_bits::<u8, 16>()) {
            test_bin_map_by(x, imm!(0, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(1, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(7, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(8, u8),
                |l, r| l >> r, |_, _| 0u8)?;
        }

        #[test]
        fn shr_u16(x in any_vector_bits::<u16, 8>()) {
            test_bin_map_by(x, imm!(0, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(1, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(8, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(15, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(16, u8),
                |l, r| l >> r, |_, _| 0u16)?;
        }

        #[test]
        fn shr_u32(x in any_vector_bits::<u32, 4>()) {
            test_bin_map_by(x, imm!(0, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(1, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(16, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(31, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(32, u8),
                |l, r| l >> r, |_, _| 0u32)?;
        }

        #[test]
        fn shr_u64(x in any_vector_bits::<u64, 2>()) {
            test_bin_map_by(x, imm!(0, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(1, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(32, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(63, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(64, u8),
                |l, r| l >> r, |_, _| 0u64)?;
        }

        #[test]
        fn shr_i16(x in any_vector_bits::<i16, 8>()) {
            test_bin_map_by(x, imm!(0, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(1, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(8, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(15, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(16, u8),
                |l, r| l >> r, |l, _| l.signum().min(0))?;
        }

        #[test]
        fn shr_i32(x in any_vector_bits::<i32, 4>()) {
            test_bin_map_by(x, imm!(0, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(1, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(16, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(31, u8),
                |l, r| l >> r, |l, r| l >> r)?;
            test_bin_map_by(x, imm!(32, u8),
                |l, r| l >> r, |l, _| l.signum().min(0))?;
        }
    }

    // Test (in)equality between corresponding elements of 16-byte vectors.
    //
    // We artifically increase the probability that elements will be equal, to
    // slightly more than 50%.
    proptest! {
        #[test]
        fn test_eq_ne_u8(m: [bool; 16], l: [u8; 16], r: [u8; 16]) {
            let r = array::from_fn(|i| if !m[i] { l[i] } else { r[i] });
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_eq(r), |l, r| (l == r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_ne(r), |l, r| (l != r).into())?;
        }

        #[test]
        fn test_eq_ne_u16(m: [bool; 8], l: [u16; 8], r: [u16; 8]) {
            let r = array::from_fn(|i| if !m[i] { l[i] } else { r[i] });
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_eq(r), |l, r| (l == r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_ne(r), |l, r| (l != r).into())?;
        }

        #[test]
        fn test_eq_ne_u32(m: [bool; 4], l: [u32; 4], r: [u32; 4]) {
            let r = array::from_fn(|i| if !m[i] { l[i] } else { r[i] });
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_eq(r), |l, r| (l == r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_ne(r), |l, r| (l != r).into())?;
        }

        #[test]
        fn test_eq_ne_i8(m: [bool; 16], l: [i8; 16], r: [i8; 16]) {
            let r = array::from_fn(|i| if !m[i] { l[i] } else { r[i] });
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_eq(r), |l, r| (l == r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_ne(r), |l, r| (l != r).into())?;
        }

        #[test]
        fn test_eq_ne_i16(m: [bool; 8], l: [i16; 8], r: [i16; 8]) {
            let r = array::from_fn(|i| if !m[i] { l[i] } else { r[i] });
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_eq(r), |l, r| (l == r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_ne(r), |l, r| (l != r).into())?;
        }

        #[test]
        fn test_eq_ne_i32(m: [bool; 4], l: [i32; 4], r: [i32; 4]) {
            let r = array::from_fn(|i| if !m[i] { l[i] } else { r[i] });
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_eq(r), |l, r| (l == r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_ne(r), |l, r| (l != r).into())?;
        }
    }

    // Test greater/lesser between corresponding elements of 16-byte vectors.
    proptest! {
        #[test]
        fn test_gt_lt_ge_le_i8(l: [i8; 16], r: [i8; 16]) {
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_gt(r), |l, r| (l > r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_lt(r), |l, r| (l < r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_ge(r), |l, r| (l >= r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_le(r), |l, r| (l <= r).into())?;
        }

        #[test]
        fn test_gt_lt_ge_le_i16(l: [i16; 8], r: [i16; 8]) {
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_gt(r), |l, r| (l > r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_lt(r), |l, r| (l < r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_ge(r), |l, r| (l >= r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_le(r), |l, r| (l <= r).into())?;
        }

        #[test]
        fn test_gt_lt_ge_le_i32(l: [i32; 4], r: [i32; 4]) {
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_gt(r), |l, r| (l > r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_lt(r), |l, r| (l < r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_ge(r), |l, r| (l >= r).into())?;
            test_bin_map(l.into(), r.into(),
                |l, r| l.test_le(r), |l, r| (l <= r).into())?;
        }
    }

    // Test range-contains for elements of 16-byte vectors.
    proptest! {
        #[test]
        fn test_within_u8(x: [u8; 16]) {
            let r = make(x).test_within(imm!(17, u8), imm!(144, u8));
            let a = array::from_fn(|i|
                (17 ..= 144).contains(&x[i]).into());
            prop_assert_eq!(r.as_array(), &a);
        }

        #[test]
        fn test_within_i8(x: [i8; 16]) {
            let r = make(x).test_within(imm!(-111, i8), imm!(16, i8));
            let a = array::from_fn(|i|
                (-111 ..= 16).contains(&x[i]).into());
            prop_assert_eq!(r.as_array(), &a);
        }
    }

    // Test maximums/minimums between corresponding elements of 16-byte vectors.
    proptest! {
        #[test]
        fn test_max_min_u8(l: [u8; 16], r: [u8; 16]) {
            test_bin_map(l, r, |l, r| l.max(r), |l, r| l.max(r))?;
            test_bin_map(l, r, |l, r| l.min(r), |l, r| l.min(r))?;
        }

        #[test]
        fn test_max_min_i16(l: [i16; 8], r: [i16; 8]) {
            test_bin_map(l, r, |l, r| l.max(r), |l, r| l.max(r))?;
            test_bin_map(l, r, |l, r| l.min(r), |l, r| l.min(r))?;
        }
    }

    // Test moving elements of 16-byte vectors left or right by constants.
    proptest! {
        #[test]
        fn move_l_by_u8(v in vector::<u8, 16>()) {
            let x = v.move_l_by(imm!(0, u8));
            prop_assert_eq!(&x.as_array()[..], &v.as_array()[..]);

            let x = v.move_l_by(imm!(1, u8));
            prop_assert_eq!(&x.as_array()[1 ..], &v.as_array()[.. 15]);
            prop_assert_eq!(&x.as_array()[.. 1], &[0u8; 1]);

            let x = v.move_l_by(imm!(8, u8));
            prop_assert_eq!(&x.as_array()[8 ..], &v.as_array()[.. 8]);
            prop_assert_eq!(&x.as_array()[.. 8], &[0u8; 8]);

            let x = v.move_l_by(imm!(15, u8));
            prop_assert_eq!(&x.as_array()[15 ..], &v.as_array()[.. 1]);
            prop_assert_eq!(&x.as_array()[.. 15], &[0u8; 15]);

            let x = v.move_l_by(imm!(16, u8));
            prop_assert_eq!(&x.as_array()[..], &[0u8; 16]);
        }

        #[test]
        fn move_r_by_u8(v in vector::<u8, 16>()) {
            let x = v.move_r_by(imm!(0, u8));
            prop_assert_eq!(&x.as_array()[..], &v.as_array()[..]);

            let x = v.move_r_by(imm!(1, u8));
            prop_assert_eq!(&x.as_array()[.. 15], &v.as_array()[1 ..]);
            prop_assert_eq!(&x.as_array()[15 ..], &[0u8; 1]);

            let x = v.move_r_by(imm!(8, u8));
            prop_assert_eq!(&x.as_array()[.. 8], &v.as_array()[8 ..]);
            prop_assert_eq!(&x.as_array()[8 ..], &[0u8; 8]);

            let x = v.move_r_by(imm!(15, u8));
            prop_assert_eq!(&x.as_array()[.. 1], &v.as_array()[15 ..]);
            prop_assert_eq!(&x.as_array()[1 ..], &[0u8; 15]);

            let x = v.move_r_by(imm!(16, u8));
            prop_assert_eq!(&x.as_array()[..], &[0u8; 16]);
        }

        #[test]
        fn move_l_by_i8(v in vector::<i8, 16>()) {
            let x = v.move_l_by(imm!(0, u8));
            prop_assert_eq!(&x.as_array()[..], &v.as_array()[..]);

            let x = v.move_l_by(imm!(1, u8));
            prop_assert_eq!(&x.as_array()[1 ..], &v.as_array()[.. 15]);
            prop_assert_eq!(&x.as_array()[.. 1], &[0i8; 1]);

            let x = v.move_l_by(imm!(8, u8));
            prop_assert_eq!(&x.as_array()[8 ..], &v.as_array()[.. 8]);
            prop_assert_eq!(&x.as_array()[.. 8], &[0i8; 8]);

            let x = v.move_l_by(imm!(15, u8));
            prop_assert_eq!(&x.as_array()[15 ..], &v.as_array()[.. 1]);
            prop_assert_eq!(&x.as_array()[.. 15], &[0i8; 15]);

            let x = v.move_l_by(imm!(16, u8));
            prop_assert_eq!(&x.as_array()[..], &[0i8; 16]);
        }

        #[test]
        fn move_r_by_i8(v in vector::<i8, 16>()) {
            let x = v.move_r_by(imm!(0, u8));
            prop_assert_eq!(&x.as_array()[..], &v.as_array()[..]);

            let x = v.move_r_by(imm!(1, u8));
            prop_assert_eq!(&x.as_array()[.. 15], &v.as_array()[1 ..]);
            prop_assert_eq!(&x.as_array()[15 ..], &[0i8; 1]);

            let x = v.move_r_by(imm!(8, u8));
            prop_assert_eq!(&x.as_array()[.. 8], &v.as_array()[8 ..]);
            prop_assert_eq!(&x.as_array()[8 ..], &[0i8; 8]);

            let x = v.move_r_by(imm!(15, u8));
            prop_assert_eq!(&x.as_array()[.. 1], &v.as_array()[15 ..]);
            prop_assert_eq!(&x.as_array()[1 ..], &[0i8; 15]);

            let x = v.move_r_by(imm!(16, u8));
            prop_assert_eq!(&x.as_array()[..], &[0i8; 16]);
        }
    }
}