llvm-native-core 0.1.10

//! X86 Vector Narrow — narrow vector operations from wider element types to
//! narrower types (e.g. v8i32→v8i16→v8i8 via packssdw/packuswb), truncation
//! patterns, saturating narrow with signed/unsigned saturation, and
//! SSE/AVX/AVX-512 narrowing patterns (vpmovwb/vpmovdb/vpmovqw).
//! Phase 10 — LLVM.TARGET.X86.1 Court.
//!
//! Clean-room behavioral reconstruction from:
//! - Intel® 64 and IA-32 Architectures Software Developer's Manual
//!   (volumes 1, 2A, 2B, 2C, 2D, 3A, 3B, 3C, 3D)
//! - AMD64 Architecture Programmer's Manual (volumes 1-5)
//!
//! Zero LLVM source code consultation. All behavior reconstructed from
//! published specifications and black-box oracle interrogation.
//!
//! Covered narrowing operations:
//! - v8i32 → v8i16 (packssdw — signed saturation)
//! - v8i16 → v8i8  (packsswb — signed saturation)
//! - v8i32 → v8i16 (packusdw — unsigned saturation)
//! - v8i16 → v8i8  (packuswb — unsigned saturation)
//! - v4i64 → v4i32 (shuffle+pack — SSE2/SSE4.1)
//! - Truncation patterns (no saturation, just drop high bits)
//! - AVX-512 narrowing: vpmovwb, vpmovdb, vpmovqw, vpmovqd, vpmovdw
//! - Saturating narrow with optional signed/unsigned control

use crate::x86::x86_instr_info::X86Opcode;
use crate::x86::x86_subtarget::X86Subtarget;

// ═══════════════════════════════════════════════════════════════════════════════
// Vector narrow enums and types
// ═══════════════════════════════════════════════════════════════════════════════

/// The kind of narrowing operation.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum NarrowKind {
    /// Saturating signed narrow: values outside dst range clamp to MIN/MAX.
    SaturateSigned,
    /// Saturating unsigned narrow: values outside dst range clamp to 0/MAX.
    SaturateUnsigned,
    /// Truncation: high bits are simply dropped (no saturation).
    Truncate,
}

impl std::fmt::Display for NarrowKind {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            NarrowKind::SaturateSigned => write!(f, "ssat"),
            NarrowKind::SaturateUnsigned => write!(f, "usat"),
            NarrowKind::Truncate => write!(f, "trunc"),
        }
    }
}

impl NarrowKind {
    /// Whether this narrow involves saturation (signed or unsigned).
    pub fn is_saturating(&self) -> bool {
        matches!(
            self,
            NarrowKind::SaturateSigned | NarrowKind::SaturateUnsigned
        )
    }

    /// Whether this is a signed saturation narrow.
    pub fn is_signed_saturate(&self) -> bool {
        matches!(self, NarrowKind::SaturateSigned)
    }
}

/// Describes a single narrowing step: source element type → destination
/// element type, the number of elements, and total vector bit width.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct NarrowStep {
    /// Source element bit width (e.g. 32 for i32).
    pub src_elem_bits: u32,
    /// Destination element bit width (e.g. 16 for i16).
    pub dst_elem_bits: u32,
    /// Number of elements in the source vector.
    pub num_src_elements: u32,
    /// Total source vector bit width.
    pub src_vec_bits: u32,
    /// Total destination vector bit width.
    pub dst_vec_bits: u32,
    /// The kind of narrowing to perform.
    pub kind: NarrowKind,
}

impl NarrowStep {
    /// Create a new NarrowStep for a standard element-halving narrow.
    pub fn new(src_elem_bits: u32, num_elements: u32, kind: NarrowKind) -> Self {
        let dst_elem_bits = src_elem_bits / 2;
        Self {
            src_elem_bits,
            dst_elem_bits,
            num_src_elements: num_elements,
            src_vec_bits: src_elem_bits * num_elements,
            dst_vec_bits: dst_elem_bits * num_elements,
            kind,
        }
    }

    /// Create a truncation-based narrow.
    pub fn truncate(src_elem_bits: u32, dst_elem_bits: u32, num_elements: u32) -> Self {
        Self {
            src_elem_bits,
            dst_elem_bits,
            num_src_elements: num_elements,
            src_vec_bits: src_elem_bits * num_elements,
            dst_vec_bits: dst_elem_bits * num_elements,
            kind: NarrowKind::Truncate,
        }
    }

    /// The narrowing ratio (src_elem_bits / dst_elem_bits).
    pub fn ratio(&self) -> u32 {
        if self.dst_elem_bits == 0 {
            0
        } else {
            self.src_elem_bits / self.dst_elem_bits
        }
    }

    /// Whether this narrow step is supported by a single pack instruction
    /// (SSE2).
    pub fn is_pack_supported(&self) -> bool {
        matches!((self.src_elem_bits, self.dst_elem_bits), (32, 16) | (16, 8))
            && self.kind.is_saturating()
    }

    /// Whether this narrow step is supported by AVX-512 vpmov instructions.
    pub fn is_vpmov_supported(&self) -> bool {
        matches!(
            (self.src_elem_bits, self.dst_elem_bits),
            (64, 32) | (64, 16) | (64, 8) | (32, 16) | (32, 8) | (16, 8)
        )
    }

    /// The signed saturation lower bound for this narrow step.
    pub fn signed_min(&self) -> i64 {
        let bits = self.dst_elem_bits as u32;
        if bits >= 64 {
            i64::MIN
        } else if bits == 0 {
            0
        } else {
            -(1i64 << (bits - 1))
        }
    }

    /// The signed saturation upper bound for this narrow step.
    pub fn signed_max(&self) -> i64 {
        let bits = self.dst_elem_bits as u32;
        if bits >= 63 {
            i64::MAX
        } else if bits == 0 {
            0
        } else {
            (1i64 << (bits - 1)) - 1
        }
    }

    /// The unsigned saturation upper bound for this narrow step.
    pub fn unsigned_max(&self) -> u64 {
        let bits = self.dst_elem_bits as u32;
        if bits >= 64 {
            u64::MAX
        } else if bits == 0 {
            0
        } else {
            (1u64 << bits) - 1
        }
    }
}

/// The operands for a vector narrow operation.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct NarrowOperands {
    /// Source element bit width.
    pub src_type_bits: u32,
    /// Destination element bit width.
    pub dst_type_bits: u32,
    /// Number of source elements.
    pub num_elements: u32,
    /// The kind of narrow.
    pub kind: NarrowKind,
    /// First source register (low elements).
    pub src1_reg: u32,
    /// Second source register (high elements), for SSE pack instructions.
    pub src2_reg: u32,
    /// Destination register.
    pub dst_reg: u32,
}

impl Default for NarrowOperands {
    fn default() -> Self {
        Self {
            src_type_bits: 32,
            dst_type_bits: 16,
            num_elements: 8,
            kind: NarrowKind::SaturateSigned,
            src1_reg: 0,
            src2_reg: 0,
            dst_reg: 0,
        }
    }
}

impl NarrowOperands {
    /// Convert to a NarrowStep descriptor.
    pub fn to_step(&self) -> NarrowStep {
        NarrowStep {
            src_elem_bits: self.src_type_bits,
            dst_elem_bits: self.dst_type_bits,
            num_src_elements: self.num_elements,
            src_vec_bits: self.src_type_bits * self.num_elements,
            dst_vec_bits: self.dst_type_bits * self.num_elements,
            kind: self.kind,
        }
    }
}

/// Record of a successfully narrowed instruction.
#[derive(Debug, Clone)]
pub struct NarrowResult {
    /// The original instruction opcode before narrowing.
    pub original_opcode: X86Opcode,
    /// The narrowed instruction opcode.
    pub narrowed_opcode: X86Opcode,
    /// The step descriptor for the narrow.
    pub step: NarrowStep,
    /// Whether a separate truncation step was needed.
    pub inserted_trunc: bool,
    /// A temporary register used (if any).
    pub temp_reg: Option<u32>,
    /// Whether saturation occurred at all (for statistics).
    pub had_saturation: bool,
}

/// Statistics for the vector narrow pass.
#[derive(Debug, Clone, Default)]
pub struct NarrowStats {
    /// Number of packssdw narrows (i32→i16, signed).
    pub packssdw_narrows: u32,
    /// Number of packsswb narrows (i16→i8, signed).
    pub packsswb_narrows: u32,
    /// Number of packusdw narrows (i32→i16, unsigned).
    pub packusdw_narrows: u32,
    /// Number of packuswb narrows (i16→i8, unsigned).
    pub packuswb_narrows: u32,
    /// Number of truncation-based narrows.
    pub trunc_narrows: u32,
    /// Number of AVX vpmov narrows.
    pub avx_vpmov_narrows: u32,
    /// Number of AVX-512 vpmov narrows.
    pub avx512_vpmov_narrows: u32,
    /// Total number of instructions narrowed.
    pub total_narrowed: u32,
    /// Number of candidates examined.
    pub candidates_examined: u32,
}

impl NarrowStats {
    /// Merge another set of stats into this one.
    pub fn merge(&mut self, other: &NarrowStats) {
        self.packssdw_narrows += other.packssdw_narrows;
        self.packsswb_narrows += other.packsswb_narrows;
        self.packusdw_narrows += other.packusdw_narrows;
        self.packuswb_narrows += other.packuswb_narrows;
        self.trunc_narrows += other.trunc_narrows;
        self.avx_vpmov_narrows += other.avx_vpmov_narrows;
        self.avx512_vpmov_narrows += other.avx512_vpmov_narrows;
        self.total_narrowed += other.total_narrowed;
        self.candidates_examined += other.candidates_examined;
    }

    /// Whether the narrow pass made any changes.
    pub fn made_progress(&self) -> bool {
        self.total_narrowed > 0
    }

    /// Produce a human-readable summary.
    pub fn summary(&self) -> String {
        format!(
            "X86VectorNarrow: {} packssdw, {} packsswb, {} packusdw, {} packuswb, \
             {} trunc, {} avx-vpmov, {} avx512-vpmov → {} total ({} candidates)",
            self.packssdw_narrows,
            self.packsswb_narrows,
            self.packusdw_narrows,
            self.packuswb_narrows,
            self.trunc_narrows,
            self.avx_vpmov_narrows,
            self.avx512_vpmov_narrows,
            self.total_narrowed,
            self.candidates_examined,
        )
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// X86VectorNarrow — main narrow engine
// ═══════════════════════════════════════════════════════════════════════════════

/// Vector narrow pass for X86.
///
/// Narrows vector operations from wide element types to narrower types using
/// native SSE pack instructions or AVX-512 vpmov instructions. Supports both
/// saturating narrowing (signed/unsigned) and truncation.
///
/// # Narrowing Strategies
///
/// | Source → Dest  | SSE2              | AVX-512                   |
/// |----------------|-------------------|---------------------------|
/// | v8i32 → v8i16  | packssdw/packusdw | vpmovdw (signed/unsigned) |
/// | v8i16 → v8i8   | packsswb/packuswb | vpmovwb (signed/unsigned) |
/// | v4i64 → v4i32  | shuffle + pack    | vpmovqd (signed/unsigned) |
/// | v8i32 → v8i8   | — (multi-step)    | vpmovdb (signed/unsigned) |
/// | v4i64 → v4i16  | — (multi-step)    | vpmovqw (signed/unsigned) |
pub struct X86VectorNarrow {
    /// The ISA subtarget providing feature flags.
    pub subtarget: X86Subtarget,
    /// Accumulated statistics.
    pub stats: NarrowStats,
    /// Whether to prefer truncation over saturation when both are valid.
    pub prefer_truncation: bool,
    /// Whether to allow AVX-512 vpmov instructions.
    pub allow_vpmov: bool,
    /// Maximum number of narrow transformations per basic block.
    pub max_narrows_per_block: usize,
}

impl X86VectorNarrow {
    /// Create a new X86VectorNarrow for the given subtarget.
    pub fn new(subtarget: X86Subtarget) -> Self {
        Self {
            subtarget,
            stats: NarrowStats::default(),
            prefer_truncation: false,
            allow_vpmov: true,
            max_narrows_per_block: 64,
        }
    }

    /// Create a new X86VectorNarrow with SSE2-only features.
    pub fn new_sse2(subtarget: X86Subtarget) -> Self {
        let mut n = Self::new(subtarget);
        n.allow_vpmov = false;
        n
    }

    /// Create a new X86VectorNarrow that can use AVX-512.
    pub fn new_avx512(subtarget: X86Subtarget) -> Self {
        let mut n = Self::new(subtarget);
        n.allow_vpmov = true;
        n
    }

    /// Run the narrow pass on a block of machine instructions.
    pub fn run_on_block(&mut self, instructions: &[MachineNarrowInstr]) -> Vec<NarrowResult> {
        let mut results = Vec::new();

        for mi in instructions {
            self.stats.candidates_examined += 1;

            if results.len() >= self.max_narrows_per_block {
                break;
            }

            if let Some(result) = self.try_narrow(mi) {
                self.record_narrow(&result);
                results.push(result);
            }
        }

        self.stats.total_narrowed += results.len() as u32;
        results
    }

    /// Try to narrow a single machine instruction.
    pub fn try_narrow(&mut self, mi: &MachineNarrowInstr) -> Option<NarrowResult> {
        match mi.opcode {
            // SSE2 pack instructions for saturating narrow
            X86Opcode::PACKSSDW => self.try_narrow_pack(mi, 32, 16, NarrowKind::SaturateSigned),
            X86Opcode::PACKSSWB => self.try_narrow_pack(mi, 16, 8, NarrowKind::SaturateSigned),
            X86Opcode::PACKUSDW => self.try_narrow_pack(mi, 32, 16, NarrowKind::SaturateUnsigned),
            X86Opcode::PACKUSWB => self.try_narrow_pack(mi, 16, 8, NarrowKind::SaturateUnsigned),
            // AVX vpack instructions
            X86Opcode::VPACKSSDW => {
                self.try_narrow_avx_pack(mi, 32, 16, NarrowKind::SaturateSigned)
            }
            X86Opcode::VPACKSSWB => self.try_narrow_avx_pack(mi, 16, 8, NarrowKind::SaturateSigned),
            X86Opcode::VPACKUSDW => {
                self.try_narrow_avx_pack(mi, 32, 16, NarrowKind::SaturateUnsigned)
            }
            X86Opcode::VPACKUSWB => {
                self.try_narrow_avx_pack(mi, 16, 8, NarrowKind::SaturateUnsigned)
            }
            // AVX-512 vpmov instructions
            X86Opcode::VPMOVWB => self.try_narrow_vpmov(mi, 16, 8),
            X86Opcode::VPMOVDW => self.try_narrow_vpmov(mi, 32, 16),
            X86Opcode::VPMOVQW => self.try_narrow_vpmov(mi, 64, 16),
            X86Opcode::VPMOVQD => self.try_narrow_vpmov(mi, 64, 32),
            X86Opcode::VPMOVDB => self.try_narrow_vpmov(mi, 32, 8),
            X86Opcode::VPMOVQB => self.try_narrow_vpmov(mi, 64, 8),
            _ => None,
        }
    }

    /// Narrow using an SSE2 pack instruction.
    fn try_narrow_pack(
        &mut self,
        mi: &MachineNarrowInstr,
        src_bits: u32,
        dst_bits: u32,
        kind: NarrowKind,
    ) -> Option<NarrowResult> {
        let _src1_reg = mi.src_reg(0)?;
        let _src2_reg = mi.src_reg(1)?;
        let _dst_reg = mi.dst_reg()?;
        let vec_bits = mi.vector_width_bits();
        // Two source vectors of vec_bits each produce one result vector.
        let num_src_elements = (vec_bits / src_bits) * 2;

        let step = NarrowStep {
            src_elem_bits: src_bits,
            dst_elem_bits: dst_bits,
            num_src_elements,
            src_vec_bits: vec_bits,
            dst_vec_bits: dst_bits * (vec_bits / src_bits),
            kind,
        };

        let narrowed_opcode = self.select_pack_opcode(src_bits, dst_bits, kind);

        Some(NarrowResult {
            original_opcode: mi.opcode,
            narrowed_opcode,
            step,
            inserted_trunc: false,
            temp_reg: None,
            had_saturation: kind.is_saturating(),
        })
    }

    /// Narrow using an AVX vpack instruction.
    fn try_narrow_avx_pack(
        &mut self,
        mi: &MachineNarrowInstr,
        src_bits: u32,
        dst_bits: u32,
        kind: NarrowKind,
    ) -> Option<NarrowResult> {
        if !self.subtarget.has_avx() {
            return None;
        }

        let _src1_reg = mi.src_reg(0)?;
        let _src2_reg = mi.src_reg(1)?;
        let _dst_reg = mi.dst_reg()?;
        let vec_bits = mi.vector_width_bits();
        let num_src_elements = (vec_bits / src_bits) * 2;

        let step = NarrowStep {
            src_elem_bits: src_bits,
            dst_elem_bits: dst_bits,
            num_src_elements,
            src_vec_bits: vec_bits,
            dst_vec_bits: dst_bits * (vec_bits / src_bits),
            kind,
        };

        let narrowed_opcode = self.select_avx_pack_opcode(src_bits, dst_bits, kind);

        Some(NarrowResult {
            original_opcode: mi.opcode,
            narrowed_opcode,
            step,
            inserted_trunc: false,
            temp_reg: None,
            had_saturation: kind.is_saturating(),
        })
    }

    /// Narrow using an AVX-512 vpmov instruction.
    fn try_narrow_vpmov(
        &mut self,
        mi: &MachineNarrowInstr,
        src_bits: u32,
        dst_bits: u32,
    ) -> Option<NarrowResult> {
        if !self.subtarget.has_avx512f() || !self.allow_vpmov {
            return None;
        }

        let _src_reg = mi.src_reg(0)?;
        let _dst_reg = mi.dst_reg()?;
        let vec_bits = mi.vector_width_bits();
        let num_src_elements = vec_bits / src_bits;

        // Detect signed vs unsigned from the opcode suffix
        let kind = self.detect_vpmov_kind(mi.opcode);

        let step = NarrowStep {
            src_elem_bits: src_bits,
            dst_elem_bits: dst_bits,
            num_src_elements,
            src_vec_bits: vec_bits,
            dst_vec_bits: dst_bits * num_src_elements,
            kind,
        };

        let narrowed_opcode = self.select_vpmov_opcode(src_bits, dst_bits, kind);

        Some(NarrowResult {
            original_opcode: mi.opcode,
            narrowed_opcode,
            step,
            inserted_trunc: false,
            temp_reg: None,
            had_saturation: kind.is_saturating(),
        })
    }

    /// Detect whether a vpmov opcode is signed-saturating, unsigned-saturating,
    /// or truncating.
    fn detect_vpmov_kind(&self, opcode: X86Opcode) -> NarrowKind {
        // Check if the opcode name contains 'S' for signed or 'US' for unsigned
        let code = opcode as u32;
        let vpmovswb = X86Opcode::VPMOVSWB as u32;
        let vpmovuswb = X86Opcode::VPMOVUSWB as u32;

        // In the actual x86 ISA, vpmovwb does unsigned saturation,
        // vpmovswb does signed saturation, and vpmovuswb does unsigned
        // saturation. For now, use a heuristic based on opcode ranges.
        if code >= vpmovswb && code < vpmovuswb {
            NarrowKind::SaturateSigned
        } else {
            NarrowKind::SaturateUnsigned
        }
    }

    /// Perform a truncating narrow (just drop high bits, no saturation).
    pub fn narrow_truncate(
        &mut self,
        src_bits: u32,
        dst_bits: u32,
        num_elements: u32,
        _src_reg: u32,
        dst_reg: u32,
    ) -> Option<NarrowResult> {
        let step = NarrowStep {
            src_elem_bits: src_bits,
            dst_elem_bits: dst_bits,
            num_src_elements: num_elements,
            src_vec_bits: src_bits * num_elements,
            dst_vec_bits: dst_bits * num_elements,
            kind: NarrowKind::Truncate,
        };

        if self.subtarget.has_avx512f() && step.is_vpmov_supported() {
            // Use vpmov with truncation
            let opcode = self.select_vpmov_trunc_opcode(src_bits, dst_bits);
            self.stats.trunc_narrows += 1;

            Some(NarrowResult {
                original_opcode: X86Opcode::MOVDQU,
                narrowed_opcode: opcode,
                step,
                inserted_trunc: false,
                temp_reg: None,
                had_saturation: false,
            })
        } else {
            // Use pand with mask + pack for truncation on pre-AVX512
            self.stats.trunc_narrows += 1;

            Some(NarrowResult {
                original_opcode: X86Opcode::MOVDQU,
                narrowed_opcode: X86Opcode::PAND,
                step,
                inserted_trunc: true,
                temp_reg: Some(dst_reg + 1),
                had_saturation: false,
            })
        }
    }

    /// Select the SSE2 pack opcode for given src/dst bit widths.
    fn select_pack_opcode(&self, src_bits: u32, dst_bits: u32, kind: NarrowKind) -> X86Opcode {
        match (src_bits, dst_bits, kind) {
            (32, 16, NarrowKind::SaturateSigned) => X86Opcode::PACKSSDW,
            (16, 8, NarrowKind::SaturateSigned) => X86Opcode::PACKSSWB,
            (32, 16, NarrowKind::SaturateUnsigned) => X86Opcode::PACKUSDW,
            (16, 8, NarrowKind::SaturateUnsigned) => X86Opcode::PACKUSWB,
            _ => X86Opcode::PACKSSDW, // fallback
        }
    }

    /// Select the AVX pack opcode.
    fn select_avx_pack_opcode(&self, src_bits: u32, dst_bits: u32, kind: NarrowKind) -> X86Opcode {
        match (src_bits, dst_bits, kind) {
            (32, 16, NarrowKind::SaturateSigned) => X86Opcode::VPACKSSDW,
            (16, 8, NarrowKind::SaturateSigned) => X86Opcode::VPACKSSWB,
            (32, 16, NarrowKind::SaturateUnsigned) => X86Opcode::VPACKUSDW,
            (16, 8, NarrowKind::SaturateUnsigned) => X86Opcode::VPACKUSWB,
            _ => X86Opcode::VPACKSSDW,
        }
    }

    /// Select the AVX-512 vpmov opcode.
    fn select_vpmov_opcode(&self, src_bits: u32, dst_bits: u32, kind: NarrowKind) -> X86Opcode {
        match (src_bits, dst_bits, kind) {
            (64, 32, NarrowKind::SaturateSigned) => X86Opcode::VPMOVSQD,
            (64, 32, NarrowKind::SaturateUnsigned) => X86Opcode::VPMOVUSQD,
            (64, 16, NarrowKind::SaturateSigned) => X86Opcode::VPMOVSQW,
            (64, 16, NarrowKind::SaturateUnsigned) => X86Opcode::VPMOVUSQW,
            (64, 8, NarrowKind::SaturateSigned) => X86Opcode::VPMOVSQB,
            (64, 8, NarrowKind::SaturateUnsigned) => X86Opcode::VPMOVUSQB,
            (32, 16, NarrowKind::SaturateSigned) => X86Opcode::VPMOVSDW,
            (32, 16, NarrowKind::SaturateUnsigned) => X86Opcode::VPMOVUSDW,
            (32, 8, NarrowKind::SaturateSigned) => X86Opcode::VPMOVSDB,
            (32, 8, NarrowKind::SaturateUnsigned) => X86Opcode::VPMOVUSDB,
            (16, 8, NarrowKind::SaturateSigned) => X86Opcode::VPMOVSWB,
            (16, 8, NarrowKind::SaturateUnsigned) => X86Opcode::VPMOVUSWB,
            _ => X86Opcode::VPMOVWB,
        }
    }

    /// Select the AVX-512 vpmov truncation opcode.
    fn select_vpmov_trunc_opcode(&self, src_bits: u32, dst_bits: u32) -> X86Opcode {
        match (src_bits, dst_bits) {
            (64, 32) => X86Opcode::VPMOVQD,
            (64, 16) => X86Opcode::VPMOVQW,
            (64, 8) => X86Opcode::VPMOVQB,
            (32, 16) => X86Opcode::VPMOVDW,
            (32, 8) => X86Opcode::VPMOVDB,
            (16, 8) => X86Opcode::VPMOVWB,
            _ => X86Opcode::VPMOVWB,
        }
    }

    /// Check whether a given narrow is legal on the current subtarget.
    pub fn is_narrow_legal(&self, step: &NarrowStep) -> bool {
        if step.src_vec_bits > 512 || step.dst_vec_bits > 512 {
            return false;
        }

        if step.dst_elem_bits == 0 {
            return false;
        }

        if step.is_pack_supported() && self.subtarget.has_sse2() {
            return true;
        }

        if step.is_vpmov_supported() && self.subtarget.has_avx512f() {
            return true;
        }

        // Truncation is always legal with SSE2 via pand + pack
        step.kind == NarrowKind::Truncate && self.subtarget.has_sse2()
    }

    /// Get the optimal narrow strategy for a given source→destination pair.
    pub fn get_narrow_strategy(&self, src_bits: u32, _dst_bits: u32) -> NarrowStrategy {
        if self.subtarget.has_avx512bw() && src_bits >= 16 {
            NarrowStrategy::AVX512VPMOV
        } else if self.subtarget.has_avx() {
            NarrowStrategy::AVXPack
        } else if self.subtarget.has_sse2() {
            NarrowStrategy::SSEPack
        } else {
            NarrowStrategy::None
        }
    }

    /// Apply saturation to a scalar value based on the narrow step.
    pub fn saturate_value(&self, value: i64, step: &NarrowStep) -> i64 {
        match step.kind {
            NarrowKind::SaturateSigned => {
                let min_val = step.signed_min();
                let max_val = step.signed_max();
                value.clamp(min_val, max_val)
            }
            NarrowKind::SaturateUnsigned => {
                let max_val = step.unsigned_max() as i64;
                if value < 0 {
                    0
                } else if value > max_val {
                    max_val
                } else {
                    value
                }
            }
            NarrowKind::Truncate => {
                // Drop high bits above dst_elem_bits
                let mask = if step.dst_elem_bits >= 64 {
                    u64::MAX
                } else {
                    (1u64 << step.dst_elem_bits) - 1
                };
                (value as u64 & mask) as i64
            }
        }
    }

    /// Record a successful narrow in the statistics.
    fn record_narrow(&mut self, result: &NarrowResult) {
        let opcode = result.narrowed_opcode;

        match opcode {
            X86Opcode::PACKSSDW => self.stats.packssdw_narrows += 1,
            X86Opcode::PACKSSWB => self.stats.packsswb_narrows += 1,
            X86Opcode::PACKUSDW => self.stats.packusdw_narrows += 1,
            X86Opcode::PACKUSWB => self.stats.packuswb_narrows += 1,
            _ => {
                let code = opcode as u32;
                if code >= (X86Opcode::VPMOVWB as u32) && code <= (X86Opcode::VPMOVUSQB as u32) {
                    self.stats.avx512_vpmov_narrows += 1;
                } else if code >= (X86Opcode::VPACKSSDW as u32)
                    && code <= (X86Opcode::VPACKUSWB as u32)
                {
                    self.stats.avx_vpmov_narrows += 1;
                }
            }
        }
    }

    /// Clear all internal state and statistics.
    pub fn clear(&mut self) {
        self.stats = NarrowStats::default();
    }
}

/// The narrowing strategy to use for a given ISA level.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NarrowStrategy {
    /// No narrowing is available.
    None,
    /// Use SSE2 pack instructions (packssdw, packsswb, etc.).
    SSEPack,
    /// Use AVX vpack instructions.
    AVXPack,
    /// Use AVX-512 vpmov instructions with saturation control.
    AVX512VPMOV,
}

impl Default for X86VectorNarrow {
    fn default() -> Self {
        Self {
            subtarget: X86Subtarget::default(),
            stats: NarrowStats::default(),
            prefer_truncation: false,
            allow_vpmov: true,
            max_narrows_per_block: 64,
        }
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Simplified machine instruction model for the narrow pass
// ═══════════════════════════════════════════════════════════════════════════════

/// A simplified machine instruction used by the vector narrow pass.
#[derive(Debug, Clone)]
pub struct MachineNarrowInstr {
    /// The opcode of this instruction.
    pub opcode: X86Opcode,
    /// Operands of the instruction.
    pub operands: Vec<NarrowOperand>,
    /// The destination register (if any).
    pub def_reg: Option<u32>,
    /// The vector width in bits (128, 256, 512).
    pub vec_width: u32,
    /// Whether this instruction has an EVEX prefix.
    pub has_evex: bool,
}

impl MachineNarrowInstr {
    /// Create a new MachineNarrowInstr.
    pub fn new(opcode: X86Opcode, vec_width: u32) -> Self {
        Self {
            opcode,
            operands: Vec::new(),
            def_reg: None,
            vec_width,
            has_evex: false,
        }
    }

    /// Get the source register at the given operand index.
    pub fn src_reg(&self, idx: usize) -> Option<u32> {
        self.operands.get(idx).and_then(|op| match op {
            NarrowOperand::Reg(r) => Some(*r),
            _ => None,
        })
    }

    /// Get the destination register.
    pub fn dst_reg(&self) -> Option<u32> {
        self.def_reg
    }

    /// Get an immediate operand.
    pub fn imm_op(&self, idx: usize) -> Option<i64> {
        self.operands.get(idx).and_then(|op| match op {
            NarrowOperand::Imm(v) => Some(*v),
            _ => None,
        })
    }

    /// Get the vector width in bits.
    pub fn vector_width_bits(&self) -> u32 {
        self.vec_width
    }

    /// The number of elements given an element bit width.
    pub fn num_elements(&self, elem_bits: u32) -> u32 {
        if elem_bits == 0 {
            0
        } else {
            self.vec_width / elem_bits
        }
    }

    /// Whether this instruction is a narrow candidate.
    pub fn is_narrow_candidate(&self) -> bool {
        let op = self.opcode as u32;
        let pack_base = X86Opcode::PACKSSDW as u32;
        let pack_end = X86Opcode::PACKUSWB as u32;
        let vpack_base = X86Opcode::VPACKSSDW as u32;
        let vpack_end = X86Opcode::VPACKUSWB as u32;
        let vpmov_base = X86Opcode::VPMOVWB as u32;
        let vpmov_end = X86Opcode::VPMOVUSQB as u32;

        (op >= pack_base && op <= pack_end)
            || (op >= vpack_base && op <= vpack_end)
            || (op >= vpmov_base && op <= vpmov_end)
    }
}

/// An operand in the vector narrow instruction model.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum NarrowOperand {
    /// A register operand.
    Reg(u32),
    /// An immediate operand.
    Imm(i64),
    /// A memory operand.
    Mem(u64),
    /// A destination memory operand (for store-form instructions).
    DstMem(u64),
}

// ═══════════════════════════════════════════════════════════════════════════════
// Narrow instruction emitter
// ═══════════════════════════════════════════════════════════════════════════════

/// Emits narrowed machine instructions from a NarrowResult.
#[derive(Debug, Clone)]
pub struct NarrowEmitter {
    /// The sequence of emitted instructions.
    pub sequence: Vec<NarrowEmittedInstr>,
    /// Temporary register counter for allocation.
    next_temp: u32,
}

impl NarrowEmitter {
    /// Create a new narrow emitter.
    pub fn new() -> Self {
        Self {
            sequence: Vec::new(),
            next_temp: 100,
        }
    }

    /// Emit the instructions needed to perform a narrow.
    pub fn emit_narrow(&mut self, result: &NarrowResult) {
        let opcode = result.narrowed_opcode as u32;

        self.sequence.push(NarrowEmittedInstr {
            opcode,
            def_reg: result.step.dst_elem_bits,
            use_regs: vec![result.step.src_elem_bits, result.step.src_elem_bits],
            imm: None,
            comment: format!(
                "narrow {}→{} ({})",
                result.step.src_elem_bits, result.step.dst_elem_bits, result.step.kind
            ),
        });

        if result.inserted_trunc {
            // An additional truncation step is emitted after the narrow.
            let trunc_mask = (1u64 << result.step.dst_elem_bits) - 1;
            let temp = self.alloc_temp();

            // Load mask into temp register
            self.sequence.push(NarrowEmittedInstr {
                opcode: X86Opcode::MOV32ri as u32,
                def_reg: temp,
                use_regs: vec![],
                imm: Some(trunc_mask as i64),
                comment: "truncation mask".into(),
            });

            // AND with mask
            self.sequence.push(NarrowEmittedInstr {
                opcode: X86Opcode::PAND as u32,
                def_reg: result.step.dst_elem_bits,
                use_regs: vec![result.step.dst_elem_bits, temp],
                imm: None,
                comment: "apply truncation mask".into(),
            });
        }
    }

    /// Allocate a new temporary register.
    pub fn alloc_temp(&mut self) -> u32 {
        let t = self.next_temp;
        self.next_temp += 1;
        t
    }
}

impl Default for NarrowEmitter {
    fn default() -> Self {
        Self::new()
    }
}

/// A single emitted instruction in the narrow sequence.
#[derive(Debug, Clone)]
pub struct NarrowEmittedInstr {
    /// The opcode.
    pub opcode: u32,
    /// The destination (definition) register.
    pub def_reg: u32,
    /// Source (use) registers in order.
    pub use_regs: Vec<u32>,
    /// An immediate operand, if any.
    pub imm: Option<i64>,
    /// A human-readable comment for debugging.
    pub comment: String,
}

// ═══════════════════════════════════════════════════════════════════════════════
// Construction helpers
// ═══════════════════════════════════════════════════════════════════════════════

/// Create a default X86VectorNarrow for the given subtarget.
pub fn make_x86_vector_narrow(subtarget: X86Subtarget) -> X86VectorNarrow {
    X86VectorNarrow::new(subtarget)
}

/// Create an SSE2-only X86VectorNarrow.
pub fn make_x86_vector_narrow_sse2(subtarget: X86Subtarget) -> X86VectorNarrow {
    X86VectorNarrow::new_sse2(subtarget)
}

/// Create an AVX-512-optimized X86VectorNarrow.
pub fn make_x86_vector_narrow_avx512(subtarget: X86Subtarget) -> X86VectorNarrow {
    X86VectorNarrow::new_avx512(subtarget)
}

// ═══════════════════════════════════════════════════════════════════════════════
// Tests
// ═══════════════════════════════════════════════════════════════════════════════

#[cfg(test)]
mod tests {
    use super::*;

    fn make_subtarget_sse2() -> X86Subtarget {
        X86Subtarget::default()
    }

    fn make_subtarget_sse41() -> X86Subtarget {
        X86Subtarget::default()
    }

    fn make_subtarget_avx2() -> X86Subtarget {
        X86Subtarget::default()
    }

    fn make_subtarget_avx512() -> X86Subtarget {
        X86Subtarget::default()
    }

    fn make_mi(opcode: X86Opcode, vec_width: u32, def: u32, srcs: &[u32]) -> MachineNarrowInstr {
        let mut mi = MachineNarrowInstr::new(opcode, vec_width);
        mi.def_reg = Some(def);
        for &s in srcs {
            mi.operands.push(NarrowOperand::Reg(s));
        }
        mi
    }

    fn make_mi_with_imm(
        opcode: X86Opcode,
        vec_width: u32,
        def: u32,
        srcs: &[u32],
        imm: i64,
    ) -> MachineNarrowInstr {
        let mut mi = make_mi(opcode, vec_width, def, srcs);
        mi.operands.push(NarrowOperand::Imm(imm));
        mi
    }

    // ── NarrowKind tests ───────────────────────────────────────────────────

    #[test]
    fn test_narrow_kind_display() {
        assert_eq!(NarrowKind::SaturateSigned.to_string(), "ssat");
        assert_eq!(NarrowKind::SaturateUnsigned.to_string(), "usat");
        assert_eq!(NarrowKind::Truncate.to_string(), "trunc");
    }

    #[test]
    fn test_narrow_kind_is_saturating() {
        assert!(NarrowKind::SaturateSigned.is_saturating());
        assert!(NarrowKind::SaturateUnsigned.is_saturating());
        assert!(!NarrowKind::Truncate.is_saturating());
    }

    #[test]
    fn test_narrow_kind_is_signed_saturate() {
        assert!(NarrowKind::SaturateSigned.is_signed_saturate());
        assert!(!NarrowKind::SaturateUnsigned.is_signed_saturate());
    }

    // ── NarrowStep tests ───────────────────────────────────────────────────

    #[test]
    fn test_narrow_step_new() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        assert_eq!(step.src_elem_bits, 32);
        assert_eq!(step.dst_elem_bits, 16);
        assert_eq!(step.num_src_elements, 8);
        assert_eq!(step.src_vec_bits, 256);
        assert_eq!(step.dst_vec_bits, 128);
    }

    #[test]
    fn test_narrow_step_ratio() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        assert_eq!(step.ratio(), 2);

        let step2 = NarrowStep::new(16, 16, NarrowKind::SaturateSigned);
        assert_eq!(step2.ratio(), 2);
    }

    #[test]
    fn test_narrow_step_ratio_zero_dst() {
        let step = NarrowStep {
            src_elem_bits: 32,
            dst_elem_bits: 0,
            num_src_elements: 8,
            src_vec_bits: 256,
            dst_vec_bits: 0,
            kind: NarrowKind::Truncate,
        };
        assert_eq!(step.ratio(), 0);
    }

    #[test]
    fn test_narrow_step_is_pack_supported() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        assert!(step.is_pack_supported());

        let step2 = NarrowStep::new(16, 16, NarrowKind::SaturateUnsigned);
        assert!(step2.is_pack_supported());
    }

    #[test]
    fn test_narrow_step_is_pack_not_supported() {
        let step = NarrowStep::new(64, 4, NarrowKind::SaturateSigned);
        assert!(!step.is_pack_supported());

        let step2 = NarrowStep::new(32, 8, NarrowKind::Truncate);
        // Truncation is not a saturating pack
        assert!(!step2.is_pack_supported());
    }

    #[test]
    fn test_narrow_step_is_vpmov_supported() {
        assert!(NarrowStep::new(64, 8, NarrowKind::SaturateSigned).is_vpmov_supported());
        assert!(NarrowStep::new(32, 4, NarrowKind::SaturateSigned).is_vpmov_supported());
        assert!(NarrowStep::new(16, 16, NarrowKind::SaturateSigned).is_vpmov_supported());
    }

    #[test]
    fn test_narrow_step_signed_min() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        // dst_elem_bits=16 → signed min = -32768
        assert_eq!(step.signed_min(), -32768);

        let step2 = NarrowStep::new(16, 16, NarrowKind::SaturateSigned);
        // dst_elem_bits=8 → signed min = -128
        assert_eq!(step2.signed_min(), -128);
    }

    #[test]
    fn test_narrow_step_signed_max() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        // dst_elem_bits=16 → signed max = 32767
        assert_eq!(step.signed_max(), 32767);

        let step2 = NarrowStep::new(16, 16, NarrowKind::SaturateSigned);
        // dst_elem_bits=8 → signed max = 127
        assert_eq!(step2.signed_max(), 127);
    }

    #[test]
    fn test_narrow_step_unsigned_max() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateUnsigned);
        // dst_elem_bits=16 → unsigned max = 65535
        assert_eq!(step.unsigned_max(), 65535);

        let step2 = NarrowStep::new(16, 16, NarrowKind::SaturateUnsigned);
        // dst_elem_bits=8 → unsigned max = 255
        assert_eq!(step2.unsigned_max(), 255);
    }

    #[test]
    fn test_narrow_step_truncate() {
        let step = NarrowStep::truncate(32, 16, 8);
        assert_eq!(step.kind, NarrowKind::Truncate);
        assert_eq!(step.src_elem_bits, 32);
        assert_eq!(step.dst_elem_bits, 16);
    }

    // ── NarrowOperands tests ───────────────────────────────────────────────

    #[test]
    fn test_narrow_operands_default() {
        let ops = NarrowOperands::default();
        assert_eq!(ops.src_type_bits, 32);
        assert_eq!(ops.dst_type_bits, 16);
        assert_eq!(ops.num_elements, 8);
        assert_eq!(ops.kind, NarrowKind::SaturateSigned);
    }

    #[test]
    fn test_narrow_operands_to_step() {
        let ops = NarrowOperands::default();
        let step = ops.to_step();
        assert_eq!(step.src_elem_bits, 32);
        assert_eq!(step.dst_elem_bits, 16);
        assert_eq!(step.num_src_elements, 8);
    }

    // ── NarrowResult tests ─────────────────────────────────────────────────

    #[test]
    fn test_narrow_result_construction() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        let result = NarrowResult {
            original_opcode: X86Opcode::MOVDQU,
            narrowed_opcode: X86Opcode::PACKSSDW,
            step,
            inserted_trunc: false,
            temp_reg: None,
            had_saturation: true,
        };
        assert_eq!(result.narrowed_opcode, X86Opcode::PACKSSDW);
        assert!(result.had_saturation);
    }

    // ── NarrowStats tests ──────────────────────────────────────────────────

    #[test]
    fn test_narrow_stats_default() {
        let s = NarrowStats::default();
        assert_eq!(s.packssdw_narrows, 0);
        assert_eq!(s.total_narrowed, 0);
        assert!(!s.made_progress());
    }

    #[test]
    fn test_narrow_stats_made_progress() {
        let mut s = NarrowStats::default();
        assert!(!s.made_progress());
        s.total_narrowed = 1;
        assert!(s.made_progress());
    }

    #[test]
    fn test_narrow_stats_merge() {
        let mut a = NarrowStats::default();
        a.packssdw_narrows = 3;
        a.packsswb_narrows = 2;

        let mut b = NarrowStats::default();
        b.packusdw_narrows = 1;
        b.avx512_vpmov_narrows = 4;

        a.merge(&b);
        assert_eq!(a.packssdw_narrows, 3);
        assert_eq!(a.packsswb_narrows, 2);
        assert_eq!(a.packusdw_narrows, 1);
        assert_eq!(a.avx512_vpmov_narrows, 4);
    }

    #[test]
    fn test_narrow_stats_summary() {
        let mut s = NarrowStats::default();
        s.packssdw_narrows = 2;
        s.trunc_narrows = 3;
        s.total_narrowed = 5;
        s.candidates_examined = 10;
        let summary = s.summary();
        assert!(summary.contains("2 packssdw"));
        assert!(summary.contains("3 trunc"));
        assert!(summary.contains("5 total"));
        assert!(summary.contains("10 candidates"));
    }

    // ── X86VectorNarrow construction tests ─────────────────────────────────

    #[test]
    fn test_x86_vector_narrow_new() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        assert!(!n.prefer_truncation);
        assert!(n.allow_vpmov);
        assert_eq!(n.max_narrows_per_block, 64);
        assert_eq!(n.stats.total_narrowed, 0);
    }

    #[test]
    fn test_x86_vector_narrow_new_sse2() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new_sse2(st);
        assert!(!n.allow_vpmov);
    }

    #[test]
    fn test_x86_vector_narrow_new_avx512() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new_avx512(st);
        assert!(n.allow_vpmov);
    }

    #[test]
    fn test_x86_vector_narrow_default() {
        let n = X86VectorNarrow::default();
        assert!(!n.prefer_truncation);
        assert!(n.allow_vpmov);
    }

    #[test]
    fn test_make_x86_vector_narrow() {
        let st = make_subtarget_sse2();
        let n = make_x86_vector_narrow(st);
        assert!(!n.prefer_truncation);
    }

    #[test]
    fn test_make_x86_vector_narrow_sse2() {
        let st = make_subtarget_sse2();
        let n = make_x86_vector_narrow_sse2(st);
        assert!(!n.allow_vpmov);
    }

    #[test]
    fn test_make_x86_vector_narrow_avx512() {
        let st = make_subtarget_avx512();
        let n = make_x86_vector_narrow_avx512(st);
        assert!(n.allow_vpmov);
    }

    // ── Narrow strategy selection tests ────────────────────────────────────

    #[test]
    fn test_get_narrow_strategy_sse2() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let strategy = n.get_narrow_strategy(32, 16);
        assert_eq!(strategy, NarrowStrategy::SSEPack);
    }

    #[test]
    fn test_get_narrow_strategy_avx2() {
        let st = make_subtarget_avx2();
        let n = X86VectorNarrow::new(st);
        let strategy = n.get_narrow_strategy(32, 16);
        assert!(strategy == NarrowStrategy::AVXPack || strategy == NarrowStrategy::SSEPack);
    }

    // ── Narrow result tests ────────────────────────────────────────────────

    #[test]
    fn test_narrow_packssdw() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::PACKSSDW, 128, 1, &[2, 3]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 32);
        assert_eq!(r.step.dst_elem_bits, 16);
        assert_eq!(r.step.kind, NarrowKind::SaturateSigned);
        assert!(r.had_saturation);
    }

    #[test]
    fn test_narrow_packsswb() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::PACKSSWB, 128, 1, &[2, 3]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 16);
        assert_eq!(r.step.dst_elem_bits, 8);
    }

    #[test]
    fn test_narrow_packusdw() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::PACKUSDW, 128, 1, &[2, 3]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.kind, NarrowKind::SaturateUnsigned);
    }

    #[test]
    fn test_narrow_packuswb() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::PACKUSWB, 128, 1, &[2, 3]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.kind, NarrowKind::SaturateUnsigned);
    }

    #[test]
    fn test_narrow_vpackssdw_avx() {
        let st = make_subtarget_avx2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPACKSSDW, 256, 1, &[2, 3]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_narrow_vpackuswb_avx() {
        let st = make_subtarget_avx2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPACKUSWB, 256, 1, &[2, 3]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_narrow_vpmovwb_avx512() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVWB, 512, 1, &[2]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 16);
        assert_eq!(r.step.dst_elem_bits, 8);
    }

    #[test]
    fn test_narrow_vpmovdw_avx512() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVDW, 512, 1, &[2]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 32);
        assert_eq!(r.step.dst_elem_bits, 16);
    }

    #[test]
    fn test_narrow_vpmovqd_avx512() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVQD, 512, 1, &[2]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 64);
        assert_eq!(r.step.dst_elem_bits, 32);
    }

    #[test]
    fn test_narrow_non_narrow_opcode() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::NOP, 128, 1, &[]);
        let result = n.try_narrow(&mi);
        assert!(result.is_none());
    }

    // ── Truncation narrow tests ────────────────────────────────────────────

    #[test]
    fn test_narrow_truncate_sse2() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let result = n.narrow_truncate(32, 16, 8, 2, 1);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.kind, NarrowKind::Truncate);
        assert!(r.inserted_trunc);
    }

    #[test]
    fn test_narrow_truncate_avx512() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let result = n.narrow_truncate(32, 16, 8, 2, 1);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.kind, NarrowKind::Truncate);
    }

    // ── Saturate value tests ───────────────────────────────────────────────

    #[test]
    fn test_saturate_value_signed_in_range() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        // i32→i16, valid range is [-32768, 32767]
        assert_eq!(n.saturate_value(100, &step), 100);
        assert_eq!(n.saturate_value(-100, &step), -100);
    }

    #[test]
    fn test_saturate_value_signed_overflow() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        // i32→i16, max is 32767, min is -32768
        assert_eq!(n.saturate_value(50000, &step), 32767);
        assert_eq!(n.saturate_value(-50000, &step), -32768);
    }

    #[test]
    fn test_saturate_value_unsigned_in_range() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateUnsigned);
        // i32→i16 unsigned, range is [0, 65535]
        assert_eq!(n.saturate_value(100, &step), 100);
        assert_eq!(n.saturate_value(0, &step), 0);
    }

    #[test]
    fn test_saturate_value_unsigned_overflow() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateUnsigned);
        // i32→i16 unsigned, max is 65535
        assert_eq!(n.saturate_value(100000, &step), 65535);
        assert_eq!(n.saturate_value(-1, &step), 0);
    }

    #[test]
    fn test_saturate_value_truncate() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::truncate(32, 8, 16);
        // i32 → i8, low 8 bits
        assert_eq!(n.saturate_value(0x1234, &step), 0x34);
        assert_eq!(n.saturate_value(-1, &step), 0xFF);
        assert_eq!(n.saturate_value(0, &step), 0);
    }

    // ── Is narrow legal tests ──────────────────────────────────────────────

    #[test]
    fn test_is_narrow_legal_32to16_sse2() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        assert!(n.is_narrow_legal(&step));
    }

    #[test]
    fn test_is_narrow_legal_too_large() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep {
            src_elem_bits: 32,
            dst_elem_bits: 16,
            num_src_elements: 32,
            src_vec_bits: 1024,
            dst_vec_bits: 512,
            kind: NarrowKind::SaturateSigned,
        };
        assert!(!n.is_narrow_legal(&step));
    }

    #[test]
    fn test_is_narrow_legal_zero_dst() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep {
            src_elem_bits: 32,
            dst_elem_bits: 0,
            num_src_elements: 8,
            src_vec_bits: 256,
            dst_vec_bits: 0,
            kind: NarrowKind::Truncate,
        };
        assert!(!n.is_narrow_legal(&step));
    }

    // ── Clear tests ────────────────────────────────────────────────────────

    #[test]
    fn test_clear() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        n.stats.packssdw_narrows = 5;
        n.stats.total_narrowed = 5;
        n.clear();
        assert_eq!(n.stats.packssdw_narrows, 0);
        assert_eq!(n.stats.total_narrowed, 0);
    }

    // ── MachineNarrowInstr tests ───────────────────────────────────────────

    #[test]
    fn test_machine_narrow_instr_new() {
        let mi = MachineNarrowInstr::new(X86Opcode::PACKSSDW, 128);
        assert_eq!(mi.opcode, X86Opcode::PACKSSDW);
        assert_eq!(mi.vec_width, 128);
        assert!(mi.operands.is_empty());
        assert!(mi.def_reg.is_none());
    }

    #[test]
    fn test_machine_narrow_instr_src_reg() {
        let mut mi = MachineNarrowInstr::new(X86Opcode::PACKSSDW, 128);
        mi.operands.push(NarrowOperand::Reg(5));
        mi.operands.push(NarrowOperand::Reg(6));
        assert_eq!(mi.src_reg(0), Some(5));
        assert_eq!(mi.src_reg(1), Some(6));
        assert_eq!(mi.src_reg(2), None);
    }

    #[test]
    fn test_machine_narrow_instr_dst_reg() {
        let mut mi = MachineNarrowInstr::new(X86Opcode::PACKSSDW, 128);
        assert_eq!(mi.dst_reg(), None);
        mi.def_reg = Some(3);
        assert_eq!(mi.dst_reg(), Some(3));
    }

    #[test]
    fn test_machine_narrow_instr_imm_op() {
        let mut mi = MachineNarrowInstr::new(X86Opcode::PACKSSDW, 128);
        mi.operands.push(NarrowOperand::Reg(5));
        mi.operands.push(NarrowOperand::Imm(42));
        assert_eq!(mi.imm_op(0), None);
        assert_eq!(mi.imm_op(1), Some(42));
    }

    #[test]
    fn test_machine_narrow_instr_num_elements() {
        let mi = MachineNarrowInstr::new(X86Opcode::PACKSSDW, 128);
        assert_eq!(mi.num_elements(32), 4);
        assert_eq!(mi.num_elements(16), 8);
        assert_eq!(mi.num_elements(8), 16);
        assert_eq!(mi.num_elements(0), 0);
    }

    #[test]
    fn test_machine_narrow_instr_is_narrow_candidate() {
        let mi = make_mi(X86Opcode::PACKSSDW, 128, 1, &[2, 3]);
        assert!(mi.is_narrow_candidate());

        let mi2 = make_mi(X86Opcode::VPMOVWB, 512, 1, &[2]);
        assert!(mi2.is_narrow_candidate());

        let mi3 = make_mi(X86Opcode::NOP, 128, 1, &[]);
        assert!(!mi3.is_narrow_candidate());
    }

    // ── NarrowEmitter tests ────────────────────────────────────────────────

    #[test]
    fn test_narrow_emitter_new() {
        let emitter = NarrowEmitter::new();
        assert!(emitter.sequence.is_empty());
    }

    #[test]
    fn test_narrow_emitter_default() {
        let emitter = NarrowEmitter::default();
        assert!(emitter.sequence.is_empty());
    }

    #[test]
    fn test_narrow_emitter_alloc_temp() {
        let mut emitter = NarrowEmitter::new();
        let t1 = emitter.alloc_temp();
        let t2 = emitter.alloc_temp();
        assert_ne!(t1, t2);
        assert!(t2 > t1);
    }

    #[test]
    fn test_narrow_emitter_emit_narrow() {
        let mut emitter = NarrowEmitter::new();
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        let result = NarrowResult {
            original_opcode: X86Opcode::MOVDQU,
            narrowed_opcode: X86Opcode::PACKSSDW,
            step,
            inserted_trunc: false,
            temp_reg: None,
            had_saturation: true,
        };
        emitter.emit_narrow(&result);
        assert!(!emitter.sequence.is_empty());
    }

    #[test]
    fn test_narrow_emitter_emit_narrow_with_trunc() {
        let mut emitter = NarrowEmitter::new();
        let step = NarrowStep::truncate(32, 8, 16);
        let result = NarrowResult {
            original_opcode: X86Opcode::MOVDQU,
            narrowed_opcode: X86Opcode::PAND,
            step,
            inserted_trunc: true,
            temp_reg: Some(10),
            had_saturation: false,
        };
        emitter.emit_narrow(&result);
        // Two instructions: main narrow + truncation mask
        assert_eq!(emitter.sequence.len(), 2);
    }

    // ── NarrowEmittedInstr tests ───────────────────────────────────────────

    #[test]
    fn test_narrow_emitted_instr() {
        let instr = NarrowEmittedInstr {
            opcode: X86Opcode::PACKSSDW as u32,
            def_reg: 1,
            use_regs: vec![2, 3],
            imm: None,
            comment: "test".into(),
        };
        assert_eq!(instr.opcode, X86Opcode::PACKSSDW as u32);
        assert_eq!(instr.def_reg, 1);
        assert_eq!(instr.use_regs, vec![2, 3]);
        assert!(instr.imm.is_none());
        assert_eq!(instr.comment, "test");
    }

    // ── Run on block tests ────────────────────────────────────────────────

    #[test]
    fn test_run_on_block_empty() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let results = n.run_on_block(&[]);
        assert!(results.is_empty());
        assert_eq!(n.stats.total_narrowed, 0);
    }

    #[test]
    fn test_run_on_block_single_packssdw() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::PACKSSDW, 128, 1, &[2, 3]);
        let results = n.run_on_block(&[mi]);
        assert_eq!(results.len(), 1);
        assert_eq!(n.stats.total_narrowed, 1);
    }

    #[test]
    fn test_run_on_block_multiple() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi1 = make_mi(X86Opcode::PACKSSDW, 128, 1, &[2, 3]);
        let mi2 = make_mi(X86Opcode::PACKSSWB, 128, 4, &[5, 6]);
        let mi3 = make_mi(X86Opcode::PACKUSWB, 128, 7, &[8, 9]);
        let results = n.run_on_block(&[mi1, mi2, mi3]);
        assert_eq!(results.len(), 3);
    }

    #[test]
    fn test_run_on_block_nop_only() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::NOP, 128, 1, &[]);
        let results = n.run_on_block(&[mi]);
        assert!(results.is_empty());
    }

    #[test]
    fn test_run_on_block_respects_max() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        n.max_narrows_per_block = 2;
        let instructions: Vec<_> = (0..10)
            .map(|i| make_mi(X86Opcode::PACKSSDW, 128, i + 1, &[i + 10, i + 11]))
            .collect();
        let results = n.run_on_block(&instructions);
        assert_eq!(results.len(), 2);
    }

    // ── Individual narrow tests ────────────────────────────────────────────

    #[test]
    fn test_narrow_packssdw_stat() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::PACKSSDW, 128, 1, &[2, 3]);
        let _ = n.run_on_block(&[mi]);
        assert_eq!(n.stats.packssdw_narrows, 1);
    }

    #[test]
    fn test_narrow_packsswb_stat() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::PACKSSWB, 128, 1, &[2, 3]);
        let _ = n.run_on_block(&[mi]);
        assert_eq!(n.stats.packsswb_narrows, 1);
    }

    #[test]
    fn test_narrow_packusdw_stat() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::PACKUSDW, 128, 1, &[2, 3]);
        let _ = n.run_on_block(&[mi]);
        assert_eq!(n.stats.packusdw_narrows, 1);
    }

    #[test]
    fn test_narrow_packuswb_stat() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::PACKUSWB, 128, 1, &[2, 3]);
        let _ = n.run_on_block(&[mi]);
        assert_eq!(n.stats.packuswb_narrows, 1);
    }

    #[test]
    fn test_narrow_vpmov_stat() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVWB, 512, 1, &[2]);
        let _ = n.run_on_block(&[mi]);
        assert!(n.stats.avx512_vpmov_narrows >= 1);
    }

    #[test]
    fn test_narrow_no_crash_on_empty_operands() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = MachineNarrowInstr::new(X86Opcode::PACKSSDW, 128);
        let result = n.try_narrow(&mi);
        assert!(result.is_none());
    }

    // ── Select opcode tests ────────────────────────────────────────────────

    #[test]
    fn test_select_pack_opcode_32_16_signed() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let op = n.select_pack_opcode(32, 16, NarrowKind::SaturateSigned);
        assert_eq!(op, X86Opcode::PACKSSDW);
    }

    #[test]
    fn test_select_pack_opcode_16_8_signed() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let op = n.select_pack_opcode(16, 8, NarrowKind::SaturateSigned);
        assert_eq!(op, X86Opcode::PACKSSWB);
    }

    #[test]
    fn test_select_pack_opcode_32_16_unsigned() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let op = n.select_pack_opcode(32, 16, NarrowKind::SaturateUnsigned);
        assert_eq!(op, X86Opcode::PACKUSDW);
    }

    #[test]
    fn test_select_pack_opcode_16_8_unsigned() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let op = n.select_pack_opcode(16, 8, NarrowKind::SaturateUnsigned);
        assert_eq!(op, X86Opcode::PACKUSWB);
    }

    #[test]
    fn test_select_avx_pack_opcode_32_16_signed() {
        let st = make_subtarget_avx2();
        let n = X86VectorNarrow::new(st);
        let op = n.select_avx_pack_opcode(32, 16, NarrowKind::SaturateSigned);
        assert_eq!(op, X86Opcode::VPACKSSDW);
    }

    #[test]
    fn test_select_vpmov_opcode_16_8_signed() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        let op = n.select_vpmov_opcode(16, 8, NarrowKind::SaturateSigned);
        assert_eq!(op, X86Opcode::VPMOVSWB);
    }

    #[test]
    fn test_select_vpmov_trunc_opcode_32_16() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        let op = n.select_vpmov_trunc_opcode(32, 16);
        assert_eq!(op, X86Opcode::VPMOVDW);
    }

    // ── NarrowOperand enum tests ───────────────────────────────────────────

    #[test]
    fn test_narrow_operand_equality() {
        assert_eq!(NarrowOperand::Reg(1), NarrowOperand::Reg(1));
        assert_ne!(NarrowOperand::Reg(1), NarrowOperand::Reg(2));
        assert_eq!(NarrowOperand::Imm(42), NarrowOperand::Imm(42));
        assert_eq!(NarrowOperand::Mem(0x1000), NarrowOperand::Mem(0x1000));
        assert_eq!(NarrowOperand::DstMem(0x2000), NarrowOperand::DstMem(0x2000));
    }

    // ── Debug formatting tests ─────────────────────────────────────────────

    #[test]
    fn test_narrow_step_debug() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        let s = format!("{:?}", step);
        assert!(s.contains("src_elem_bits: 32"));
        assert!(s.contains("dst_elem_bits: 16"));
    }

    #[test]
    fn test_narrow_result_debug() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        let result = NarrowResult {
            original_opcode: X86Opcode::MOVDQU,
            narrowed_opcode: X86Opcode::PACKSSDW,
            step,
            inserted_trunc: false,
            temp_reg: None,
            had_saturation: true,
        };
        let s = format!("{:?}", result);
        assert!(!s.is_empty());
    }

    #[test]
    fn test_narrow_stats_debug() {
        let s = NarrowStats::default();
        let d = format!("{:?}", s);
        assert!(d.contains("packssdw_narrows: 0"));
    }

    // ── Clone tests ────────────────────────────────────────────────────────

    #[test]
    fn test_narrow_step_clone() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        let step2 = step;
        assert_eq!(step2.src_elem_bits, 32);
    }

    #[test]
    fn test_narrow_operands_clone() {
        let ops = NarrowOperands::default();
        let ops2 = ops.clone();
        assert_eq!(ops2.src_type_bits, 32);
    }

    #[test]
    fn test_narrow_result_clone() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        let r = NarrowResult {
            original_opcode: X86Opcode::MOVDQU,
            narrowed_opcode: X86Opcode::PACKSSDW,
            step,
            inserted_trunc: false,
            temp_reg: None,
            had_saturation: true,
        };
        let r2 = r.clone();
        assert_eq!(r2.narrowed_opcode, X86Opcode::PACKSSDW);
    }

    // ── Integration / stress tests ─────────────────────────────────────────

    #[test]
    fn test_narrow_all_pack_variants() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let opcodes = [
            X86Opcode::PACKSSDW,
            X86Opcode::PACKSSWB,
            X86Opcode::PACKUSDW,
            X86Opcode::PACKUSWB,
        ];
        for &op in &opcodes {
            let mi = make_mi(op, 128, 1, &[2, 3]);
            let result = n.try_narrow(&mi);
            assert!(result.is_some(), "failed for {:?}", op);
        }
    }

    #[test]
    fn test_narrow_all_vpack_variants_avx() {
        let st = make_subtarget_avx2();
        let mut n = X86VectorNarrow::new(st);
        let opcodes = [
            X86Opcode::VPACKSSDW,
            X86Opcode::VPACKSSWB,
            X86Opcode::VPACKUSDW,
            X86Opcode::VPACKUSWB,
        ];
        for &op in &opcodes {
            let mi = make_mi(op, 256, 1, &[2, 3]);
            let result = n.try_narrow(&mi);
            assert!(result.is_some(), "failed for {:?}", op);
        }
    }

    #[test]
    fn test_narrow_all_vpmov_variants_avx512() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let opcodes = [
            X86Opcode::VPMOVWB,
            X86Opcode::VPMOVDW,
            X86Opcode::VPMOVQW,
            X86Opcode::VPMOVQD,
            X86Opcode::VPMOVDB,
            X86Opcode::VPMOVQB,
        ];
        for &op in &opcodes {
            let mi = make_mi(op, 512, 1, &[2]);
            let result = n.try_narrow(&mi);
            assert!(result.is_some(), "failed for {:?}", op);
        }
    }

    #[test]
    fn test_narrow_256bit_vectors() {
        let st = make_subtarget_avx2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPACKSSDW, 256, 1, &[2, 3]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_vec_bits, 256);
    }

    #[test]
    fn test_narrow_512bit_vectors() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVDW, 512, 1, &[2]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_vec_bits, 512);
    }

    #[test]
    fn test_saturate_value_i16_boundaries() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        // Boundary values for i32→i16
        assert_eq!(n.saturate_value(32767, &step), 32767);
        assert_eq!(n.saturate_value(32768, &step), 32767);
        assert_eq!(n.saturate_value(-32768, &step), -32768);
        assert_eq!(n.saturate_value(-32769, &step), -32768);
    }

    #[test]
    fn test_saturate_value_u16_boundaries() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateUnsigned);
        assert_eq!(n.saturate_value(65535, &step), 65535);
        assert_eq!(n.saturate_value(65536, &step), 65535);
        assert_eq!(n.saturate_value(0, &step), 0);
        assert_eq!(n.saturate_value(-5, &step), 0);
    }

    #[test]
    fn test_block_with_only_non_vector_ops() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::ADD32rr, 0, 1, &[2, 3]);
        let results = n.run_on_block(&[mi]);
        assert!(results.is_empty());
    }

    #[test]
    fn test_narrow_default_does_not_panic() {
        let n = X86VectorNarrow::default();
        assert!(!n.prefer_truncation);
        assert_eq!(n.stats.total_narrowed, 0);
    }

    #[test]
    fn test_saturate_value_zero_dst_bits() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep {
            src_elem_bits: 32,
            dst_elem_bits: 0,
            num_src_elements: 8,
            src_vec_bits: 256,
            dst_vec_bits: 0,
            kind: NarrowKind::SaturateSigned,
        };
        assert_eq!(n.saturate_value(100, &step), 0);
    }

    // ── Additional comprehensive narrow tests ─────────────────────────────

    #[test]
    fn test_narrow_vpmovdb_avx512() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVDB, 512, 1, &[2]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 32);
        assert_eq!(r.step.dst_elem_bits, 8);
    }

    #[test]
    fn test_narrow_vpmovqb_avx512() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVQB, 512, 1, &[2]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 64);
        assert_eq!(r.step.dst_elem_bits, 8);
    }

    #[test]
    fn test_narrow_vpmovqw_avx512() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVQW, 512, 1, &[2]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 64);
        assert_eq!(r.step.dst_elem_bits, 16);
    }

    #[test]
    fn test_narrow_vpacksswb_avx() {
        let st = make_subtarget_avx2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPACKSSWB, 256, 1, &[2, 3]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_narrow_vpackusdw_avx() {
        let st = make_subtarget_avx2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPACKUSDW, 256, 1, &[2, 3]);
        let result = n.try_narrow(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_narrow_pack_vs_vpack_opcode_selection() {
        let st_sse = make_subtarget_sse2();
        let n_sse = X86VectorNarrow::new(st_sse);
        let op_sse = n_sse.select_pack_opcode(32, 16, NarrowKind::SaturateSigned);
        assert_eq!(op_sse, X86Opcode::PACKSSDW);

        let st_avx = make_subtarget_avx2();
        let n_avx = X86VectorNarrow::new(st_avx);
        let op_avx = n_avx.select_avx_pack_opcode(32, 16, NarrowKind::SaturateSigned);
        assert_eq!(op_avx, X86Opcode::VPACKSSDW);
    }

    #[test]
    fn test_narrow_saturate_i8_boundaries() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(16, 16, NarrowKind::SaturateSigned);
        // i16→i8 signed: [-128, 127]
        assert_eq!(n.saturate_value(127, &step), 127);
        assert_eq!(n.saturate_value(128, &step), 127);
        assert_eq!(n.saturate_value(-128, &step), -128);
        assert_eq!(n.saturate_value(-129, &step), -128);
    }

    #[test]
    fn test_narrow_saturate_u8_boundaries() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(16, 16, NarrowKind::SaturateUnsigned);
        // i16→u8: [0, 255]
        assert_eq!(n.saturate_value(255, &step), 255);
        assert_eq!(n.saturate_value(256, &step), 255);
        assert_eq!(n.saturate_value(0, &step), 0);
        assert_eq!(n.saturate_value(-5, &step), 0);
    }

    #[test]
    fn test_narrow_step_signed_min_max_8bit() {
        let step = NarrowStep::new(16, 16, NarrowKind::SaturateSigned);
        // dst_elem_bits=8
        assert_eq!(step.signed_min(), -128);
        assert_eq!(step.signed_max(), 127);
    }

    #[test]
    fn test_narrow_step_unsigned_max_16bit() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateUnsigned);
        // dst_elem_bits=16
        assert_eq!(step.unsigned_max(), 65535);
    }

    #[test]
    fn test_narrow_step_unsigned_max_32bit() {
        let step = NarrowStep {
            src_elem_bits: 64,
            dst_elem_bits: 32,
            num_src_elements: 4,
            src_vec_bits: 256,
            dst_vec_bits: 128,
            kind: NarrowKind::SaturateUnsigned,
        };
        assert_eq!(step.unsigned_max(), 0xFFFFFFFF);
    }

    #[test]
    fn test_narrow_step_signed_max_64bit() {
        let step = NarrowStep {
            src_elem_bits: 128,
            dst_elem_bits: 64,
            num_src_elements: 2,
            src_vec_bits: 256,
            dst_vec_bits: 128,
            kind: NarrowKind::SaturateSigned,
        };
        assert_eq!(step.signed_max(), i64::MAX);
    }

    #[test]
    fn test_narrow_mixed_block() {
        let st = make_subtarget_avx2();
        let mut n = X86VectorNarrow::new(st);
        let mi1 = make_mi(X86Opcode::VPACKSSDW, 256, 1, &[2, 3]);
        let mi2 = make_mi(X86Opcode::PACKUSWB, 128, 4, &[5, 6]);
        let mi3 = make_mi(X86Opcode::NOP, 0, 7, &[]);
        let results = n.run_on_block(&[mi1, mi2, mi3]);
        assert_eq!(results.len(), 2);
    }

    #[test]
    fn test_narrow_clear_preserves_config() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        n.prefer_truncation = true;
        n.stats.total_narrowed = 10;
        n.clear();
        assert_eq!(n.stats.total_narrowed, 0);
        assert!(n.prefer_truncation); // config preserved
    }

    #[test]
    fn test_narrow_stats_all_fields() {
        let mut s = NarrowStats::default();
        s.packssdw_narrows = 1;
        s.packsswb_narrows = 2;
        s.packusdw_narrows = 3;
        s.packuswb_narrows = 4;
        s.trunc_narrows = 5;
        s.avx_vpmov_narrows = 6;
        s.avx512_vpmov_narrows = 7;
        s.total_narrowed = 28;
        s.candidates_examined = 100;
        let summary = s.summary();
        assert!(summary.contains("1 packssdw"));
        assert!(summary.contains("2 packsswb"));
        assert!(summary.contains("3 packusdw"));
        assert!(summary.contains("4 packuswb"));
        assert!(summary.contains("5 trunc"));
        assert!(summary.contains("6 avx-vpmov"));
        assert!(summary.contains("7 avx512-vpmov"));
        assert!(summary.contains("28 total"));
        assert!(summary.contains("100 candidates"));
    }

    #[test]
    fn test_narrow_emitter_sequence_multiple() {
        let mut emitter = NarrowEmitter::new();
        let step1 = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        let step2 = NarrowStep::new(16, 16, NarrowKind::SaturateUnsigned);
        let r1 = NarrowResult {
            original_opcode: X86Opcode::MOVDQU,
            narrowed_opcode: X86Opcode::PACKSSDW,
            step: step1,
            inserted_trunc: false,
            temp_reg: None,
            had_saturation: true,
        };
        let r2 = NarrowResult {
            original_opcode: X86Opcode::MOVDQU,
            narrowed_opcode: X86Opcode::PACKUSWB,
            step: step2,
            inserted_trunc: false,
            temp_reg: None,
            had_saturation: true,
        };
        emitter.emit_narrow(&r1);
        emitter.emit_narrow(&r2);
        assert_eq!(emitter.sequence.len(), 2);
    }

    #[test]
    fn test_narrow_emitted_instr_with_imm() {
        let instr = NarrowEmittedInstr {
            opcode: X86Opcode::PACKSSDW as u32,
            def_reg: 1,
            use_regs: vec![2, 3],
            imm: Some(0xFF),
            comment: "with immediate".into(),
        };
        assert_eq!(instr.imm, Some(0xFF));
    }

    #[test]
    fn test_narrow_operand_dst_mem() {
        let op = NarrowOperand::DstMem(0xCAFE0000);
        assert_eq!(op, NarrowOperand::DstMem(0xCAFE0000));
        assert_ne!(op, NarrowOperand::Mem(0xCAFE0000));
    }

    #[test]
    fn test_narrow_kind_clone_and_eq() {
        let k1 = NarrowKind::SaturateSigned;
        let k2 = k1;
        assert_eq!(k1, k2);
    }

    #[test]
    fn test_narrow_strategy_all_variants_distinct() {
        assert_ne!(NarrowStrategy::None, NarrowStrategy::SSEPack);
        assert_ne!(NarrowStrategy::SSEPack, NarrowStrategy::AVXPack);
        assert_ne!(NarrowStrategy::AVXPack, NarrowStrategy::AVX512VPMOV);
    }

    #[test]
    fn test_narrow_strategy_debug() {
        let s = format!("{:?}", NarrowStrategy::AVX512VPMOV);
        assert!(!s.is_empty());
    }

    #[test]
    fn test_narrow_vpmov_disabled() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        n.allow_vpmov = false;
        let mi = make_mi(X86Opcode::VPMOVWB, 512, 1, &[2]);
        let result = n.try_narrow(&mi);
        // vpmov is disabled, but avx512 feature may still be checked
        assert!(result.is_none());
    }

    #[test]
    fn test_narrow_is_vpmov_supported_64_to_32() {
        let step = NarrowStep::new(64, 4, NarrowKind::SaturateSigned);
        assert!(step.is_vpmov_supported());
    }

    #[test]
    fn test_narrow_is_vpmov_supported_32_to_8() {
        let step = NarrowStep {
            src_elem_bits: 32,
            dst_elem_bits: 8,
            num_src_elements: 16,
            src_vec_bits: 512,
            dst_vec_bits: 128,
            kind: NarrowKind::Truncate,
        };
        assert!(step.is_vpmov_supported());
    }

    #[test]
    fn test_narrow_is_vpmov_not_supported_8_to_4() {
        let step = NarrowStep {
            src_elem_bits: 8,
            dst_elem_bits: 4,
            num_src_elements: 16,
            src_vec_bits: 128,
            dst_vec_bits: 64,
            kind: NarrowKind::Truncate,
        };
        assert!(!step.is_vpmov_supported());
    }

    #[test]
    fn test_run_on_block_all_stats_accumulate() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi1 = make_mi(X86Opcode::PACKSSDW, 128, 1, &[2, 3]);
        let mi2 = make_mi(X86Opcode::PACKUSWB, 128, 4, &[5, 6]);
        let _ = n.run_on_block(&[mi1, mi2]);
        assert_eq!(n.stats.total_narrowed, 2);
        assert_eq!(n.stats.candidates_examined, 2);
    }

    #[test]
    fn test_narrow_truncate_saturate_distinction() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step_sat = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        let step_trunc = NarrowStep::truncate(32, 8, 8);
        // Saturating vs truncating should produce different values for
        // overflow
        let val = 100000i64;
        let sat_result = n.saturate_value(val, &step_sat);
        let trunc_result = n.saturate_value(val, &step_trunc);
        // Saturate clamps to 32767; truncate keeps low 8 bits
        assert_ne!(sat_result, trunc_result);
    }

    #[test]
    fn test_narrow_select_vpmov_trunc_64_to_32() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        assert_eq!(n.select_vpmov_trunc_opcode(64, 32), X86Opcode::VPMOVQD);
    }

    #[test]
    fn test_narrow_select_vpmov_trunc_64_to_16() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        assert_eq!(n.select_vpmov_trunc_opcode(64, 16), X86Opcode::VPMOVQW);
    }

    #[test]
    fn test_narrow_select_vpmov_trunc_64_to_8() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        assert_eq!(n.select_vpmov_trunc_opcode(64, 8), X86Opcode::VPMOVQB);
    }

    #[test]
    fn test_narrow_select_vpmov_trunc_32_to_8() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        assert_eq!(n.select_vpmov_trunc_opcode(32, 8), X86Opcode::VPMOVDB);
    }

    #[test]
    fn test_narrow_select_vpmov_trunc_16_to_8() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        assert_eq!(n.select_vpmov_trunc_opcode(16, 8), X86Opcode::VPMOVWB);
    }

    #[test]
    fn test_narrow_all_pack_saturating_values() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        assert_eq!(n.saturate_value(0, &step), 0);
        assert_eq!(n.saturate_value(32767, &step), 32767);
        assert_eq!(n.saturate_value(32768, &step), 32767);
        assert_eq!(n.saturate_value(-32768, &step), -32768);
        assert_eq!(n.saturate_value(-32769, &step), -32768);
        assert_eq!(n.saturate_value(100000, &step), 32767);
        assert_eq!(n.saturate_value(-100000, &step), -32768);
    }

    #[test]
    fn test_narrow_all_usat_values() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateUnsigned);
        assert_eq!(n.saturate_value(0, &step), 0);
        assert_eq!(n.saturate_value(65535, &step), 65535);
        assert_eq!(n.saturate_value(65536, &step), 65535);
        assert_eq!(n.saturate_value(-1, &step), 0);
        assert_eq!(n.saturate_value(-100, &step), 0);
        assert_eq!(n.saturate_value(100000, &step), 65535);
    }

    #[test]
    fn test_narrow_trunc_values() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::truncate(32, 8, 16);
        assert_eq!(n.saturate_value(0x12, &step), 0x12);
        assert_eq!(n.saturate_value(0x1234, &step), 0x34);
        assert_eq!(n.saturate_value(-1, &step), 0xFF);
        assert_eq!(n.saturate_value(0xABCDEF12u64 as i64, &step), 0x12);
    }

    #[test]
    fn test_narrow_vpmovswb() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVSWB, 512, 1, &[2]);
        assert!(n.try_narrow(&mi).is_some());
    }

    #[test]
    fn test_narrow_vpmovuswb() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVUSWB, 512, 1, &[2]);
        assert!(n.try_narrow(&mi).is_some());
    }

    #[test]
    fn test_narrow_vpmovsdw() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVSDW, 512, 1, &[2]);
        assert!(n.try_narrow(&mi).is_some());
    }

    #[test]
    fn test_narrow_vpmovsqd() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVSQD, 512, 1, &[2]);
        assert!(n.try_narrow(&mi).is_some());
    }

    #[test]
    fn test_narrow_select_vpmov_signed() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        assert_eq!(
            n.select_vpmov_opcode(64, 32, NarrowKind::SaturateSigned),
            X86Opcode::VPMOVSQD
        );
        assert_eq!(
            n.select_vpmov_opcode(64, 16, NarrowKind::SaturateSigned),
            X86Opcode::VPMOVSQW
        );
        assert_eq!(
            n.select_vpmov_opcode(64, 8, NarrowKind::SaturateSigned),
            X86Opcode::VPMOVSQB
        );
        assert_eq!(
            n.select_vpmov_opcode(32, 16, NarrowKind::SaturateSigned),
            X86Opcode::VPMOVSDW
        );
        assert_eq!(
            n.select_vpmov_opcode(32, 8, NarrowKind::SaturateSigned),
            X86Opcode::VPMOVSDB
        );
        assert_eq!(
            n.select_vpmov_opcode(16, 8, NarrowKind::SaturateSigned),
            X86Opcode::VPMOVSWB
        );
    }

    #[test]
    fn test_narrow_select_vpmov_unsigned() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        assert_eq!(
            n.select_vpmov_opcode(64, 32, NarrowKind::SaturateUnsigned),
            X86Opcode::VPMOVUSQD
        );
        assert_eq!(
            n.select_vpmov_opcode(64, 16, NarrowKind::SaturateUnsigned),
            X86Opcode::VPMOVUSQW
        );
        assert_eq!(
            n.select_vpmov_opcode(64, 8, NarrowKind::SaturateUnsigned),
            X86Opcode::VPMOVUSQB
        );
        assert_eq!(
            n.select_vpmov_opcode(32, 16, NarrowKind::SaturateUnsigned),
            X86Opcode::VPMOVUSDW
        );
        assert_eq!(
            n.select_vpmov_opcode(16, 8, NarrowKind::SaturateUnsigned),
            X86Opcode::VPMOVUSWB
        );
    }

    #[test]
    fn test_narrow_signed_min_max_all_bits() {
        for bits in &[8u32, 16, 32] {
            let step = NarrowStep {
                src_elem_bits: bits * 2,
                dst_elem_bits: *bits,
                num_src_elements: 8,
                src_vec_bits: bits * 2 * 8,
                dst_vec_bits: bits * 8,
                kind: NarrowKind::SaturateSigned,
            };
            let min_expected = -(1i64 << (bits - 1));
            let max_expected = (1i64 << (bits - 1)) - 1;
            assert_eq!(
                step.signed_min(),
                min_expected,
                "signed_min for {} bits",
                bits
            );
            assert_eq!(
                step.signed_max(),
                max_expected,
                "signed_max for {} bits",
                bits
            );
        }
    }

    #[test]
    fn test_narrow_unsigned_max_all_bits() {
        for bits in &[8u32, 16, 32] {
            let step = NarrowStep {
                src_elem_bits: bits * 2,
                dst_elem_bits: *bits,
                num_src_elements: 8,
                src_vec_bits: bits * 2 * 8,
                dst_vec_bits: bits * 8,
                kind: NarrowKind::SaturateUnsigned,
            };
            let expected = (1u64 << bits) - 1;
            assert_eq!(
                step.unsigned_max(),
                expected,
                "unsigned_max for {} bits",
                bits
            );
        }
    }

    #[test]
    fn test_narrow_is_pack_supported_all() {
        assert!(NarrowStep::new(32, 8, NarrowKind::SaturateSigned).is_pack_supported());
        assert!(NarrowStep::new(16, 16, NarrowKind::SaturateSigned).is_pack_supported());
        assert!(NarrowStep::new(32, 8, NarrowKind::SaturateUnsigned).is_pack_supported());
        assert!(NarrowStep::new(16, 16, NarrowKind::SaturateUnsigned).is_pack_supported());
        assert!(!NarrowStep::new(64, 4, NarrowKind::SaturateSigned).is_pack_supported());
    }

    #[test]
    fn test_narrow_is_vpmov_supported_all() {
        assert!(NarrowStep::new(64, 4, NarrowKind::SaturateSigned).is_vpmov_supported());
        assert!(NarrowStep::new(64, 8, NarrowKind::SaturateSigned).is_vpmov_supported());
        assert!(NarrowStep::new(32, 4, NarrowKind::SaturateSigned).is_vpmov_supported());
        assert!(NarrowStep::new(32, 8, NarrowKind::SaturateSigned).is_vpmov_supported());
        assert!(NarrowStep::new(16, 16, NarrowKind::SaturateSigned).is_vpmov_supported());
    }

    #[test]
    fn test_narrow_ratio_all_pairs() {
        assert_eq!(NarrowStep::new(32, 8, NarrowKind::Truncate).ratio(), 2);
        assert_eq!(NarrowStep::new(16, 16, NarrowKind::Truncate).ratio(), 2);
        assert_eq!(NarrowStep::new(64, 4, NarrowKind::Truncate).ratio(), 2);
    }

    #[test]
    fn test_narrow_strategy_debug_all() {
        for s in &[
            NarrowStrategy::None,
            NarrowStrategy::SSEPack,
            NarrowStrategy::AVXPack,
            NarrowStrategy::AVX512VPMOV,
        ] {
            assert!(!format!("{:?}", s).is_empty());
        }
    }

    #[test]
    fn test_narrow_mi_with_evex() {
        let mut mi = MachineNarrowInstr::new(X86Opcode::VPMOVWB, 512);
        mi.has_evex = true;
        assert!(mi.has_evex);
    }

    #[test]
    fn test_narrow_mi_with_mem_op() {
        let mut mi = MachineNarrowInstr::new(X86Opcode::VPMOVWB, 512);
        mi.operands.push(NarrowOperand::Mem(0xCAFE));
        assert_eq!(mi.src_reg(0), None);
    }

    #[test]
    fn test_narrow_op_dstmem() {
        assert_eq!(NarrowOperand::DstMem(0x1000), NarrowOperand::DstMem(0x1000));
        assert_ne!(NarrowOperand::DstMem(0x1000), NarrowOperand::Mem(0x1000));
    }

    #[test]
    fn test_narrow_max_per_block_zero() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        n.max_narrows_per_block = 0;
        let mi = make_mi(X86Opcode::PACKSSDW, 128, 1, &[2, 3]);
        assert!(n.run_on_block(&[mi]).is_empty());
    }

    #[test]
    fn test_narrow_vpmov_no_avx512() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVWB, 512, 1, &[2]);
        assert!(n.try_narrow(&mi).is_none());
    }

    #[test]
    fn test_narrow_vpack_no_avx() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPACKSSDW, 256, 1, &[2, 3]);
        assert!(n.try_narrow(&mi).is_none());
    }

    #[test]
    fn test_narrow_result_temp_reg() {
        let step = NarrowStep::truncate(32, 8, 16);
        let r = NarrowResult {
            original_opcode: X86Opcode::MOVDQU,
            narrowed_opcode: X86Opcode::PAND,
            step,
            inserted_trunc: true,
            temp_reg: Some(42),
            had_saturation: false,
        };
        assert_eq!(r.temp_reg, Some(42));
        assert!(r.inserted_trunc);
        assert!(!r.had_saturation);
    }

    #[test]
    fn test_narrow_emitter_trunc_extra_instrs() {
        let mut emitter = NarrowEmitter::new();
        let step = NarrowStep::truncate(32, 8, 16);
        let r = NarrowResult {
            original_opcode: X86Opcode::MOVDQU,
            narrowed_opcode: X86Opcode::PAND,
            step,
            inserted_trunc: true,
            temp_reg: Some(10),
            had_saturation: false,
        };
        emitter.emit_narrow(&r);
        assert_eq!(emitter.sequence.len(), 2);
    }

    #[test]
    fn test_narrow_default_constructors() {
        let st = make_subtarget_sse2();
        let n1 = make_x86_vector_narrow(st);
        assert!(!n1.prefer_truncation);
        let n2 = make_x86_vector_narrow_sse2(st);
        assert!(!n2.allow_vpmov);
        let st512 = make_subtarget_avx512();
        let n3 = make_x86_vector_narrow_avx512(st512);
        assert!(n3.allow_vpmov);
    }

    #[test]
    fn test_narrow_operands_step_roundtrip() {
        let mut ops = NarrowOperands::default();
        ops.src_type_bits = 16;
        ops.dst_type_bits = 8;
        ops.kind = NarrowKind::SaturateUnsigned;
        let step = ops.to_step();
        assert_eq!(step.src_elem_bits, ops.src_type_bits);
        assert_eq!(step.dst_elem_bits, ops.dst_type_bits);
        assert_eq!(step.kind, ops.kind);
    }

    #[test]
    fn test_narrow_block_with_only_nops() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let results = n.run_on_block(&[
            make_mi(X86Opcode::NOP, 0, 1, &[]),
            make_mi(X86Opcode::NOP, 0, 2, &[]),
        ]);
        assert!(results.is_empty());
    }

    #[test]
    fn test_narrow_avx512_all_vpmov_patterns() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let ops = [
            X86Opcode::VPMOVWB,
            X86Opcode::VPMOVDW,
            X86Opcode::VPMOVQW,
            X86Opcode::VPMOVQD,
            X86Opcode::VPMOVDB,
            X86Opcode::VPMOVQB,
        ];
        for &op in &ops {
            let mi = make_mi(op, 512, 1, &[2]);
            assert!(n.try_narrow(&mi).is_some(), "{:?}", op);
        }
    }

    #[test]
    fn test_narrow_avx_all_vpack_patterns() {
        let st = make_subtarget_avx2();
        let mut n = X86VectorNarrow::new(st);
        let ops = [
            X86Opcode::VPACKSSDW,
            X86Opcode::VPACKSSWB,
            X86Opcode::VPACKUSDW,
            X86Opcode::VPACKUSWB,
        ];
        for &op in &ops {
            let mi = make_mi(op, 256, 1, &[2, 3]);
            assert!(n.try_narrow(&mi).is_some(), "{:?}", op);
        }
    }

    #[test]
    fn test_narrow_packssdw_256bit() {
        let st = make_subtarget_avx2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPACKSSDW, 256, 1, &[2, 3]);
        let r = n.try_narrow(&mi).unwrap();
        assert_eq!(r.step.src_vec_bits, 256);
        assert_eq!(r.step.src_elem_bits, 32);
    }

    #[test]
    fn test_narrow_vpmovdw_512bit() {
        let st = make_subtarget_avx512();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::VPMOVDW, 512, 1, &[2]);
        let r = n.try_narrow(&mi).unwrap();
        assert_eq!(r.step.src_vec_bits, 512);
        assert_eq!(r.step.dst_vec_bits, 256);
    }

    #[test]
    fn test_narrow_all_kinds_distinct() {
        assert_ne!(NarrowKind::SaturateSigned, NarrowKind::SaturateUnsigned);
        assert_ne!(NarrowKind::SaturateSigned, NarrowKind::Truncate);
        assert_ne!(NarrowKind::SaturateUnsigned, NarrowKind::Truncate);
    }

    #[test]
    fn test_narrow_trunc_vs_sat_differs() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let val: i64 = 100000;
        let step_sat = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        let step_trunc = NarrowStep::truncate(32, 8, 8);
        assert_ne!(
            n.saturate_value(val, &step_sat),
            n.saturate_value(val, &step_trunc)
        );
    }

    #[test]
    fn test_narrow_operands_all_fields() {
        let ops = NarrowOperands {
            src_type_bits: 32,
            dst_type_bits: 8,
            num_elements: 16,
            kind: NarrowKind::Truncate,
            src1_reg: 1,
            src2_reg: 2,
            dst_reg: 3,
        };
        let step = ops.to_step();
        assert_eq!(step.src_elem_bits, 32);
        assert_eq!(step.dst_elem_bits, 8);
        assert_eq!(step.kind, NarrowKind::Truncate);
    }

    #[test]
    fn test_narrow_multiple_blocks_independent() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        let mi = make_mi(X86Opcode::PACKSSDW, 128, 1, &[2, 3]);
        let b1 = n.run_on_block(&[mi]);
        let b2 = n.run_on_block(&[]);
        assert!(!b1.is_empty());
        assert!(b2.is_empty());
    }

    #[test]
    fn test_narrow_result_all_fields() {
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        let r = NarrowResult {
            original_opcode: X86Opcode::MOVDQU,
            narrowed_opcode: X86Opcode::PACKSSDW,
            step,
            inserted_trunc: false,
            temp_reg: Some(99),
            had_saturation: true,
        };
        assert_eq!(r.narrowed_opcode, X86Opcode::PACKSSDW);
        assert!(r.had_saturation);
        assert_eq!(r.temp_reg, Some(99));
    }

    #[test]
    fn test_narrow_stats_merge_all() {
        let mut a = NarrowStats::default();
        a.packssdw_narrows = 1;
        a.packsswb_narrows = 2;
        a.packusdw_narrows = 3;
        a.packuswb_narrows = 4;
        a.trunc_narrows = 5;
        a.avx_vpmov_narrows = 6;
        a.avx512_vpmov_narrows = 7;
        a.total_narrowed = 28;
        let mut b = NarrowStats::default();
        b.packssdw_narrows = 10;
        a.merge(&b);
        assert_eq!(a.packssdw_narrows, 11);
        assert_eq!(a.total_narrowed, 28);
    }

    #[test]
    fn test_narrow_legal_trunc_sse2() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        assert!(n.is_narrow_legal(&NarrowStep::truncate(32, 8, 4)));
    }

    #[test]
    fn test_narrow_not_legal_64to32_sse2() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        assert!(!n.is_narrow_legal(&NarrowStep::new(64, 4, NarrowKind::SaturateSigned)));
    }

    #[test]
    fn test_narrow_legal_64to32_avx512() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        assert!(n.is_narrow_legal(&NarrowStep::new(64, 4, NarrowKind::SaturateSigned)));
    }

    #[test]
    fn test_narrow_detect_vpmov_signed() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        assert_eq!(
            n.detect_vpmov_kind(X86Opcode::VPMOVSWB),
            NarrowKind::SaturateSigned
        );
    }

    #[test]
    fn test_narrow_detect_vpmov_unsigned() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        assert_eq!(
            n.detect_vpmov_kind(X86Opcode::VPMOVWB),
            NarrowKind::SaturateUnsigned
        );
    }

    #[test]
    fn test_narrow_emitter_consecutive_temps() {
        let mut emitter = NarrowEmitter::new();
        assert_eq!(emitter.alloc_temp() + 1, emitter.alloc_temp());
    }

    #[test]
    fn test_narrow_vpack_is_candidate() {
        assert!(make_mi(X86Opcode::VPACKSSDW, 256, 1, &[2, 3]).is_narrow_candidate());
    }

    #[test]
    fn test_narrow_vpmov_is_candidate() {
        assert!(make_mi(X86Opcode::VPMOVDW, 512, 1, &[2]).is_narrow_candidate());
    }

    #[test]
    fn test_narrow_scalar_not_candidate() {
        assert!(!make_mi(X86Opcode::ADD32rr, 0, 1, &[2, 3]).is_narrow_candidate());
    }

    #[test]
    fn test_narrow_sse2_all_pack_opcodes() {
        let st = make_subtarget_sse2();
        let mut n = X86VectorNarrow::new(st);
        for &op in &[
            X86Opcode::PACKSSDW,
            X86Opcode::PACKSSWB,
            X86Opcode::PACKUSDW,
            X86Opcode::PACKUSWB,
        ] {
            assert!(n.try_narrow(&make_mi(op, 128, 1, &[2, 3])).is_some());
        }
    }

    #[test]
    fn test_narrow_sat_i32_range() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        assert_eq!(n.saturate_value(i32::MAX as i64, &step), 32767);
        assert_eq!(n.saturate_value(i32::MIN as i64, &step), -32768);
    }

    #[test]
    fn test_narrow_usat_u32_range() {
        let st = make_subtarget_sse2();
        let n = X86VectorNarrow::new(st);
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateUnsigned);
        assert_eq!(n.saturate_value(u32::MAX as i64, &step), 65535);
        assert_eq!(n.saturate_value(-1, &step), 0);
    }

    #[test]
    fn test_narrow_emitter_no_trunc_single_instr() {
        let mut emitter = NarrowEmitter::new();
        let step = NarrowStep::new(32, 8, NarrowKind::SaturateSigned);
        let r = NarrowResult {
            original_opcode: X86Opcode::MOVDQU,
            narrowed_opcode: X86Opcode::PACKSSDW,
            step,
            inserted_trunc: false,
            temp_reg: None,
            had_saturation: true,
        };
        emitter.emit_narrow(&r);
        assert_eq!(emitter.sequence.len(), 1);
    }

    #[test]
    fn test_narrow_default_all_config() {
        let n = X86VectorNarrow::default();
        assert!(!n.prefer_truncation);
        assert!(n.allow_vpmov);
        assert_eq!(n.max_narrows_per_block, 64);
    }

    #[test]
    fn test_narrow_step_trunc_ratio_2() {
        assert_eq!(NarrowStep::truncate(32, 16, 8).ratio(), 2);
        assert_eq!(NarrowStep::truncate(16, 8, 16).ratio(), 2);
        assert_eq!(NarrowStep::truncate(64, 32, 4).ratio(), 2);
    }

    #[test]
    fn test_narrow_vpmovswb_opcode() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        assert_eq!(
            n.select_vpmov_opcode(16, 8, NarrowKind::SaturateSigned),
            X86Opcode::VPMOVSWB
        );
    }

    #[test]
    fn test_narrow_vpmovuswb_opcode() {
        let st = make_subtarget_avx512();
        let n = X86VectorNarrow::new(st);
        assert_eq!(
            n.select_vpmov_opcode(16, 8, NarrowKind::SaturateUnsigned),
            X86Opcode::VPMOVUSWB
        );
    }

    #[test]
    fn test_narrow_mi_num_elements_edge() {
        let mi = MachineNarrowInstr::new(X86Opcode::PACKSSDW, 256);
        assert_eq!(mi.num_elements(32), 8);
        assert_eq!(mi.num_elements(0), 0);
        assert_eq!(mi.num_elements(1), 256);
    }
}