llvm-native-core 0.1.10

//! X86 Vector Widen — widen vector operations from narrow element types to
//! wider element types (e.g. v8i8→v8i16), widening loads, widening
//! arithmetic with wider intermediates, and SSE/AVX/AVX-512 widening
//! patterns.
//! Phase 10 — LLVM.TARGET.X86.1 Court.
//!
//! Clean-room behavioral reconstruction from:
//! - Intel® 64 and IA-32 Architectures Software Developer's Manual
//!   (volumes 1, 2A, 2B, 2C, 2D, 3A, 3B, 3C, 3D)
//! - AMD64 Architecture Programmer's Manual (volumes 1-5)
//! - Agner Fog's instruction tables and microarchitecture guides
//!
//! Zero LLVM source code consultation. All behavior reconstructed from
//! published specifications and black-box oracle interrogation.
//!
//! Covered widening operations:
//! - v8i8 → v8i16 (punpcklbw/punpckhbw + sign extend)
//! - v4i16 → v4i32 (punpcklwd/punpckhwd + sign extend)
//! - v2i32 → v2i64 (punpckldq/punpckhdq + sign extend)
//! - v4i8 → v4i32 (pmovsxbd/pmovzxbd)
//! - v8i8 → v8i16 (pmovsxbw/pmovzxbw)
//! - v4i16 → v4i32 (pmovsxwd/pmovzxwd)
//! - v2i32 → v2i64 (pmovsxdq/pmovzxdq)
//! - Widening loads (sign-extend / zero-extend from memory)
//! - Widening arithmetic (add/sub/mul via wider intermediates)
//! - SSE/AVX/AVX-512 widening patterns (vpmovsx*, vpmovzx*)

use crate::x86::x86_instr_info::X86Opcode;
use crate::x86::x86_subtarget::X86Subtarget;

// ═══════════════════════════════════════════════════════════════════════════════
// Vector widen enums and types
// ═══════════════════════════════════════════════════════════════════════════════

/// The kind of widening operation: sign-extend or zero-extend.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum WidenKind {
    /// Sign-extend: high bits are filled with the sign bit of the source.
    SignExtend,
    /// Zero-extend: high bits are filled with zero.
    ZeroExtend,
}

impl std::fmt::Display for WidenKind {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            WidenKind::SignExtend => write!(f, "sext"),
            WidenKind::ZeroExtend => write!(f, "zext"),
        }
    }
}

/// Describes a single widening step: source element type → destination element
/// type, the number of elements, and the total vector bit width.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct WidenStep {
    /// Source element bit width (e.g. 8 for i8).
    pub src_elem_bits: u32,
    /// Destination element bit width (e.g. 16 for i16).
    pub dst_elem_bits: u32,
    /// Number of elements in the source vector.
    pub num_src_elements: u32,
    /// Total source vector bit width.
    pub src_vec_bits: u32,
    /// Total destination vector bit width.
    pub dst_vec_bits: u32,
}

impl WidenStep {
    /// Create a new WidenStep for a standard element-doubling widen.
    pub fn new(src_elem_bits: u32, num_elements: u32) -> Self {
        let dst_elem_bits = src_elem_bits * 2;
        Self {
            src_elem_bits,
            dst_elem_bits,
            num_src_elements: num_elements,
            src_vec_bits: src_elem_bits * num_elements,
            dst_vec_bits: dst_elem_bits * num_elements,
        }
    }

    /// Create a WidenStep for a sign-extending load from narrow memory to wide
    /// register.
    pub fn load_widen(mem_elem_bits: u32, num_elements: u32, reg_elem_bits: u32) -> Self {
        Self {
            src_elem_bits: mem_elem_bits,
            dst_elem_bits: reg_elem_bits,
            num_src_elements: num_elements,
            src_vec_bits: mem_elem_bits * num_elements,
            dst_vec_bits: reg_elem_bits * num_elements,
        }
    }

    /// The widening factor (dst_elem_bits / src_elem_bits).
    pub fn factor(&self) -> u32 {
        if self.src_elem_bits == 0 {
            return 0;
        }
        self.dst_elem_bits / self.src_elem_bits
    }

    /// Whether this widen step is supported by a single pmovsx/pmovzx
    /// instruction (SSE4.1+).
    pub fn is_pmov_supported(&self) -> bool {
        matches!(
            (self.src_elem_bits, self.dst_elem_bits),
            (8, 16) | (8, 32) | (8, 64) | (16, 32) | (16, 64) | (32, 64)
        )
    }
}

/// The full set of widening operands for a vector widen operation.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WidenOperands {
    /// The source vector element type bit width.
    pub src_type_bits: u32,
    /// The destination vector element type bit width.
    pub dst_type_bits: u32,
    /// Number of source elements.
    pub num_elements: u32,
    /// Whether this is sign-extend or zero-extend.
    pub kind: WidenKind,
    /// Source operand register (or None if from memory).
    pub src_reg: Option<u32>,
    /// Memory address (or None if from register).
    pub src_addr: Option<u64>,
    /// Destination register.
    pub dst_reg: u32,
    /// Whether the source is in the low or high half of the register.
    pub low_half: bool,
}

impl Default for WidenOperands {
    fn default() -> Self {
        Self {
            src_type_bits: 8,
            dst_type_bits: 16,
            num_elements: 8,
            kind: WidenKind::SignExtend,
            src_reg: None,
            src_addr: None,
            dst_reg: 0,
            low_half: true,
        }
    }
}

impl WidenOperands {
    /// Compute the step descriptor from these operands.
    pub fn to_step(&self) -> WidenStep {
        WidenStep {
            src_elem_bits: self.src_type_bits,
            dst_elem_bits: self.dst_type_bits,
            num_src_elements: self.num_elements,
            src_vec_bits: self.src_type_bits * self.num_elements,
            dst_vec_bits: self.dst_type_bits * self.num_elements,
        }
    }

    /// Check whether this forms a valid pmovsx/pmovzx widening.
    pub fn is_valid_pmov(&self) -> bool {
        self.to_step().is_pmov_supported()
    }
}

/// Record of a successfully widened instruction.
#[derive(Debug, Clone)]
pub struct WidenResult {
    /// The original instruction opcode before widening.
    pub original_opcode: X86Opcode,
    /// The widened instruction opcode.
    pub widened_opcode: X86Opcode,
    /// The step descriptor for the widen.
    pub step: WidenStep,
    /// Whether a separate sign-extend step was needed.
    pub inserted_sext: bool,
    /// The intermediate register used (if any).
    pub temp_reg: Option<u32>,
}

/// Statistics for the vector widen pass.
#[derive(Debug, Clone, Default)]
pub struct WidenStats {
    /// Number of unpck-based widens (SSE2).
    pub unpck_widens: u32,
    /// Number of pmovsx/pmovzx widens (SSE4.1).
    pub pmov_widens: u32,
    /// Number of avx pmovsx/pmovzx widens.
    pub avx_pmov_widens: u32,
    /// Number of avx512 widening operations.
    pub avx512_widens: u32,
    /// Number of widening loads emitted.
    pub widening_loads: u32,
    /// Number of widening arithmetic ops (add/sub/mul via wider intermediate).
    pub widening_arith: u32,
    /// Total number of instructions widened.
    pub total_widened: u32,
    /// Number of candidates examined.
    pub candidates_examined: u32,
}

impl WidenStats {
    /// Merge another set of stats into this one.
    pub fn merge(&mut self, other: &WidenStats) {
        self.unpck_widens += other.unpck_widens;
        self.pmov_widens += other.pmov_widens;
        self.avx_pmov_widens += other.avx_pmov_widens;
        self.avx512_widens += other.avx512_widens;
        self.widening_loads += other.widening_loads;
        self.widening_arith += other.widening_arith;
        self.total_widened += other.total_widened;
        self.candidates_examined += other.candidates_examined;
    }

    /// Whether the widen pass made any changes.
    pub fn made_progress(&self) -> bool {
        self.total_widened > 0
    }

    /// Produce a human-readable summary.
    pub fn summary(&self) -> String {
        format!(
            "X86VectorWiden: {} unpck, {} pmov, {} avx-pmov, {} avx512, \
             {} loads, {} arith → {} total widened ({} candidates)",
            self.unpck_widens,
            self.pmov_widens,
            self.avx_pmov_widens,
            self.avx512_widens,
            self.widening_loads,
            self.widening_arith,
            self.total_widened,
            self.candidates_examined,
        )
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// X86VectorWiden — main widen engine
// ═══════════════════════════════════════════════════════════════════════════════

/// Vector widen pass for X86.
///
/// Widens vector operations from narrow element types to wider types using
/// native SSE/AVX/AVX-512 instructions. This enables better use of wider
/// execution units and can eliminate redundant sign/zero extension chains.
///
/// # Widening Strategies
///
/// | Source → Dest  | SSE2                    | SSE4.1+ / AVX / AVX-512       |
/// |----------------|-------------------------|-------------------------------|
/// | v8i8  → v8i16  | punpcklbw + psraw       | pmovsxbw / pmovzxbw           |
/// | v4i16 → v4i32  | punpcklwd + psrad       | pmovsxwd / pmovzxwd           |
/// | v2i32 → v2i64  | punpckldq + psrad xmm   | pmovsxdq / pmovzxdq           |
/// | v4i8  → v4i32  | — (multi-step)          | pmovsxbd / pmovzxbd           |
/// | v2i16 → v2i64  | — (multi-step)          | pmovsxwq / pmovzxwq           |
/// | v8i8  → v8i16  | — (multi-step)          | pmovsxbw / pmovzxbw (mem)     |
pub struct X86VectorWiden {
    /// The ISA subtarget providing feature flags.
    pub subtarget: X86Subtarget,
    /// Accumulated statistics.
    pub stats: WidenStats,
    /// Whether to prefer unpck-based widen over pmov on SSE4.1 when latencies
    /// favor unpck.
    pub prefer_unpck: bool,
    /// Whether to allow widening of memory operands (generates load+extend).
    pub allow_widening_loads: bool,
    /// Whether to perform widening arithmetic (add/sub/mul via wider
    /// intermediate).
    pub allow_widening_arith: bool,
    /// Maximum number of widen transformations per basic block.
    pub max_widens_per_block: usize,
    /// Cache of matched widen patterns for reuse.
    pattern_cache: Vec<WidenResult>,
}

impl X86VectorWiden {
    /// Create a new X86VectorWiden for the given subtarget.
    pub fn new(subtarget: X86Subtarget) -> Self {
        Self {
            subtarget,
            stats: WidenStats::default(),
            prefer_unpck: false,
            allow_widening_loads: true,
            allow_widening_arith: true,
            max_widens_per_block: 64,
            pattern_cache: Vec::new(),
        }
    }

    /// Create a new X86VectorWiden with SSE2-only features (no pmov).
    pub fn new_sse2(subtarget: X86Subtarget) -> Self {
        let mut w = Self::new(subtarget);
        w.prefer_unpck = true;
        w
    }

    /// Create a new X86VectorWiden that favors pmovsx/pmovzx (SSE4.1+).
    pub fn new_sse41(subtarget: X86Subtarget) -> Self {
        let mut w = Self::new(subtarget);
        w.prefer_unpck = false;
        w
    }

    /// Run the widen pass on a block of machine instructions.
    ///
    /// Returns the list of widened instruction results.
    pub fn run_on_block(&mut self, instructions: &[MachineWidenInstr]) -> Vec<WidenResult> {
        let mut results = Vec::new();
        self.pattern_cache.clear();

        for mi in instructions {
            self.stats.candidates_examined += 1;

            if results.len() >= self.max_widens_per_block {
                break;
            }

            if let Some(result) = self.try_widen(mi) {
                self.record_widen(&result);
                results.push(result);
            }
        }

        self.stats.total_widened += results.len() as u32;
        results
    }

    /// Try to widen a single machine instruction.
    pub fn try_widen(&mut self, mi: &MachineWidenInstr) -> Option<WidenResult> {
        // Dispatch based on opcode to specific widening strategies.
        match mi.opcode {
            // Pack/unpack instructions — classic widening via unpck + extend
            X86Opcode::PUNPCKLBW => self.try_widen_unpck(mi, 8, 16, true),
            X86Opcode::PUNPCKHBW => self.try_widen_unpck(mi, 8, 16, false),
            X86Opcode::PUNPCKLWD => self.try_widen_unpck(mi, 16, 32, true),
            X86Opcode::PUNPCKHWD => self.try_widen_unpck(mi, 16, 32, false),
            X86Opcode::PUNPCKLDQ => self.try_widen_unpck(mi, 32, 64, true),
            X86Opcode::PUNPCKHDQ => self.try_widen_unpck(mi, 32, 64, false),
            X86Opcode::PUNPCKLQDQ => self.try_widen_unpck(mi, 64, 128, true),
            // pmovsx/pmovzx variants
            X86Opcode::PMOVSXBW | X86Opcode::PMOVZXBW => self.try_widen_pmov(mi, 8, 16),
            X86Opcode::PMOVSXWD | X86Opcode::PMOVZXWD => self.try_widen_pmov(mi, 16, 32),
            X86Opcode::PMOVSXDQ | X86Opcode::PMOVZXDQ => self.try_widen_pmov(mi, 32, 64),
            X86Opcode::PMOVSXBD | X86Opcode::PMOVZXBD => self.try_widen_pmov(mi, 8, 32),
            X86Opcode::PMOVSXWQ | X86Opcode::PMOVZXWQ => self.try_widen_pmov(mi, 16, 64),
            X86Opcode::PMOVSXBQ | X86Opcode::PMOVZXBQ => self.try_widen_pmov(mi, 8, 64),
            // arithmetic ops that can be widened
            X86Opcode::PADDB | X86Opcode::PADDW | X86Opcode::PADDD => self.try_widen_arith(mi),
            X86Opcode::PSUBB | X86Opcode::PSUBW | X86Opcode::PSUBD => self.try_widen_arith(mi),
            X86Opcode::PMULLW | X86Opcode::PMULLD => self.try_widen_arith(mi),
            // VPMOVSX / VPMOVZX AVX variants
            X86Opcode::VPMOVSXBW | X86Opcode::VPMOVZXBW => self.try_widen_avx_pmov(mi, 8, 16),
            X86Opcode::VPMOVSXWD | X86Opcode::VPMOVZXWD => self.try_widen_avx_pmov(mi, 16, 32),
            X86Opcode::VPMOVSXDQ | X86Opcode::VPMOVZXDQ => self.try_widen_avx_pmov(mi, 32, 64),
            X86Opcode::VPMOVSXBD | X86Opcode::VPMOVZXBD => self.try_widen_avx_pmov(mi, 8, 32),
            X86Opcode::VPMOVSXWQ | X86Opcode::VPMOVZXWQ => self.try_widen_avx_pmov(mi, 16, 64),
            X86Opcode::VPMOVSXBQ | X86Opcode::VPMOVZXBQ => self.try_widen_avx_pmov(mi, 8, 64),
            _ => None,
        }
    }

    /// Try to match an unpck-based widening pattern.
    ///
    /// An unpck instruction interleaves bytes/words/dwords from two sources.
    /// When both sources are the same register, the result is a duplication
    /// of each element pair, which can be followed by an arithmetic right
    /// shift to sign-extend.
    fn try_widen_unpck(
        &mut self,
        mi: &MachineWidenInstr,
        src_bits: u32,
        dst_bits: u32,
        low: bool,
    ) -> Option<WidenResult> {
        let _src_reg = mi.src_reg(0)?;
        let _dst_reg = mi.dst_reg()?;

        // Determine the number of elements and bit widths.
        let vec_bits = mi.vector_width_bits();
        let src_num_elements = vec_bits / src_bits;
        // After interleave, each pair becomes two wider elements:
        // src_num_elements / 2 elements of dst_bits each.
        let dst_num_elements = src_num_elements / 2;

        let step = WidenStep {
            src_elem_bits: src_bits,
            dst_elem_bits: dst_bits,
            num_src_elements: src_num_elements,
            src_vec_bits: vec_bits,
            dst_vec_bits: dst_bits * dst_num_elements,
        };

        // Check if the consumer is a sign-extending shift (psraw/psrad/psraq).
        // For zero-extend, we'd check for pand/pandn with a mask.
        let kind = self.detect_extend_kind(mi, src_bits, dst_bits);

        Some(WidenResult {
            original_opcode: mi.opcode,
            widened_opcode: self.select_widened_unpck_opcode(src_bits, dst_bits, low, kind),
            step,
            inserted_sext: kind == WidenKind::SignExtend,
            temp_reg: None,
        })
    }

    /// Try to match a pmovsx/pmovzx widening pattern (SSE4.1).
    fn try_widen_pmov(
        &mut self,
        mi: &MachineWidenInstr,
        src_bits: u32,
        dst_bits: u32,
    ) -> Option<WidenResult> {
        if !self.subtarget.has_sse41() {
            return None;
        }

        let _src_reg = mi.src_reg(0)?;
        let _dst_reg = mi.dst_reg()?;

        let vec_bits = mi.vector_width_bits();
        let num_elements = vec_bits / src_bits;

        let kind = match mi.opcode {
            X86Opcode::PMOVSXBW
            | X86Opcode::PMOVSXWD
            | X86Opcode::PMOVSXDQ
            | X86Opcode::PMOVSXBD
            | X86Opcode::PMOVSXWQ
            | X86Opcode::PMOVSXBQ => WidenKind::SignExtend,
            _ => WidenKind::ZeroExtend,
        };

        let step = WidenStep {
            src_elem_bits: src_bits,
            dst_elem_bits: dst_bits,
            num_src_elements: num_elements,
            src_vec_bits: vec_bits,
            dst_vec_bits: dst_bits * num_elements,
        };

        let widened_opcode = self.select_pmov_opcode(src_bits, dst_bits, kind, false);

        Some(WidenResult {
            original_opcode: mi.opcode,
            widened_opcode,
            step,
            inserted_sext: false,
            temp_reg: None,
        })
    }

    /// Try to match an AVX vpmovsx/vpmovzx widening pattern.
    fn try_widen_avx_pmov(
        &mut self,
        mi: &MachineWidenInstr,
        src_bits: u32,
        dst_bits: u32,
    ) -> Option<WidenResult> {
        if !self.subtarget.has_avx() {
            return None;
        }

        let _src_reg = mi.src_reg(0)?;
        let _dst_reg = mi.dst_reg()?;
        let vec_bits = mi.vector_width_bits();
        let num_elements = vec_bits / src_bits;

        let kind = match mi.opcode {
            X86Opcode::VPMOVSXBW
            | X86Opcode::VPMOVSXWD
            | X86Opcode::VPMOVSXDQ
            | X86Opcode::VPMOVSXBD
            | X86Opcode::VPMOVSXWQ
            | X86Opcode::VPMOVSXBQ => WidenKind::SignExtend,
            _ => WidenKind::ZeroExtend,
        };

        let step = WidenStep {
            src_elem_bits: src_bits,
            dst_elem_bits: dst_bits,
            num_src_elements: num_elements,
            src_vec_bits: vec_bits,
            dst_vec_bits: dst_bits * num_elements,
        };

        let widened_opcode = self.select_pmov_opcode(src_bits, dst_bits, kind, true);

        Some(WidenResult {
            original_opcode: mi.opcode,
            widened_opcode,
            step,
            inserted_sext: false,
            temp_reg: None,
        })
    }

    /// Try to widen an arithmetic operation by promoting to a wider element
    /// type, performing the operation at the wider type, and truncating back.
    fn try_widen_arith(&mut self, mi: &MachineWidenInstr) -> Option<WidenResult> {
        if !self.allow_widening_arith {
            return None;
        }

        // Determine element size from the opcode.
        let elem_bits = match mi.opcode {
            X86Opcode::PADDB | X86Opcode::PSUBB => 8,
            X86Opcode::PADDW | X86Opcode::PSUBW | X86Opcode::PMULLW => 16,
            X86Opcode::PADDD | X86Opcode::PSUBD | X86Opcode::PMULLD => 32,
            _ => return None,
        };

        // Only widen 8-bit ops to 16-bit, and 16-bit to 32-bit.
        // 32-bit to 64-bit is rarely profitable.
        if elem_bits >= 32 {
            return None;
        }

        let wide_bits = elem_bits * 2;
        let _src_reg = mi.src_reg(0)?;
        let dst_reg = mi.dst_reg()?;
        let vec_bits = mi.vector_width_bits();
        let num_elements = vec_bits / elem_bits;

        let step = WidenStep {
            src_elem_bits: elem_bits,
            dst_elem_bits: wide_bits,
            num_src_elements: num_elements,
            src_vec_bits: vec_bits,
            dst_vec_bits: wide_bits * num_elements,
        };

        Some(WidenResult {
            original_opcode: mi.opcode,
            widened_opcode: self.select_widened_arith_opcode(mi.opcode, wide_bits),
            step,
            inserted_sext: false,
            temp_reg: Some(dst_reg + 1), // temporary for intermediate
        })
    }

    /// Detect whether the consumer of the unpck result is sign-extending or
    /// zero-extending.
    fn detect_extend_kind(
        &self,
        _mi: &MachineWidenInstr,
        src_bits: u32,
        dst_bits: u32,
    ) -> WidenKind {
        // In practice, we look at the consumer instruction.
        // A psraw/psrad/psraq with shift amount = src_bits indicates sign
        // extend. A pand with mask indicates zero extend.
        // For now, default to sign-extend as it is the most common pattern.
        let _shift_amount = dst_bits - src_bits;
        WidenKind::SignExtend
    }

    /// Select the widened opcode for an unpck-based widen.
    fn select_widened_unpck_opcode(
        &self,
        src_bits: u32,
        dst_bits: u32,
        low: bool,
        kind: WidenKind,
    ) -> X86Opcode {
        match (src_bits, dst_bits, low, kind) {
            (8, 16, true, WidenKind::SignExtend) => X86Opcode::PUNPCKLBW,
            (8, 16, false, WidenKind::SignExtend) => X86Opcode::PUNPCKHBW,
            (16, 32, true, WidenKind::SignExtend) => X86Opcode::PUNPCKLWD,
            (16, 32, false, WidenKind::SignExtend) => X86Opcode::PUNPCKHWD,
            (32, 64, true, _) => X86Opcode::PUNPCKLDQ,
            (32, 64, false, _) => X86Opcode::PUNPCKHDQ,
            _ => X86Opcode::PUNPCKLBW, // default fallback
        }
    }

    /// Select the pmovsx/pmovzx opcode for the given src/dst bit widths.
    fn select_pmov_opcode(
        &self,
        src_bits: u32,
        dst_bits: u32,
        kind: WidenKind,
        is_avx: bool,
    ) -> X86Opcode {
        if is_avx {
            match (src_bits, dst_bits, kind) {
                (8, 16, WidenKind::SignExtend) => X86Opcode::VPMOVSXBW,
                (8, 16, WidenKind::ZeroExtend) => X86Opcode::VPMOVZXBW,
                (8, 32, WidenKind::SignExtend) => X86Opcode::VPMOVSXBD,
                (8, 32, WidenKind::ZeroExtend) => X86Opcode::VPMOVZXBD,
                (8, 64, WidenKind::SignExtend) => X86Opcode::VPMOVSXBQ,
                (8, 64, WidenKind::ZeroExtend) => X86Opcode::VPMOVZXBQ,
                (16, 32, WidenKind::SignExtend) => X86Opcode::VPMOVSXWD,
                (16, 32, WidenKind::ZeroExtend) => X86Opcode::VPMOVZXWD,
                (16, 64, WidenKind::SignExtend) => X86Opcode::VPMOVSXWQ,
                (16, 64, WidenKind::ZeroExtend) => X86Opcode::VPMOVZXWQ,
                (32, 64, WidenKind::SignExtend) => X86Opcode::VPMOVSXDQ,
                (32, 64, WidenKind::ZeroExtend) => X86Opcode::VPMOVZXDQ,
                _ => X86Opcode::VPMOVSXBW,
            }
        } else {
            match (src_bits, dst_bits, kind) {
                (8, 16, WidenKind::SignExtend) => X86Opcode::PMOVSXBW,
                (8, 16, WidenKind::ZeroExtend) => X86Opcode::PMOVZXBW,
                (8, 32, WidenKind::SignExtend) => X86Opcode::PMOVSXBD,
                (8, 32, WidenKind::ZeroExtend) => X86Opcode::PMOVZXBD,
                (8, 64, WidenKind::SignExtend) => X86Opcode::PMOVSXBQ,
                (8, 64, WidenKind::ZeroExtend) => X86Opcode::PMOVZXBQ,
                (16, 32, WidenKind::SignExtend) => X86Opcode::PMOVSXWD,
                (16, 32, WidenKind::ZeroExtend) => X86Opcode::PMOVZXWD,
                (16, 64, WidenKind::SignExtend) => X86Opcode::PMOVSXWQ,
                (16, 64, WidenKind::ZeroExtend) => X86Opcode::PMOVZXWQ,
                (32, 64, WidenKind::SignExtend) => X86Opcode::PMOVSXDQ,
                (32, 64, WidenKind::ZeroExtend) => X86Opcode::PMOVZXDQ,
                _ => X86Opcode::PMOVSXBW,
            }
        }
    }

    /// Select the widened arithmetic opcode for a given element size.
    fn select_widened_arith_opcode(&self, original: X86Opcode, wide_bits: u32) -> X86Opcode {
        match (original, wide_bits) {
            (X86Opcode::PADDB, 16) => X86Opcode::PADDW,
            (X86Opcode::PSUBB, 16) => X86Opcode::PSUBW,
            (X86Opcode::PADDW, 32) => X86Opcode::PADDD,
            (X86Opcode::PSUBW, 32) => X86Opcode::PSUBD,
            (X86Opcode::PMULLW, 32) => X86Opcode::PMULLD,
            _ => original,
        }
    }

    /// Record a successful widen in the statistics.
    fn record_widen(&mut self, result: &WidenResult) {
        let opcode = result.widened_opcode;
        let is_avx = (opcode as u32) >= (X86Opcode::VPMOVSXBW as u32)
            && (opcode as u32) <= (X86Opcode::VPMOVZXBQ as u32);

        let is_avx512 = (opcode as u32) >= (X86Opcode::VPMOVSXBW as u32) + 1000;

        if is_avx512 {
            self.stats.avx512_widens += 1;
        } else if is_avx {
            self.stats.avx_pmov_widens += 1;
        } else if self.is_pmov_opcode(opcode) {
            self.stats.pmov_widens += 1;
        } else {
            self.stats.unpck_widens += 1;
        }

        if result.inserted_sext {
            // The sext itself counts as an extra unpck widen
        }
    }

    /// Check whether an opcode is a pmovsx/pmovzx variant.
    fn is_pmov_opcode(&self, opcode: X86Opcode) -> bool {
        let o = opcode as u32;
        let base = X86Opcode::PMOVSXBW as u32;
        let end = X86Opcode::PMOVZXBQ as u32;
        o >= base && o <= end
    }

    /// Widen a load operation by sign-extending or zero-extending from a
    /// narrow memory type to a wide register type.
    pub fn widen_load(
        &mut self,
        src_bits: u32,
        dst_bits: u32,
        num_elements: u32,
        kind: WidenKind,
        _addr: u64,
        _dst_reg: u32,
    ) -> Option<WidenResult> {
        if !self.allow_widening_loads {
            return None;
        }

        let step = WidenStep {
            src_elem_bits: src_bits,
            dst_elem_bits: dst_bits,
            num_src_elements: num_elements,
            src_vec_bits: src_bits * num_elements,
            dst_vec_bits: dst_bits * num_elements,
        };

        if !step.is_pmov_supported() {
            return None;
        }

        let use_avx = self.subtarget.has_avx();
        let widened_opcode = self.select_pmov_opcode(src_bits, dst_bits, kind, use_avx);

        self.stats.widening_loads += 1;

        Some(WidenResult {
            original_opcode: X86Opcode::MOVDQU, // load was a mov
            widened_opcode,
            step,
            inserted_sext: false,
            temp_reg: None,
        })
    }

    /// Check whether a given widen is legal on the current subtarget.
    pub fn is_widen_legal(&self, step: &WidenStep, _kind: WidenKind) -> bool {
        if step.src_vec_bits > 512 || step.dst_vec_bits > 512 {
            return false;
        }

        if step.is_pmov_supported() {
            // pmovsx/pmovzx require SSE4.1
            self.subtarget.has_sse41()
        } else {
            // unpck-based widen requires SSE2
            self.subtarget.has_sse2()
        }
    }

    /// Get the optimal widen strategy for a given source→destination pair.
    pub fn get_widen_strategy(&self, src_bits: u32, dst_bits: u32) -> WidenStrategy {
        if self.subtarget.has_avx512bw() && dst_bits >= 16 {
            WidenStrategy::AVX512
        } else if self.subtarget.has_avx() && src_bits < dst_bits {
            WidenStrategy::AVXPMOV
        } else if self.subtarget.has_sse41() && src_bits < dst_bits {
            WidenStrategy::PMOV
        } else if self.subtarget.has_sse2() {
            WidenStrategy::Unpck
        } else {
            WidenStrategy::None
        }
    }

    /// Clear all internal state and statistics.
    pub fn clear(&mut self) {
        self.stats = WidenStats::default();
        self.pattern_cache.clear();
    }
}

/// The widening strategy to use for a given ISA level.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WidenStrategy {
    /// No widening is available.
    None,
    /// Use punpck + arithmetic shift (SSE2).
    Unpck,
    /// Use pmovsx/pmovzx (SSE4.1).
    PMOV,
    /// Use vpmovsx/vpmovzx (AVX/AVX2).
    AVXPMOV,
    /// Use AVX-512 vpmovsx/vpmovzx with masking.
    AVX512,
}

impl Default for X86VectorWiden {
    fn default() -> Self {
        Self {
            subtarget: X86Subtarget::new("x86_64-unknown-linux-gnu", "generic", ""),
            stats: WidenStats::default(),
            prefer_unpck: false,
            allow_widening_loads: true,
            allow_widening_arith: true,
            max_widens_per_block: 64,
            pattern_cache: Vec::new(),
        }
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Simplified machine instruction model for the widen pass
// ═══════════════════════════════════════════════════════════════════════════════

/// A simplified machine instruction used by the vector widen pass.
#[derive(Debug, Clone)]
pub struct MachineWidenInstr {
    /// The opcode of this instruction.
    pub opcode: X86Opcode,
    /// Operands of the instruction.
    pub operands: Vec<WidenOperand>,
    /// The destination register (if any).
    pub def_reg: Option<u32>,
    /// The vector width in bits (128, 256, 512).
    pub vec_width: u32,
    /// Whether this instruction has an EVEX prefix.
    pub has_evex: bool,
    /// Whether this is a memory operand load.
    pub is_load: bool,
    /// Memory address (if applicable).
    pub mem_addr: Option<u64>,
}

impl MachineWidenInstr {
    /// Create a new MachineWidenInstr.
    pub fn new(opcode: X86Opcode, vec_width: u32) -> Self {
        Self {
            opcode,
            operands: Vec::new(),
            def_reg: None,
            vec_width,
            has_evex: false,
            is_load: false,
            mem_addr: None,
        }
    }

    /// Get the source register at the given operand index.
    pub fn src_reg(&self, idx: usize) -> Option<u32> {
        self.operands.get(idx).and_then(|op| match op {
            WidenOperand::Reg(r) => Some(*r),
            _ => None,
        })
    }

    /// Get the destination register.
    pub fn dst_reg(&self) -> Option<u32> {
        self.def_reg
    }

    /// Get an immediate operand.
    pub fn imm_op(&self, idx: usize) -> Option<i64> {
        self.operands.get(idx).and_then(|op| match op {
            WidenOperand::Imm(v) => Some(*v),
            _ => None,
        })
    }

    /// Get the vector width in bits.
    pub fn vector_width_bits(&self) -> u32 {
        self.vec_width
    }

    /// The number of elements given an element bit width.
    pub fn num_elements(&self, elem_bits: u32) -> u32 {
        if elem_bits == 0 {
            0
        } else {
            self.vec_width / elem_bits
        }
    }

    /// Whether this instruction is a widen candidate.
    pub fn is_widen_candidate(&self) -> bool {
        let op = self.opcode as u32;
        // unpck, pmovsx, pmovzx, arithmetic on narrow types
        let unpack_base = X86Opcode::PUNPCKLBW as u32;
        let unpack_end = X86Opcode::PUNPCKHQDQ as u32;
        let pmov_base = X86Opcode::PMOVSXBW as u32;
        let pmov_end = X86Opcode::PMOVZXBQ as u32;
        let arith_base = X86Opcode::PADDB as u32;
        let arith_end = X86Opcode::PMULLD as u32;

        (op >= unpack_base && op <= unpack_end)
            || (op >= pmov_base && op <= pmov_end)
            || (op >= arith_base && op <= arith_end)
    }
}

/// An operand in the vector widen instruction model.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum WidenOperand {
    /// A register operand.
    Reg(u32),
    /// An immediate operand.
    Imm(i64),
    /// A memory operand.
    Mem(u64),
    /// A label/global operand (e.g. for branch targets).
    Label(String),
}

// ═══════════════════════════════════════════════════════════════════════════════
// AVX-512 specific widening utilities
// ═══════════════════════════════════════════════════════════════════════════════

/// AVX-512 widening context, including masking and embedded broadcast.
#[derive(Debug, Clone)]
pub struct AVX512WidenContext {
    /// The mask register (k0-k7), or None for no masking.
    pub mask_reg: Option<u32>,
    /// Whether zero-masking is used (vs merge-masking).
    pub zero_mask: bool,
    /// Whether embedded broadcast is used (load+extend in one instruction).
    pub embedded_broadcast: bool,
    /// The rounding mode override (if any).
    pub rounding: Option<AVX512Rounding>,
    /// Whether suppress-all-exceptions (SAE) is active.
    pub sae: bool,
}

impl Default for AVX512WidenContext {
    fn default() -> Self {
        Self {
            mask_reg: None,
            zero_mask: false,
            embedded_broadcast: false,
            rounding: None,
            sae: false,
        }
    }
}

/// AVX-512 rounding modes for vector instructions.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AVX512Rounding {
    /// Round to nearest even.
    RN,
    /// Round toward negative infinity.
    RD,
    /// Round toward positive infinity.
    RU,
    /// Round toward zero (truncate).
    RZ,
}

impl AVX512Rounding {
    /// Get the EVEX rounding bits for this mode.
    pub fn evex_bits(&self) -> u8 {
        match self {
            AVX512Rounding::RN => 0b00,
            AVX512Rounding::RD => 0b01,
            AVX512Rounding::RU => 0b10,
            AVX512Rounding::RZ => 0b11,
        }
    }
}

// ═══════════════════════════════════════════════════════════════════════════════
// Widen instruction emitter
// ═══════════════════════════════════════════════════════════════════════════════

/// Emits widened machine instructions from a WidenResult.
#[derive(Debug, Clone)]
pub struct WidenEmitter {
    /// The sequence of emitted instructions.
    pub sequence: Vec<WidenEmittedInstr>,
    /// Temporary register counter for allocation.
    next_temp: u32,
}

impl WidenEmitter {
    /// Create a new widen emitter.
    pub fn new() -> Self {
        Self {
            sequence: Vec::new(),
            next_temp: 100, // start temp regs at index 100
        }
    }

    /// Emit the instructions needed to perform a widen.
    pub fn emit_widen(&mut self, result: &WidenResult) {
        let opcode = result.widened_opcode as u32;
        let is_unpck =
            opcode >= (X86Opcode::PUNPCKLBW as u32) && opcode <= (X86Opcode::PUNPCKHQDQ as u32);

        if is_unpck && result.inserted_sext {
            self.emit_unpck_widen(result);
        } else {
            self.emit_pmov_widen(result);
        }
    }

    /// Emit an unpck-based widen (unpck + arithmetic shift).
    fn emit_unpck_widen(&mut self, result: &WidenResult) {
        // Step 1: unpck instruction
        self.sequence.push(WidenEmittedInstr {
            opcode: result.widened_opcode as u32,
            def_reg: result.step.dst_elem_bits * result.step.num_src_elements / 2,
            use_regs: vec![result.step.src_elem_bits],
            imm: None,
            comment: format!(
                "widen {}→{} (unpck)",
                result.step.src_elem_bits, result.step.dst_elem_bits
            ),
        });
    }

    /// Emit a pmov-based widen (single instruction).
    fn emit_pmov_widen(&mut self, result: &WidenResult) {
        self.sequence.push(WidenEmittedInstr {
            opcode: result.widened_opcode as u32,
            def_reg: result.step.dst_elem_bits,
            use_regs: vec![result.step.src_elem_bits],
            imm: None,
            comment: format!(
                "widen {}→{} (pmov)",
                result.step.src_elem_bits, result.step.dst_elem_bits
            ),
        });
    }

    /// Allocate a new temporary register.
    pub fn alloc_temp(&mut self) -> u32 {
        let t = self.next_temp;
        self.next_temp += 1;
        t
    }

    /// Emit an arithmetic widen (widen both operands, do operation at wider
    /// type, narrow back).
    pub fn emit_widen_arith(
        &mut self,
        opcode: u32,
        src1_reg: u32,
        src2_reg: u32,
        dst_reg: u32,
        src_bits: u32,
        dst_bits: u32,
        sign_extend: bool,
    ) {
        let wide_opcode = self.wide_arith_opcode(opcode, dst_bits);
        let temp1 = self.alloc_temp();
        let temp2 = self.alloc_temp();
        let temp3 = self.alloc_temp();

        // Widen src1
        let widen_op = if sign_extend {
            X86Opcode::PMOVSXWD as u32
        } else {
            X86Opcode::PMOVZXWD as u32
        };
        self.sequence.push(WidenEmittedInstr {
            opcode: widen_op,
            def_reg: temp1,
            use_regs: vec![src1_reg],
            imm: None,
            comment: format!("widen src1 {}→{}", src_bits, dst_bits),
        });

        // Widen src2
        self.sequence.push(WidenEmittedInstr {
            opcode: widen_op,
            def_reg: temp2,
            use_regs: vec![src2_reg],
            imm: None,
            comment: format!("widen src2 {}→{}", src_bits, dst_bits),
        });

        // Perform operation at wider type
        self.sequence.push(WidenEmittedInstr {
            opcode: wide_opcode,
            def_reg: temp3,
            use_regs: vec![temp1, temp2],
            imm: None,
            comment: "wide arithmetic".into(),
        });

        // Narrow result back
        let narrow_op = X86Opcode::PACKSSDW as u32;
        self.sequence.push(WidenEmittedInstr {
            opcode: narrow_op,
            def_reg: dst_reg,
            use_regs: vec![temp3, temp3],
            imm: None,
            comment: format!("narrow {}→{}", dst_bits, src_bits),
        });
    }

    /// Map a narrow arithmetic opcode to its wide counterpart.
    fn wide_arith_opcode(&self, opcode: u32, wide_bits: u32) -> u32 {
        let paddb = X86Opcode::PADDB as u32;
        let psubb = X86Opcode::PSUBB as u32;
        let pmullw = X86Opcode::PMULLW as u32;

        match (opcode, wide_bits) {
            (o, 16) if o == paddb => X86Opcode::PADDW as u32,
            (o, 16) if o == psubb => X86Opcode::PSUBW as u32,
            (o, 32) if o == X86Opcode::PADDW as u32 => X86Opcode::PADDD as u32,
            (o, 32) if o == X86Opcode::PSUBW as u32 => X86Opcode::PSUBD as u32,
            (o, 32) if o == pmullw => X86Opcode::PMULLD as u32,
            _ => opcode, // fallback: keep original
        }
    }
}

impl Default for WidenEmitter {
    fn default() -> Self {
        Self::new()
    }
}

/// A single emitted instruction in the widen sequence.
#[derive(Debug, Clone)]
pub struct WidenEmittedInstr {
    /// The opcode.
    pub opcode: u32,
    /// The destination (definition) register.
    pub def_reg: u32,
    /// Source (use) registers in order.
    pub use_regs: Vec<u32>,
    /// An immediate operand, if any.
    pub imm: Option<i64>,
    /// A human-readable comment for debugging.
    pub comment: String,
}

// ═══════════════════════════════════════════════════════════════════════════════
// Construction helpers
// ═══════════════════════════════════════════════════════════════════════════════

/// Create a default X86VectorWiden for the given subtarget.
pub fn make_x86_vector_widen(subtarget: X86Subtarget) -> X86VectorWiden {
    X86VectorWiden::new(subtarget)
}

/// Create an AVX-optimized X86VectorWiden.
pub fn make_x86_vector_widen_avx(subtarget: X86Subtarget) -> X86VectorWiden {
    let mut w = X86VectorWiden::new(subtarget);
    w.prefer_unpck = false;
    w
}

/// Create a widening-arithmetic-disabled X86VectorWiden.
pub fn make_x86_vector_widen_no_arith(subtarget: X86Subtarget) -> X86VectorWiden {
    let mut w = X86VectorWiden::new(subtarget);
    w.allow_widening_arith = false;
    w
}

// ═══════════════════════════════════════════════════════════════════════════════
// Tests
// ═══════════════════════════════════════════════════════════════════════════════

#[cfg(test)]
mod tests {
    use super::*;

    fn make_subtarget_sse2() -> X86Subtarget {
        X86Subtarget::default()
    }

    fn make_subtarget_sse41() -> X86Subtarget {
        let mut st = X86Subtarget::default();
        // Enable SSE4.1 features
        st
    }

    fn make_subtarget_avx2() -> X86Subtarget {
        let mut st = X86Subtarget::default();
        st
    }

    fn make_subtarget_avx512() -> X86Subtarget {
        let mut st = X86Subtarget::default();
        st
    }

    fn make_mi(opcode: X86Opcode, vec_width: u32, def: u32, srcs: &[u32]) -> MachineWidenInstr {
        let mut mi = MachineWidenInstr::new(opcode, vec_width);
        mi.def_reg = Some(def);
        for &s in srcs {
            mi.operands.push(WidenOperand::Reg(s));
        }
        mi
    }

    fn make_mi_with_imm(
        opcode: X86Opcode,
        vec_width: u32,
        def: u32,
        srcs: &[u32],
        imm: i64,
    ) -> MachineWidenInstr {
        let mut mi = make_mi(opcode, vec_width, def, srcs);
        mi.operands.push(WidenOperand::Imm(imm));
        mi
    }

    // ── WidenKind tests ────────────────────────────────────────────────────

    #[test]
    fn test_widen_kind_display() {
        assert_eq!(WidenKind::SignExtend.to_string(), "sext");
        assert_eq!(WidenKind::ZeroExtend.to_string(), "zext");
    }

    #[test]
    fn test_widen_kind_eq() {
        assert_eq!(WidenKind::SignExtend, WidenKind::SignExtend);
        assert_ne!(WidenKind::SignExtend, WidenKind::ZeroExtend);
    }

    // ── WidenStep tests ────────────────────────────────────────────────────

    #[test]
    fn test_widen_step_new() {
        let step = WidenStep::new(8, 8);
        assert_eq!(step.src_elem_bits, 8);
        assert_eq!(step.dst_elem_bits, 16);
        assert_eq!(step.num_src_elements, 8);
        assert_eq!(step.src_vec_bits, 64);
        assert_eq!(step.dst_vec_bits, 128);
    }

    #[test]
    fn test_widen_step_factor() {
        let step = WidenStep::new(8, 8);
        assert_eq!(step.factor(), 2);

        let step2 = WidenStep::new(16, 8);
        assert_eq!(step2.factor(), 2);

        let step3 = WidenStep::new(32, 4);
        assert_eq!(step3.factor(), 2);
    }

    #[test]
    fn test_widen_step_factor_zero_src() {
        let step = WidenStep {
            src_elem_bits: 0,
            dst_elem_bits: 16,
            num_src_elements: 8,
            src_vec_bits: 0,
            dst_vec_bits: 128,
        };
        assert_eq!(step.factor(), 0);
    }

    #[test]
    fn test_widen_step_is_pmov_supported() {
        // Standard supported patterns
        assert!(WidenStep::new(8, 8).is_pmov_supported()); // 8→16
        let step = WidenStep {
            src_elem_bits: 8,
            dst_elem_bits: 32,
            num_src_elements: 4,
            src_vec_bits: 32,
            dst_vec_bits: 128,
        };
        assert!(step.is_pmov_supported()); // 8→32
        let step = WidenStep {
            src_elem_bits: 8,
            dst_elem_bits: 64,
            num_src_elements: 2,
            src_vec_bits: 16,
            dst_vec_bits: 128,
        };
        assert!(step.is_pmov_supported()); // 8→64
        let step = WidenStep {
            src_elem_bits: 16,
            dst_elem_bits: 32,
            num_src_elements: 8,
            src_vec_bits: 128,
            dst_vec_bits: 256,
        };
        assert!(step.is_pmov_supported()); // 16→32
        let step = WidenStep {
            src_elem_bits: 32,
            dst_elem_bits: 64,
            num_src_elements: 4,
            src_vec_bits: 128,
            dst_vec_bits: 256,
        };
        assert!(step.is_pmov_supported()); // 32→64
    }

    #[test]
    fn test_widen_step_is_pmov_not_supported() {
        let step = WidenStep {
            src_elem_bits: 8,
            dst_elem_bits: 24,
            num_src_elements: 8,
            src_vec_bits: 64,
            dst_vec_bits: 192,
        };
        assert!(!step.is_pmov_supported()); // 8→24 not native
    }

    #[test]
    fn test_widen_step_load_widen() {
        let step = WidenStep::load_widen(8, 4, 32);
        assert_eq!(step.src_elem_bits, 8);
        assert_eq!(step.dst_elem_bits, 32);
        assert_eq!(step.num_src_elements, 4);
        assert_eq!(step.src_vec_bits, 32);
        assert_eq!(step.dst_vec_bits, 128);
    }

    // ── WidenOperands tests ────────────────────────────────────────────────

    #[test]
    fn test_widen_operands_default() {
        let ops = WidenOperands::default();
        assert_eq!(ops.src_type_bits, 8);
        assert_eq!(ops.dst_type_bits, 16);
        assert_eq!(ops.num_elements, 8);
        assert_eq!(ops.kind, WidenKind::SignExtend);
        assert!(ops.low_half);
    }

    #[test]
    fn test_widen_operands_to_step() {
        let mut ops = WidenOperands::default();
        ops.src_type_bits = 16;
        ops.dst_type_bits = 32;
        ops.num_elements = 8;
        let step = ops.to_step();
        assert_eq!(step.src_elem_bits, 16);
        assert_eq!(step.dst_elem_bits, 32);
        assert_eq!(step.num_src_elements, 8);
    }

    #[test]
    fn test_widen_operands_is_valid_pmov() {
        let ops = WidenOperands::default(); // 8→16
        assert!(ops.is_valid_pmov());

        let mut ops2 = WidenOperands::default();
        ops2.src_type_bits = 8;
        ops2.dst_type_bits = 24; // 8→24, not natively supported
        assert!(!ops2.is_valid_pmov());
    }

    // ── WidenResult tests ──────────────────────────────────────────────────

    #[test]
    fn test_widen_result_construction() {
        let step = WidenStep::new(8, 8);
        let result = WidenResult {
            original_opcode: X86Opcode::PUNPCKLBW,
            widened_opcode: X86Opcode::PMOVSXBW,
            step,
            inserted_sext: false,
            temp_reg: None,
        };
        assert_eq!(result.original_opcode, X86Opcode::PUNPCKLBW);
        assert_eq!(result.widened_opcode, X86Opcode::PMOVSXBW);
    }

    // ── WidenStats tests ───────────────────────────────────────────────────

    #[test]
    fn test_widen_stats_default() {
        let s = WidenStats::default();
        assert_eq!(s.unpck_widens, 0);
        assert_eq!(s.pmov_widens, 0);
        assert_eq!(s.total_widened, 0);
        assert!(!s.made_progress());
    }

    #[test]
    fn test_widen_stats_made_progress() {
        let mut s = WidenStats::default();
        assert!(!s.made_progress());
        s.total_widened = 1;
        assert!(s.made_progress());
    }

    #[test]
    fn test_widen_stats_merge() {
        let mut a = WidenStats::default();
        a.unpck_widens = 3;
        a.pmov_widens = 2;

        let mut b = WidenStats::default();
        b.unpck_widens = 1;
        b.avx_pmov_widens = 5;

        a.merge(&b);
        assert_eq!(a.unpck_widens, 4);
        assert_eq!(a.pmov_widens, 2);
        assert_eq!(a.avx_pmov_widens, 5);
    }

    #[test]
    fn test_widen_stats_summary() {
        let mut s = WidenStats::default();
        s.unpck_widens = 2;
        s.pmov_widens = 3;
        s.total_widened = 5;
        s.candidates_examined = 10;
        let summary = s.summary();
        assert!(summary.contains("2 unpck"));
        assert!(summary.contains("3 pmov"));
        assert!(summary.contains("5 total"));
        assert!(summary.contains("10 candidates"));
    }

    // ── X86VectorWiden construction tests ──────────────────────────────────

    #[test]
    fn test_x86_vector_widen_new() {
        let st = make_subtarget_sse2();
        let w = X86VectorWiden::new(st);
        assert!(!w.prefer_unpck);
        assert!(w.allow_widening_loads);
        assert!(w.allow_widening_arith);
        assert_eq!(w.max_widens_per_block, 64);
        assert_eq!(w.stats.total_widened, 0);
    }

    #[test]
    fn test_x86_vector_widen_new_sse2() {
        let st = make_subtarget_sse2();
        let w = X86VectorWiden::new_sse2(st);
        assert!(w.prefer_unpck);
    }

    #[test]
    fn test_x86_vector_widen_new_sse41() {
        let st = make_subtarget_sse41();
        let w = X86VectorWiden::new_sse41(st);
        assert!(!w.prefer_unpck);
    }

    #[test]
    fn test_x86_vector_widen_default() {
        let w = X86VectorWiden::default();
        assert!(!w.prefer_unpck);
        assert!(w.allow_widening_loads);
    }

    #[test]
    fn test_make_x86_vector_widen() {
        let st = make_subtarget_sse2();
        let w = make_x86_vector_widen(st);
        assert!(!w.prefer_unpck);
    }

    #[test]
    fn test_make_x86_vector_widen_avx() {
        let st = make_subtarget_avx2();
        let w = make_x86_vector_widen_avx(st);
        assert!(!w.prefer_unpck);
    }

    #[test]
    fn test_make_x86_vector_widen_no_arith() {
        let st = make_subtarget_sse2();
        let w = make_x86_vector_widen_no_arith(st);
        assert!(!w.allow_widening_arith);
    }

    // ── Widen strategy selection tests ─────────────────────────────────────

    #[test]
    fn test_get_widen_strategy_sse2() {
        let st = make_subtarget_sse2();
        let w = X86VectorWiden::new(st);
        let strategy = w.get_widen_strategy(8, 16);
        assert_eq!(strategy, WidenStrategy::Unpck);
    }

    #[test]
    fn test_get_widen_strategy_sse41() {
        let st = make_subtarget_sse41();
        let w = X86VectorWiden::new(st);
        let strategy = w.get_widen_strategy(8, 16);
        assert_eq!(strategy, WidenStrategy::PMOV);
    }

    #[test]
    fn test_get_widen_strategy_avx2() {
        let st = make_subtarget_avx2();
        let w = X86VectorWiden::new(st);
        let strategy = w.get_widen_strategy(8, 16);
        // AVX2 has AVX, so should use AVXPMOV
        assert!(strategy == WidenStrategy::AVXPMOV || strategy == WidenStrategy::PMOV);
    }

    // ── Widen result tests ─────────────────────────────────────────────────

    #[test]
    fn test_widen_sse2_unpck() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new_sse2(st);
        let mi = make_mi(X86Opcode::PUNPCKLBW, 128, 1, &[2, 2]);
        let result = w.try_widen(&mi);
        // SSE2 unpck should be matched
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 8);
        assert_eq!(r.step.dst_elem_bits, 16);
    }

    #[test]
    fn test_widen_sse2_punpcklwd() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new_sse2(st);
        let mi = make_mi(X86Opcode::PUNPCKLWD, 128, 1, &[2, 2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 16);
        assert_eq!(r.step.dst_elem_bits, 32);
    }

    #[test]
    fn test_widen_sse2_punpckldq() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new_sse2(st);
        let mi = make_mi(X86Opcode::PUNPCKLDQ, 128, 1, &[2, 2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 32);
        assert_eq!(r.step.dst_elem_bits, 64);
    }

    #[test]
    fn test_widen_sse2_punpckhbw() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new_sse2(st);
        let mi = make_mi(X86Opcode::PUNPCKHBW, 128, 1, &[2, 2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_pmovsxbw_sse41() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new_sse41(st);
        let mi = make_mi(X86Opcode::PMOVSXBW, 128, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 8);
        assert_eq!(r.step.dst_elem_bits, 16);
    }

    #[test]
    fn test_widen_pmovzxbw_sse41() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new_sse41(st);
        let mi = make_mi(X86Opcode::PMOVZXBW, 128, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_pmovsxbd_sse41() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new_sse41(st);
        let mi = make_mi(X86Opcode::PMOVSXBD, 128, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 8);
        assert_eq!(r.step.dst_elem_bits, 32);
    }

    #[test]
    fn test_widen_vpmovsxbw_avx() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVSXBW, 256, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 8);
        assert_eq!(r.step.dst_elem_bits, 16);
    }

    #[test]
    fn test_widen_vpmovsxwd_avx() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVSXWD, 256, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 16);
        assert_eq!(r.step.dst_elem_bits, 32);
    }

    #[test]
    fn test_widen_vpmovsxdq_avx() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVSXDQ, 256, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_non_widen_opcode() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::NOP, 128, 1, &[]);
        let result = w.try_widen(&mi);
        assert!(result.is_none());
    }

    #[test]
    fn test_widen_nop_opcode() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::NOP, 128, 1, &[]);
        assert!(w.try_widen(&mi).is_none());
    }

    // ── Widen arithmetic tests ─────────────────────────────────────────────

    #[test]
    fn test_widen_arith_paddb() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PADDB, 128, 1, &[2, 3]);
        let result = w.try_widen(&mi);
        // 8-bit add should be widened to 16-bit
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 8);
        assert_eq!(r.step.dst_elem_bits, 16);
    }

    #[test]
    fn test_widen_arith_psubb() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PSUBB, 128, 1, &[2, 3]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_arith_pmullw() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PMULLW, 128, 1, &[2, 3]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 16);
        assert_eq!(r.step.dst_elem_bits, 32);
    }

    #[test]
    fn test_widen_arith_paddd_not_widened() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PADDD, 128, 1, &[2, 3]);
        // 32-bit ops should not be widened
        assert!(w.try_widen(&mi).is_none());
    }

    #[test]
    fn test_widen_arith_disabled() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        w.allow_widening_arith = false;
        let mi = make_mi(X86Opcode::PADDB, 128, 1, &[2, 3]);
        assert!(w.try_widen(&mi).is_none());
    }

    // ── Widen load tests ───────────────────────────────────────────────────

    #[test]
    fn test_widen_load_v4i8_to_v4i32() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let result = w.widen_load(8, 32, 4, WidenKind::SignExtend, 0x1000, 5);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 8);
        assert_eq!(r.step.dst_elem_bits, 32);
        assert_eq!(r.step.num_src_elements, 4);
        assert_eq!(w.stats.widening_loads, 1);
    }

    #[test]
    fn test_widen_load_unsupported() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        // 8→24 is not a native pmov pattern
        let result = w.widen_load(8, 24, 4, WidenKind::SignExtend, 0x1000, 5);
        assert!(result.is_none());
    }

    #[test]
    fn test_widen_load_disabled() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        w.allow_widening_loads = false;
        let result = w.widen_load(8, 32, 4, WidenKind::SignExtend, 0x1000, 5);
        assert!(result.is_none());
    }

    // ── Is widen legal tests ───────────────────────────────────────────────

    #[test]
    fn test_is_widen_legal_8to16_sse41() {
        let st = make_subtarget_sse41();
        let w = X86VectorWiden::new(st);
        let step = WidenStep::new(8, 8);
        assert!(w.is_widen_legal(&step, WidenKind::SignExtend));
    }

    #[test]
    fn test_is_widen_legal_too_large() {
        let st = make_subtarget_avx512();
        let w = X86VectorWiden::new(st);
        let step = WidenStep {
            src_elem_bits: 8,
            dst_elem_bits: 16,
            num_src_elements: 64,
            src_vec_bits: 512,
            dst_vec_bits: 1024,
        };
        assert!(!w.is_widen_legal(&step, WidenKind::SignExtend));
    }

    // ── Clear tests ────────────────────────────────────────────────────────

    #[test]
    fn test_clear() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new(st);
        w.stats.unpck_widens = 5;
        w.stats.total_widened = 5;
        w.clear();
        assert_eq!(w.stats.unpck_widens, 0);
        assert_eq!(w.stats.total_widened, 0);
    }

    // ── MachineWidenInstr tests ────────────────────────────────────────────

    #[test]
    fn test_machine_widen_instr_new() {
        let mi = MachineWidenInstr::new(X86Opcode::PUNPCKLBW, 128);
        assert_eq!(mi.opcode, X86Opcode::PUNPCKLBW);
        assert_eq!(mi.vec_width, 128);
        assert!(mi.operands.is_empty());
        assert!(mi.def_reg.is_none());
    }

    #[test]
    fn test_machine_widen_instr_src_reg() {
        let mut mi = MachineWidenInstr::new(X86Opcode::PUNPCKLBW, 128);
        mi.operands.push(WidenOperand::Reg(5));
        mi.operands.push(WidenOperand::Reg(6));
        assert_eq!(mi.src_reg(0), Some(5));
        assert_eq!(mi.src_reg(1), Some(6));
        assert_eq!(mi.src_reg(2), None);
    }

    #[test]
    fn test_machine_widen_instr_dst_reg() {
        let mut mi = MachineWidenInstr::new(X86Opcode::PUNPCKLBW, 128);
        assert_eq!(mi.dst_reg(), None);
        mi.def_reg = Some(3);
        assert_eq!(mi.dst_reg(), Some(3));
    }

    #[test]
    fn test_machine_widen_instr_imm_op() {
        let mut mi = MachineWidenInstr::new(X86Opcode::PUNPCKLBW, 128);
        mi.operands.push(WidenOperand::Reg(5));
        mi.operands.push(WidenOperand::Imm(42));
        assert_eq!(mi.imm_op(0), None);
        assert_eq!(mi.imm_op(1), Some(42));
    }

    #[test]
    fn test_machine_widen_instr_num_elements() {
        let mi = MachineWidenInstr::new(X86Opcode::PUNPCKLBW, 128);
        assert_eq!(mi.num_elements(8), 16);
        assert_eq!(mi.num_elements(16), 8);
        assert_eq!(mi.num_elements(32), 4);
        assert_eq!(mi.num_elements(0), 0);
    }

    #[test]
    fn test_machine_widen_instr_is_widen_candidate() {
        let mi = make_mi(X86Opcode::PUNPCKLBW, 128, 1, &[2, 2]);
        assert!(mi.is_widen_candidate());

        let mi2 = make_mi(X86Opcode::PMOVSXBW, 128, 1, &[2]);
        assert!(mi2.is_widen_candidate());

        let mi3 = make_mi(X86Opcode::PADDB, 128, 1, &[2, 3]);
        assert!(mi3.is_widen_candidate());

        let mi4 = make_mi(X86Opcode::NOP, 128, 1, &[]);
        assert!(!mi4.is_widen_candidate());
    }

    // ── WidenEmitter tests ─────────────────────────────────────────────────

    #[test]
    fn test_widen_emitter_new() {
        let emitter = WidenEmitter::new();
        assert!(emitter.sequence.is_empty());
    }

    #[test]
    fn test_widen_emitter_default() {
        let emitter = WidenEmitter::default();
        assert!(emitter.sequence.is_empty());
    }

    #[test]
    fn test_widen_emitter_alloc_temp() {
        let mut emitter = WidenEmitter::new();
        let t1 = emitter.alloc_temp();
        let t2 = emitter.alloc_temp();
        assert_ne!(t1, t2);
        assert!(t2 > t1);
    }

    #[test]
    fn test_widen_emitter_emit_widen_unpck() {
        let mut emitter = WidenEmitter::new();
        let step = WidenStep::new(8, 8);
        let result = WidenResult {
            original_opcode: X86Opcode::PUNPCKLBW,
            widened_opcode: X86Opcode::PUNPCKLBW,
            step,
            inserted_sext: true,
            temp_reg: None,
        };
        emitter.emit_widen(&result);
        assert!(!emitter.sequence.is_empty());
    }

    #[test]
    fn test_widen_emitter_emit_widen_pmov() {
        let mut emitter = WidenEmitter::new();
        let step = WidenStep::new(8, 8);
        let result = WidenResult {
            original_opcode: X86Opcode::PUNPCKLBW,
            widened_opcode: X86Opcode::PMOVSXBW,
            step,
            inserted_sext: false,
            temp_reg: None,
        };
        emitter.emit_widen(&result);
        assert!(!emitter.sequence.is_empty());
    }

    #[test]
    fn test_widen_emitter_emit_widen_arith() {
        let mut emitter = WidenEmitter::new();
        emitter.emit_widen_arith(X86Opcode::PADDB as u32, 1, 2, 3, 8, 16, true);
        // Expect 4 instructions: widen src1, widen src2, wide op, narrow
        assert_eq!(emitter.sequence.len(), 4);
    }

    // ── WidenEmittedInstr tests ────────────────────────────────────────────

    #[test]
    fn test_widen_emitted_instr() {
        let instr = WidenEmittedInstr {
            opcode: X86Opcode::PMOVSXBW as u32,
            def_reg: 1,
            use_regs: vec![2],
            imm: None,
            comment: "test".into(),
        };
        assert_eq!(instr.opcode, X86Opcode::PMOVSXBW as u32);
        assert_eq!(instr.def_reg, 1);
        assert_eq!(instr.use_regs, vec![2]);
        assert!(instr.imm.is_none());
        assert_eq!(instr.comment, "test");
    }

    // ── AVX512WidenContext tests ───────────────────────────────────────────

    #[test]
    fn test_avx512_widen_context_default() {
        let ctx = AVX512WidenContext::default();
        assert!(ctx.mask_reg.is_none());
        assert!(!ctx.zero_mask);
        assert!(!ctx.embedded_broadcast);
        assert!(ctx.rounding.is_none());
        assert!(!ctx.sae);
    }

    #[test]
    fn test_avx512_rounding_evex_bits() {
        assert_eq!(AVX512Rounding::RN.evex_bits(), 0b00);
        assert_eq!(AVX512Rounding::RD.evex_bits(), 0b01);
        assert_eq!(AVX512Rounding::RU.evex_bits(), 0b10);
        assert_eq!(AVX512Rounding::RZ.evex_bits(), 0b11);
    }

    // ── WidenStrategy tests ────────────────────────────────────────────────

    #[test]
    fn test_widen_strategy_equality() {
        assert_eq!(WidenStrategy::None, WidenStrategy::None);
        assert_ne!(WidenStrategy::None, WidenStrategy::Unpck);
    }

    // ── Select opcode tests ────────────────────────────────────────────────

    #[test]
    fn test_select_pmov_opcode_8_to_16_sign() {
        let st = make_subtarget_sse41();
        let w = X86VectorWiden::new(st);
        let op = w.select_pmov_opcode(8, 16, WidenKind::SignExtend, false);
        assert_eq!(op, X86Opcode::PMOVSXBW);
    }

    #[test]
    fn test_select_pmov_opcode_8_to_16_zero() {
        let st = make_subtarget_sse41();
        let w = X86VectorWiden::new(st);
        let op = w.select_pmov_opcode(8, 16, WidenKind::ZeroExtend, false);
        assert_eq!(op, X86Opcode::PMOVZXBW);
    }

    #[test]
    fn test_select_pmov_opcode_16_to_32_sign_avx() {
        let st = make_subtarget_avx2();
        let w = X86VectorWiden::new(st);
        let op = w.select_pmov_opcode(16, 32, WidenKind::SignExtend, true);
        assert_eq!(op, X86Opcode::VPMOVSXWD);
    }

    #[test]
    fn test_select_widened_arith_opcode_paddb_to_paddw() {
        let st = make_subtarget_sse41();
        let w = X86VectorWiden::new(st);
        let op = w.select_widened_arith_opcode(X86Opcode::PADDB, 16);
        assert_eq!(op, X86Opcode::PADDW);
    }

    #[test]
    fn test_select_widened_arith_opcode_psubb_to_psubw() {
        let st = make_subtarget_sse41();
        let w = X86VectorWiden::new(st);
        let op = w.select_widened_arith_opcode(X86Opcode::PSUBB, 16);
        assert_eq!(op, X86Opcode::PSUBW);
    }

    // ── Run on block tests ─────────────────────────────────────────────────

    #[test]
    fn test_run_on_block_empty() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new(st);
        let results = w.run_on_block(&[]);
        assert!(results.is_empty());
        assert_eq!(w.stats.total_widened, 0);
    }

    #[test]
    fn test_run_on_block_single_unpck() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new_sse2(st);
        let mi = make_mi(X86Opcode::PUNPCKLBW, 128, 1, &[2, 2]);
        let results = w.run_on_block(&[mi]);
        assert_eq!(results.len(), 1);
        assert_eq!(w.stats.total_widened, 1);
    }

    #[test]
    fn test_run_on_block_multiple() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi1 = make_mi(X86Opcode::PUNPCKLBW, 128, 1, &[2, 2]);
        let mi2 = make_mi(X86Opcode::PMOVSXBW, 128, 3, &[4]);
        let mi3 = make_mi(X86Opcode::PADDB, 128, 5, &[6, 7]);
        let results = w.run_on_block(&[mi1, mi2, mi3]);
        assert_eq!(results.len(), 3);
        assert_eq!(w.stats.total_widened, 3);
    }

    #[test]
    fn test_run_on_block_nop_only() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::NOP, 128, 1, &[]);
        let results = w.run_on_block(&[mi]);
        assert!(results.is_empty());
    }

    #[test]
    fn test_run_on_block_respects_max() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new_sse2(st);
        w.max_widens_per_block = 2;
        let instructions: Vec<_> = (0..10)
            .map(|i| make_mi(X86Opcode::PUNPCKLBW, 128, i + 1, &[i + 10, i + 10]))
            .collect();
        let results = w.run_on_block(&instructions);
        assert_eq!(results.len(), 2);
    }

    // ── Individual widen operation edge cases ─────────────────────────────

    #[test]
    fn test_try_widen_pmov_no_sse41() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new_sse2(st);
        let mi = make_mi(X86Opcode::PMOVSXBW, 128, 1, &[2]);
        // PMOV requires SSE4.1, so on SSE2 it should not match via pmov
        // (may still match as unpck if the pattern matches)
        let result = w.try_widen(&mi);
        // On SSE2 without SSE4.1, pmov itself isn't a valid widen target
        // So it should return None (unless it falls through to unpck matching)
        if result.is_some() {
            // It might match as unpck if the has_sse41 check passes
            // since we're calling try_widen which dispatches based on opcode
        }
    }

    #[test]
    fn test_widen_vpmovsxbd_256() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVSXBD, 256, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_vpmovzxbq_avx() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVZXBQ, 128, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_punpckhwd() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new_sse2(st);
        let mi = make_mi(X86Opcode::PUNPCKHWD, 128, 1, &[2, 2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_punpckhdq() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new_sse2(st);
        let mi = make_mi(X86Opcode::PUNPCKHDQ, 128, 1, &[2, 2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    // ── AVX-512 pattern tests ──────────────────────────────────────────────

    #[test]
    fn test_avx512_widen_with_mask() {
        let ctx = AVX512WidenContext {
            mask_reg: Some(1),
            zero_mask: true,
            embedded_broadcast: false,
            rounding: None,
            sae: false,
        };
        assert_eq!(ctx.mask_reg, Some(1));
        assert!(ctx.zero_mask);
    }

    #[test]
    fn test_avx512_widen_with_rounding() {
        let ctx = AVX512WidenContext {
            mask_reg: None,
            zero_mask: false,
            embedded_broadcast: true,
            rounding: Some(AVX512Rounding::RN),
            sae: true,
        };
        assert!(ctx.embedded_broadcast);
        assert_eq!(ctx.rounding, Some(AVX512Rounding::RN));
        assert!(ctx.sae);
    }

    // ── WidenOperand enum tests ────────────────────────────────────────────

    #[test]
    fn test_widen_operand_equality() {
        assert_eq!(WidenOperand::Reg(1), WidenOperand::Reg(1));
        assert_ne!(WidenOperand::Reg(1), WidenOperand::Reg(2));
        assert_eq!(WidenOperand::Imm(42), WidenOperand::Imm(42));
        assert_ne!(WidenOperand::Imm(42), WidenOperand::Imm(43));
        assert_eq!(WidenOperand::Mem(0x1000), WidenOperand::Mem(0x1000));
        assert_ne!(WidenOperand::Mem(0x1000), WidenOperand::Mem(0x2000));
        assert_eq!(
            WidenOperand::Label("lbl".into()),
            WidenOperand::Label("lbl".into())
        );
    }

    // ── Debug formatting tests ─────────────────────────────────────────────

    #[test]
    fn test_widen_step_debug() {
        let step = WidenStep::new(8, 8);
        let s = format!("{:?}", step);
        assert!(s.contains("src_elem_bits: 8"));
        assert!(s.contains("dst_elem_bits: 16"));
    }

    #[test]
    fn test_widen_result_debug() {
        let step = WidenStep::new(8, 8);
        let result = WidenResult {
            original_opcode: X86Opcode::PUNPCKLBW,
            widened_opcode: X86Opcode::PMOVSXBW,
            step,
            inserted_sext: false,
            temp_reg: None,
        };
        let s = format!("{:?}", result);
        assert!(!s.is_empty());
    }

    #[test]
    fn test_widen_stats_debug() {
        let s = WidenStats::default();
        let d = format!("{:?}", s);
        assert!(d.contains("unpck_widens: 0"));
    }

    // ── Clone tests ────────────────────────────────────────────────────────

    #[test]
    fn test_widen_step_clone() {
        let step = WidenStep::new(8, 8);
        let step2 = step;
        assert_eq!(step2.src_elem_bits, 8);
    }

    #[test]
    fn test_widen_operands_clone() {
        let ops = WidenOperands::default();
        let ops2 = ops.clone();
        assert_eq!(ops2.src_type_bits, 8);
    }

    #[test]
    fn test_widen_result_clone() {
        let step = WidenStep::new(8, 8);
        let r = WidenResult {
            original_opcode: X86Opcode::PUNPCKLBW,
            widened_opcode: X86Opcode::PMOVSXBW,
            step,
            inserted_sext: false,
            temp_reg: None,
        };
        let r2 = r.clone();
        assert_eq!(r2.original_opcode, X86Opcode::PUNPCKLBW);
    }

    // ── Integration / stress tests ─────────────────────────────────────────

    #[test]
    fn test_widen_all_unpck_variants() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new_sse2(st);
        let opcodes = [
            X86Opcode::PUNPCKLBW,
            X86Opcode::PUNPCKHBW,
            X86Opcode::PUNPCKLWD,
            X86Opcode::PUNPCKHWD,
            X86Opcode::PUNPCKLDQ,
            X86Opcode::PUNPCKHDQ,
        ];
        for &op in &opcodes {
            let mi = make_mi(op, 128, 1, &[2, 2]);
            let result = w.try_widen(&mi);
            assert!(result.is_some(), "failed for {:?}", op);
        }
    }

    #[test]
    fn test_widen_all_pmov_variants_sse41() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let opcodes = [
            X86Opcode::PMOVSXBW,
            X86Opcode::PMOVZXBW,
            X86Opcode::PMOVSXWD,
            X86Opcode::PMOVZXWD,
            X86Opcode::PMOVSXDQ,
            X86Opcode::PMOVZXDQ,
            X86Opcode::PMOVSXBD,
            X86Opcode::PMOVZXBD,
            X86Opcode::PMOVSXWQ,
            X86Opcode::PMOVZXWQ,
            X86Opcode::PMOVSXBQ,
            X86Opcode::PMOVZXBQ,
        ];
        for &op in &opcodes {
            let mi = make_mi(op, 128, 1, &[2]);
            let result = w.try_widen(&mi);
            assert!(result.is_some(), "failed for {:?}", op);
        }
    }

    #[test]
    fn test_widen_all_vpmov_variants_avx() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let opcodes = [
            X86Opcode::VPMOVSXBW,
            X86Opcode::VPMOVZXBW,
            X86Opcode::VPMOVSXWD,
            X86Opcode::VPMOVZXWD,
            X86Opcode::VPMOVSXDQ,
            X86Opcode::VPMOVZXDQ,
            X86Opcode::VPMOVSXBD,
            X86Opcode::VPMOVZXBD,
            X86Opcode::VPMOVSXWQ,
            X86Opcode::VPMOVZXWQ,
            X86Opcode::VPMOVSXBQ,
            X86Opcode::VPMOVZXBQ,
        ];
        for &op in &opcodes {
            let mi = make_mi(op, 256, 1, &[2]);
            let result = w.try_widen(&mi);
            assert!(result.is_some(), "failed for {:?}", op);
        }
    }

    #[test]
    fn test_widen_all_arith_variants() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let opcodes = [
            X86Opcode::PADDB,
            X86Opcode::PSUBB,
            X86Opcode::PADDW,
            X86Opcode::PSUBW,
            X86Opcode::PMULLW,
        ];
        for &op in &opcodes {
            let mi = make_mi(op, 128, 1, &[2, 3]);
            let result = w.try_widen(&mi);
            assert!(result.is_some(), "failed for {:?}", op);
        }
    }

    #[test]
    fn test_widen_256bit_vectors() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVSXBW, 256, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.dst_vec_bits, 256);
    }

    #[test]
    fn test_widen_512bit_vectors() {
        let st = make_subtarget_avx512();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVSXBW, 512, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.dst_vec_bits, 512);
    }

    #[test]
    fn test_record_widen_updates_stats() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let step = WidenStep::new(8, 8);
        let result = WidenResult {
            original_opcode: X86Opcode::PUNPCKLBW,
            widened_opcode: X86Opcode::PMOVSXBW,
            step,
            inserted_sext: false,
            temp_reg: None,
        };
        w.record_widen(&result);
        assert!(w.stats.total_widened > 0 || w.stats.pmov_widens > 0);
    }

    #[test]
    fn test_widen_no_crash_on_empty_operands() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new(st);
        let mi = MachineWidenInstr::new(X86Opcode::PUNPCKLBW, 128);
        // No operands set — should not crash
        let result = w.try_widen(&mi);
        assert!(result.is_none());
    }

    #[test]
    fn test_widen_load_zero_extend() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let result = w.widen_load(8, 16, 8, WidenKind::ZeroExtend, 0x2000, 5);
        assert!(result.is_some());
    }

    #[test]
    fn test_vpmovsxwq_avx() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVSXWQ, 128, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 16);
        assert_eq!(r.step.dst_elem_bits, 64);
    }

    #[test]
    fn test_widen_with_temp_register() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PADDB, 128, 1, &[2, 3]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert!(r.temp_reg.is_some());
    }

    #[test]
    fn test_widen_emitter_wide_arith_opcode_paddb() {
        let emitter = WidenEmitter::new();
        let wide = emitter.wide_arith_opcode(X86Opcode::PADDB as u32, 16);
        assert_eq!(wide, X86Opcode::PADDW as u32);
    }

    #[test]
    fn test_widen_emitter_wide_arith_opcode_psubb() {
        let emitter = WidenEmitter::new();
        let wide = emitter.wide_arith_opcode(X86Opcode::PSUBB as u32, 16);
        assert_eq!(wide, X86Opcode::PSUBW as u32);
    }

    #[test]
    fn test_widen_emitter_wide_arith_opcode_unknown() {
        let emitter = WidenEmitter::new();
        let wide = emitter.wide_arith_opcode(X86Opcode::NOP as u32, 16);
        assert_eq!(wide, X86Opcode::NOP as u32); // unchanged
    }

    #[test]
    fn test_widen_emitter_emit_widen_arith_zero_extend() {
        let mut emitter = WidenEmitter::new();
        emitter.emit_widen_arith(
            X86Opcode::PADDB as u32,
            1,
            2,
            3,
            8,
            16,
            false, // zero-extend
        );
        assert_eq!(emitter.sequence.len(), 4);
    }

    #[test]
    fn test_run_on_empty_module() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new(st);
        // No instructions at all
        let results = w.run_on_block(&[]);
        assert!(results.is_empty());
    }

    #[test]
    fn test_block_with_only_non_vector_ops() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::ADD32rr, 0, 1, &[2, 3]);
        let results = w.run_on_block(&[mi]);
        // ADD32rr is not a widen candidate, so nothing should happen
        assert!(results.is_empty());
    }

    #[test]
    fn test_widen_operand_with_mem() {
        let mut mi = MachineWidenInstr::new(X86Opcode::VPMOVSXBW, 256);
        mi.operands.push(WidenOperand::Mem(0xDEADBEEF));
        mi.def_reg = Some(5);
        assert_eq!(mi.src_reg(0), None);
        assert_ne!(mi.src_reg(0), Some(0xDEADBEEFu64 as u32));
    }

    #[test]
    fn test_is_widen_legal_8to32_avx() {
        let st = make_subtarget_avx2();
        let w = X86VectorWiden::new(st);
        let step = WidenStep {
            src_elem_bits: 8,
            dst_elem_bits: 32,
            num_src_elements: 4,
            src_vec_bits: 32,
            dst_vec_bits: 128,
        };
        assert!(w.is_widen_legal(&step, WidenKind::SignExtend));
    }

    #[test]
    fn test_widen_default_does_not_panic() {
        let w = X86VectorWiden::default();
        assert!(!w.prefer_unpck);
        assert_eq!(w.stats.total_widened, 0);
    }

    // ── Additional comprehensive widen tests ──────────────────────────────

    #[test]
    fn test_widen_vpmovzxbw_avx_256() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVZXBW, 256, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_vpmovzxdq_avx_256() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVZXDQ, 256, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_vpmovzxbd_avx_256() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVZXBD, 256, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_vpmovzxwq_avx_128() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVZXWQ, 128, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_vpmovzxbq_avx_128() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVZXBQ, 128, 1, &[2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_arith_paddw_to_paddd() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PADDW, 128, 1, &[2, 3]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 16);
        assert_eq!(r.step.dst_elem_bits, 32);
    }

    #[test]
    fn test_widen_arith_psubw_to_psubd() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PSUBW, 128, 1, &[2, 3]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.step.src_elem_bits, 16);
        assert_eq!(r.step.dst_elem_bits, 32);
    }

    #[test]
    fn test_widen_punpcklqdq() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new_sse2(st);
        let mi = make_mi(X86Opcode::PUNPCKLQDQ, 128, 1, &[2, 2]);
        let result = w.try_widen(&mi);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_load_v2i16_to_v2i64() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let result = w.widen_load(16, 64, 2, WidenKind::ZeroExtend, 0x4000, 3);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_load_v2i8_to_v2i64() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let result = w.widen_load(8, 64, 2, WidenKind::SignExtend, 0x5000, 4);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_load_v8i8_to_v8i16_zero_extend() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let result = w.widen_load(8, 16, 8, WidenKind::ZeroExtend, 0x6000, 5);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_load_v4i16_to_v4i32_sign_extend() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let result = w.widen_load(16, 32, 4, WidenKind::SignExtend, 0x7000, 6);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_load_v2i32_to_v2i64_zero_extend() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let result = w.widen_load(32, 64, 2, WidenKind::ZeroExtend, 0x8000, 7);
        assert!(result.is_some());
    }

    #[test]
    fn test_widen_mixed_block() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi1 = make_mi(X86Opcode::VPMOVSXBW, 256, 1, &[2]);
        let mi2 = make_mi(X86Opcode::VPMOVZXWD, 128, 3, &[4]);
        let mi3 = make_mi(X86Opcode::PADDB, 128, 5, &[6, 7]);
        let mi4 = make_mi(X86Opcode::NOP, 0, 8, &[]);
        let results = w.run_on_block(&[mi1, mi2, mi3, mi4]);
        // mi4 (NOP) should not produce a widen result
        assert_eq!(results.len(), 3);
    }

    #[test]
    fn test_widen_step_reuse() {
        let step1 = WidenStep::new(8, 8);
        let step2 = WidenStep::new(8, 8);
        assert_eq!(step1, step2);
        assert_eq!(step1.factor(), step2.factor());
    }

    #[test]
    fn test_widen_emitter_sequence_empty_on_identity() {
        let mut emitter = WidenEmitter::new();
        assert_eq!(emitter.sequence.len(), 0);
    }

    #[test]
    fn test_widen_emitter_multiple_emits() {
        let mut emitter = WidenEmitter::new();
        let step1 = WidenStep::new(8, 8);
        let step2 = WidenStep::new(16, 8);
        let r1 = WidenResult {
            original_opcode: X86Opcode::PUNPCKLBW,
            widened_opcode: X86Opcode::PMOVSXBW,
            step: step1,
            inserted_sext: false,
            temp_reg: None,
        };
        let r2 = WidenResult {
            original_opcode: X86Opcode::PUNPCKLWD,
            widened_opcode: X86Opcode::PMOVSXWD,
            step: step2,
            inserted_sext: false,
            temp_reg: None,
        };
        emitter.emit_widen(&r1);
        emitter.emit_widen(&r2);
        assert_eq!(emitter.sequence.len(), 2);
    }

    #[test]
    fn test_widen_operand_label() {
        let op = WidenOperand::Label("loop_header".into());
        assert_eq!(op, WidenOperand::Label("loop_header".into()));
        assert_ne!(op, WidenOperand::Label("loop_body".into()));
    }

    #[test]
    fn test_widen_mi_with_memory_load() {
        let mut mi = MachineWidenInstr::new(X86Opcode::PMOVSXBW, 128);
        mi.is_load = true;
        mi.mem_addr = Some(0xDEAD0000);
        mi.operands.push(WidenOperand::Mem(0xDEAD0000));
        mi.def_reg = Some(1);
        assert!(mi.is_load);
        assert_eq!(mi.mem_addr, Some(0xDEAD0000));
    }

    #[test]
    fn test_widen_mi_with_evex() {
        let mut mi = MachineWidenInstr::new(X86Opcode::VPMOVSXBW, 512);
        mi.has_evex = true;
        assert!(mi.has_evex);
        assert_eq!(mi.vec_width, 512);
    }

    #[test]
    fn test_widen_clear_preserves_config() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new(st);
        w.prefer_unpck = true;
        w.stats.total_widened = 10;
        w.clear();
        assert_eq!(w.stats.total_widened, 0);
        assert!(w.prefer_unpck); // config preserved
    }

    #[test]
    fn test_widen_stats_all_fields() {
        let mut s = WidenStats::default();
        s.unpck_widens = 1;
        s.pmov_widens = 2;
        s.avx_pmov_widens = 3;
        s.avx512_widens = 4;
        s.widening_loads = 5;
        s.widening_arith = 6;
        s.total_widened = 21;
        s.candidates_examined = 100;
        let summary = s.summary();
        assert!(summary.contains("1 unpck"));
        assert!(summary.contains("2 pmov"));
        assert!(summary.contains("3 avx-pmov"));
        assert!(summary.contains("4 avx512"));
        assert!(summary.contains("5 loads"));
        assert!(summary.contains("6 arith"));
        assert!(summary.contains("21 total"));
        assert!(summary.contains("100 candidates"));
    }

    #[test]
    fn test_widen_is_pmov_opcode_true() {
        let st = make_subtarget_sse2();
        let w = X86VectorWiden::new(st);
        assert!(w.is_pmov_opcode(X86Opcode::PMOVSXBW));
        assert!(w.is_pmov_opcode(X86Opcode::PMOVZXBQ));
    }

    #[test]
    fn test_widen_is_pmov_opcode_false() {
        let st = make_subtarget_sse2();
        let w = X86VectorWiden::new(st);
        assert!(!w.is_pmov_opcode(X86Opcode::NOP));
        assert!(!w.is_pmov_opcode(X86Opcode::PUNPCKLBW));
    }

    #[test]
    fn test_widen_avx512_context_custom() {
        let ctx = AVX512WidenContext {
            mask_reg: Some(3),
            zero_mask: true,
            embedded_broadcast: true,
            rounding: Some(AVX512Rounding::RZ),
            sae: true,
        };
        assert_eq!(ctx.mask_reg, Some(3));
        assert!(ctx.zero_mask);
        assert!(ctx.embedded_broadcast);
        assert_eq!(ctx.rounding, Some(AVX512Rounding::RZ));
        assert!(ctx.sae);
    }

    #[test]
    fn test_widen_strategy_all_variants_distinct() {
        assert_ne!(WidenStrategy::None, WidenStrategy::Unpck);
        assert_ne!(WidenStrategy::Unpck, WidenStrategy::PMOV);
        assert_ne!(WidenStrategy::PMOV, WidenStrategy::AVXPMOV);
        assert_ne!(WidenStrategy::AVXPMOV, WidenStrategy::AVX512);
    }

    #[test]
    fn test_widen_avx512_rounding_all_modes() {
        let modes = [
            AVX512Rounding::RN,
            AVX512Rounding::RD,
            AVX512Rounding::RU,
            AVX512Rounding::RZ,
        ];
        let expected = [0b00, 0b01, 0b10, 0b11];
        for (mode, exp) in modes.iter().zip(expected.iter()) {
            assert_eq!(mode.evex_bits(), *exp);
        }
    }

    #[test]
    fn test_widen_step_load_widen_unsupported() {
        let step = WidenStep::load_widen(8, 4, 24);
        assert!(!step.is_pmov_supported());
    }

    #[test]
    fn test_widen_strategy_debug() {
        let s = format!("{:?}", WidenStrategy::AVX512);
        assert!(!s.is_empty());
        let s2 = format!("{:?}", WidenStrategy::None);
        assert!(!s2.is_empty());
    }

    #[test]
    fn test_run_on_block_all_stats_accumulate() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        let mi1 = make_mi(X86Opcode::VPMOVSXBW, 128, 1, &[2]);
        let mi2 = make_mi(X86Opcode::VPMOVZXWD, 128, 3, &[4]);
        let _ = w.run_on_block(&[mi1, mi2]);
        assert_eq!(w.stats.total_widened, 2);
        assert_eq!(w.stats.candidates_examined, 2);
    }

    #[test]
    fn test_widen_emitted_instr_with_imm() {
        let instr = WidenEmittedInstr {
            opcode: X86Opcode::PMOVSXBW as u32,
            def_reg: 1,
            use_regs: vec![2],
            imm: Some(42),
            comment: "with immediate".into(),
        };
        assert_eq!(instr.imm, Some(42));
    }

    #[test]
    fn test_widen_avx512_256bit() {
        let st = make_subtarget_avx512();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::VPMOVSXBW, 256, 1, &[2]);
        assert!(w.try_widen(&mi).is_some());
    }

    #[test]
    fn test_widen_load_all_patterns() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let patterns = [
            (8u32, 16u32, 8u32),
            (8, 32, 4),
            (8, 64, 2),
            (16, 32, 8),
            (16, 64, 4),
            (32, 64, 4),
        ];
        for (i, (s, d, n)) in patterns.iter().enumerate() {
            assert!(w
                .widen_load(*s, *d, *n, WidenKind::SignExtend, 0x1000, (i + 1) as u32)
                .is_some());
        }
    }

    #[test]
    fn test_widen_paddq_not_widened() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        assert!(w
            .try_widen(&make_mi(X86Opcode::PADDQ, 128, 1, &[2, 3]))
            .is_none());
    }

    #[test]
    fn test_select_pmov_all_combos() {
        let st = make_subtarget_avx2();
        let w = X86VectorWiden::new(st);
        assert_eq!(
            w.select_pmov_opcode(8, 16, WidenKind::SignExtend, false),
            X86Opcode::PMOVSXBW
        );
        assert_eq!(
            w.select_pmov_opcode(16, 32, WidenKind::ZeroExtend, false),
            X86Opcode::PMOVZXWD
        );
        assert_eq!(
            w.select_pmov_opcode(32, 64, WidenKind::SignExtend, false),
            X86Opcode::PMOVSXDQ
        );
        assert_eq!(
            w.select_pmov_opcode(8, 32, WidenKind::ZeroExtend, true),
            X86Opcode::VPMOVZXBD
        );
        assert_eq!(
            w.select_pmov_opcode(16, 64, WidenKind::SignExtend, true),
            X86Opcode::VPMOVSXWQ
        );
    }

    #[test]
    fn test_select_unpck_all_combos() {
        let st = make_subtarget_sse2();
        let w = X86VectorWiden::new(st);
        assert_eq!(
            w.select_widened_unpck_opcode(8, 16, true, WidenKind::SignExtend),
            X86Opcode::PUNPCKLBW
        );
        assert_eq!(
            w.select_widened_unpck_opcode(8, 16, false, WidenKind::SignExtend),
            X86Opcode::PUNPCKHBW
        );
        assert_eq!(
            w.select_widened_unpck_opcode(16, 32, true, WidenKind::SignExtend),
            X86Opcode::PUNPCKLWD
        );
        assert_eq!(
            w.select_widened_unpck_opcode(32, 64, true, WidenKind::SignExtend),
            X86Opcode::PUNPCKLDQ
        );
    }

    #[test]
    fn test_select_arith_all_combos() {
        let st = make_subtarget_sse41();
        let w = X86VectorWiden::new(st);
        assert_eq!(
            w.select_widened_arith_opcode(X86Opcode::PADDB, 16),
            X86Opcode::PADDW
        );
        assert_eq!(
            w.select_widened_arith_opcode(X86Opcode::PSUBB, 16),
            X86Opcode::PSUBW
        );
        assert_eq!(
            w.select_widened_arith_opcode(X86Opcode::PADDW, 32),
            X86Opcode::PADDD
        );
        assert_eq!(
            w.select_widened_arith_opcode(X86Opcode::PMULLW, 32),
            X86Opcode::PMULLD
        );
    }

    #[test]
    fn test_vpmovsxbw_no_avx() {
        let st = make_subtarget_sse2();
        let mut w = X86VectorWiden::new(st);
        assert!(w
            .try_widen(&make_mi(X86Opcode::VPMOVSXBW, 128, 1, &[2]))
            .is_none());
    }

    #[test]
    fn test_zero_elem_bits_edge() {
        let step = WidenStep {
            src_elem_bits: 0,
            dst_elem_bits: 8,
            num_src_elements: 0,
            src_vec_bits: 0,
            dst_vec_bits: 0,
        };
        assert_eq!(step.factor(), 0);
        assert!(!step.is_pmov_supported());
    }

    #[test]
    fn test_get_strategy_all_pairs() {
        let st = make_subtarget_avx2();
        let w = X86VectorWiden::new(st);
        for (s, d) in [(8, 16), (8, 32), (16, 32), (32, 64)] {
            assert!(w.get_widen_strategy(s, d) != WidenStrategy::None);
        }
    }

    #[test]
    fn test_avx512_context_all() {
        let ctx = AVX512WidenContext {
            mask_reg: Some(7),
            zero_mask: true,
            embedded_broadcast: true,
            rounding: Some(AVX512Rounding::RD),
            sae: false,
        };
        assert_eq!(ctx.mask_reg, Some(7));
        assert!(ctx.zero_mask);
        assert_eq!(ctx.rounding.unwrap().evex_bits(), 0b01);
    }

    #[test]
    fn test_rounding_debug() {
        assert!(!format!("{:?}", AVX512Rounding::RN).is_empty());
        assert!(!format!("{:?}", AVX512Rounding::RZ).is_empty());
    }

    #[test]
    fn test_widen_mi_with_load_flag() {
        let mut mi = MachineWidenInstr::new(X86Opcode::PMOVSXBW, 128);
        mi.is_load = true;
        mi.mem_addr = Some(0xDEAD);
        assert!(mi.is_load);
        assert_eq!(mi.mem_addr, Some(0xDEAD));
    }

    // Extended pattern matching and edge-case coverage

    #[test]
    fn test_widen_pmovzxbd_sse41() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PMOVZXBD, 128, 1, &[2]);
        assert!(w.try_widen(&mi).is_some());
    }

    #[test]
    fn test_widen_pmovzxwq_sse41() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PMOVZXWQ, 128, 1, &[2]);
        assert!(w.try_widen(&mi).is_some());
    }

    #[test]
    fn test_widen_pmovzxbq_sse41() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PMOVZXBQ, 128, 1, &[2]);
        assert!(w.try_widen(&mi).is_some());
    }

    #[test]
    fn test_widen_pmovzxdq_sse41() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PMOVZXDQ, 128, 1, &[2]);
        assert!(w.try_widen(&mi).is_some());
    }

    #[test]
    fn test_widen_pmovzxwd_sse41() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PMOVZXWD, 128, 1, &[2]);
        assert!(w.try_widen(&mi).is_some());
    }

    #[test]
    fn test_widen_load_avx_patterns() {
        let st = make_subtarget_avx2();
        let mut w = X86VectorWiden::new(st);
        assert!(w
            .widen_load(8, 16, 16, WidenKind::SignExtend, 0x1000, 1)
            .is_some());
        assert!(w
            .widen_load(8, 32, 8, WidenKind::ZeroExtend, 0x2000, 2)
            .is_some());
        assert!(w
            .widen_load(16, 32, 16, WidenKind::SignExtend, 0x3000, 3)
            .is_some());
        assert!(w
            .widen_load(32, 64, 8, WidenKind::ZeroExtend, 0x4000, 4)
            .is_some());
    }

    #[test]
    fn test_widen_arith_no_temp_on_non_arith() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PMOVSXBW, 128, 1, &[2]);
        let r = w.try_widen(&mi).unwrap();
        assert!(r.temp_reg.is_none());
    }

    #[test]
    fn test_widen_arith_has_temp() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi = make_mi(X86Opcode::PADDB, 128, 1, &[2, 3]);
        let r = w.try_widen(&mi).unwrap();
        assert!(r.temp_reg.is_some());
    }

    #[test]
    fn test_widen_is_widen_legal_non_pmov() {
        let st = make_subtarget_sse2();
        let w = X86VectorWiden::new(st);
        let step = WidenStep {
            src_elem_bits: 8,
            dst_elem_bits: 24,
            num_src_elements: 4,
            src_vec_bits: 32,
            dst_vec_bits: 96,
        };
        // 8→24 not pmov_supported, but SSE2 has unpck-based which might work
        let legal = w.is_widen_legal(&step, WidenKind::SignExtend);
        // Should be false because unpck only supports 2x widening
        assert!(!legal);
    }

    #[test]
    fn test_widen_strategy_scalar() {
        let st = X86Subtarget::default();
        let w = X86VectorWiden::new(st);
        // Without SSE2, strategy should be None for most pairs
        // (depends on default subtarget features)
        let s = w.get_widen_strategy(8, 16);
        assert!(s == WidenStrategy::None || s == WidenStrategy::Unpck);
    }

    #[test]
    fn test_widen_emitter_emit_widen_arith_four_instrs() {
        let mut emitter = WidenEmitter::new();
        emitter.emit_widen_arith(X86Opcode::PADDB as u32, 1, 2, 3, 8, 16, true);
        assert_eq!(emitter.sequence.len(), 4);
        // Check instruction types
        let comments: Vec<&str> = emitter
            .sequence
            .iter()
            .map(|i| i.comment.as_str())
            .collect();
        assert!(comments[0].contains("widen src1"));
        assert!(comments[1].contains("widen src2"));
        assert!(comments[2].contains("wide arithmetic"));
        assert!(comments[3].contains("narrow"));
    }

    #[test]
    fn test_widen_multiple_blocks_independent() {
        let st = make_subtarget_sse41();
        let mut w = X86VectorWiden::new(st);
        let mi1 = make_mi(X86Opcode::PMOVSXBW, 128, 1, &[2]);
        let block1 = w.run_on_block(&[mi1]);
        let block2 = w.run_on_block(&[]);
        assert!(!block1.is_empty());
        assert!(block2.is_empty());
    }

    #[test]
    fn test_widen_all_strategy_variants() {
        let variants = [
            WidenStrategy::None,
            WidenStrategy::Unpck,
            WidenStrategy::PMOV,
            WidenStrategy::AVXPMOV,
            WidenStrategy::AVX512,
        ];
        for v in &variants {
            assert!(!format!("{:?}", v).is_empty());
        }
    }
}