llvm-native-core 0.1.13

//! X86 Instruction Selection Golden Table — complete, feature-gated,
//! priority-ranked instruction selection pattern database covering every
//! IR opcode the X86 backend must lower.
//!
//! ## Architecture
//!
//! ```text
//! IR Instruction
//!      │
//!      ▼
//! ┌─────────────────────────────────────────┐
//! │   ISelPatternMatcher                    │
//! │   ┌───────────────────────────────────┐ │
//! │   │ 1. Extract IR opcode + type       │ │
//! │   │ 2. Query X86ISelGoldenTable       │ │
//! │   │ 3. Filter by FeatureGate          │ │
//! │   │ 4. Sort candidates by priority    │ │
//! │   │ 5. Check constraints & conditions │ │
//! │   │ 6. Return best ISelPatternRule    │ │
//! │   └───────────────────────────────────┘ │
//! └─────────────────────────────────────────┘
//!      │
//!      ▼
//! Selected X86Opcode + operand mapping
//! ```
//!
//! ## Pattern Categories
//!
//! | Priority Range | Category                        |
//! |----------------|---------------------------------|
//! | 10000–9000     | AVX‑512 / AVX‑VNNI / AMX       |
//! | 8999–7000      | AVX2 / FMA / BMI2              |
//! | 6999–5000      | AVX / BMI / ADX                |
//! | 4999–3000      | SSE4.2 / SSE4.1 / SSSE3 / SSE3 |
//! | 2999–2000      | SSE2 / SSE                     |
//! | 1999–1000      | Base x86‑64 / i686             |
//! | 999–0          | Fallback / generic patterns    |
//!
//! Clean-room behavioral reconstruction from:
//! - Intel® 64 and IA-32 Architectures Software Developer's Manual
//! - AMD64 Architecture Programmer's Manual
//! - LLVM Language Reference
//! - X86‑64 ABI specifications
//!
//! Zero LLVM source code consultation.

use std::collections::{BTreeMap, HashMap, HashSet};
use std::fmt;

use crate::opcode::Opcode;
use crate::types::{Type, TypeKind};
use crate::x86::x86_instr_info::X86Opcode;

// ============================================================================
// Domain types
// ============================================================================

/// Width of an integer operand in bits.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[repr(u8)]
pub enum IntWidth {
    I8 = 8,
    I16 = 16,
    I32 = 32,
    I64 = 64,
}

impl IntWidth {
    pub fn bits(&self) -> u8 {
        *self as u8
    }

    pub fn from_bits(bits: u32) -> Option<Self> {
        match bits {
            8 => Some(Self::I8),
            16 => Some(Self::I16),
            32 => Some(Self::I32),
            64 => Some(Self::I64),
            _ => None,
        }
    }

    pub fn as_str(&self) -> &'static str {
        match self {
            Self::I8 => "i8",
            Self::I16 => "i16",
            Self::I32 => "i32",
            Self::I64 => "i64",
        }
    }
}

impl fmt::Display for IntWidth {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}", self.as_str())
    }
}

/// Floating-point width.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[repr(u8)]
pub enum FloatWidth {
    F16 = 0,
    F32 = 1,
    F64 = 2,
}

impl FloatWidth {
    pub fn bits(&self) -> u8 {
        match self {
            Self::F16 => 16,
            Self::F32 => 32,
            Self::F64 => 64,
        }
    }

    pub fn as_str(&self) -> &'static str {
        match self {
            Self::F16 => "f16",
            Self::F32 => "f32",
            Self::F64 => "f64",
        }
    }
}

impl fmt::Display for FloatWidth {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}", self.as_str())
    }
}

/// Vector lane count.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[repr(u8)]
pub enum VectorWidth {
    V2 = 2,
    V4 = 4,
    V8 = 8,
    V16 = 16,
    V32 = 32,
    V64 = 64,
}

impl VectorWidth {
    pub fn lanes(&self) -> u8 {
        *self as u8
    }

    pub fn from_lanes(n: u32) -> Option<Self> {
        match n {
            2 => Some(Self::V2),
            4 => Some(Self::V4),
            8 => Some(Self::V8),
            16 => Some(Self::V16),
            32 => Some(Self::V32),
            64 => Some(Self::V64),
            _ => None,
        }
    }
}

/// Operand type constraint for pattern matching.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum TypeConstraint {
    /// Must be a specific integer width.
    Int(IntWidth),
    /// Must be a specific float width.
    Float(FloatWidth),
    /// Must be integer of any width.
    AnyInt,
    /// Must be float of any width.
    AnyFloat,
    /// Must be a pointer.
    Pointer,
    /// Must be integer or pointer.
    IntOrPtr,
    /// Must be a vector with given element type and lane count.
    Vector {
        elem: Box<TypeConstraint>,
        lanes: VectorWidth,
    },
    /// Any type.
    Any,
    /// Must match the result type.
    SameAsResult,
    /// Exact type name match (for user-specified types).
    Exact(String),
}

impl TypeConstraint {
    pub fn matches(&self, other: &Self) -> bool {
        match (self, other) {
            (Self::Any, _) | (_, Self::Any) => true,
            (Self::AnyInt, Self::Int(_)) => true,
            (Self::AnyFloat, Self::Float(_)) => true,
            (a, b) => a == b,
        }
    }

    pub fn as_str(&self) -> String {
        match self {
            Self::Int(w) => w.as_str().to_string(),
            Self::Float(w) => w.as_str().to_string(),
            Self::AnyInt => "int".to_string(),
            Self::AnyFloat => "float".to_string(),
            Self::Pointer => "ptr".to_string(),
            Self::IntOrPtr => "int|ptr".to_string(),
            Self::Vector { elem, lanes } => format!("<{} x {}>", lanes.lanes(), elem.as_str()),
            Self::Any => "any".to_string(),
            Self::SameAsResult => "=result".to_string(),
            Self::Exact(s) => s.clone(),
        }
    }
}

impl fmt::Display for TypeConstraint {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}", self.as_str())
    }
}

// ============================================================================
// CPU Feature Model
// ============================================================================

/// X86 CPU features used for pattern gating.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[repr(u16)]
pub enum X86Feature {
    // Baseline
    X86,
    X86_64,
    // SSE family
    SSE,
    SSE2,
    SSE3,
    SSSE3,
    SSE41,
    SSE42,
    // AVX family
    AVX,
    AVX2,
    AVX512F,
    AVX512BW,
    AVX512DQ,
    AVX512VL,
    AVX512CD,
    AVX512ER,
    AVX512PF,
    AVX512VBMI,
    AVX512VBMI2,
    AVX512VNNI,
    AVX512BITALG,
    AVX512BF16,
    AVX512FP16,
    AVXVNNI,
    AVXIFMA,
    AVXNECONVERT,
    AVXVNNIINT8,
    AVXVNNIINT16,
    // FMA
    FMA,
    FMA4,
    F16C,
    // BMI / ADX
    BMI,
    BMI2,
    ADX,
    // Bit manipulation
    ABM,
    POPCNT,
    LZCNT,
    // Crypto / Hash
    AES,
    PCLMUL,
    SHA,
    SM3,
    SM4,
    VAES,
    VPCLMULQDQ,
    // Special
    CMOV,
    CX8,
    CX16,
    MOVBE,
    RDRAND,
    RDSEED,
    FSGSBASE,
    RDTSCP,
    SGX,
    CET,
    MPX,
    RTM,
    HLE,
    TSXLDTRK,
    // Memory / Cache
    PREFETCHW,
    PREFETCHWT1,
    CLFLUSHOPT,
    CLWB,
    CLZERO,
    // String
    SSE4A,
    XOP,
    // AMX
    AMXTILE,
    AMXINT8,
    AMXBF16,
    AMXFP16,
    // Others
    GFNI,
    VPCLMUL,
    MOVDIRI,
    MOVDIR64B,
    ENQCMD,
    WAITPKG,
    SERIALIZE,
    PCONFIG,
    PTWRITE,
    INVPCID,
    // VNNI / DP
    AVX512VPOPCNTDQ,
    AVX5124FMAPS,
    AVX5124VNNIW,
}

impl X86Feature {
    pub fn name(&self) -> &'static str {
        match self {
            Self::X86 => "x86",
            Self::X86_64 => "x86_64",
            Self::SSE => "sse",
            Self::SSE2 => "sse2",
            Self::SSE3 => "sse3",
            Self::SSSE3 => "ssse3",
            Self::SSE41 => "sse4.1",
            Self::SSE42 => "sse4.2",
            Self::AVX => "avx",
            Self::AVX2 => "avx2",
            Self::AVX512F => "avx512f",
            Self::AVX512BW => "avx512bw",
            Self::AVX512DQ => "avx512dq",
            Self::AVX512VL => "avx512vl",
            Self::AVX512CD => "avx512cd",
            Self::AVX512ER => "avx512er",
            Self::AVX512PF => "avx512pf",
            Self::AVX512VBMI => "avx512vbmi",
            Self::AVX512VBMI2 => "avx512vbmi2",
            Self::AVX512VNNI => "avx512vnni",
            Self::AVX512BITALG => "avx512bitalg",
            Self::AVX512BF16 => "avx512bf16",
            Self::AVX512FP16 => "avx512fp16",
            Self::AVXVNNI => "avxvnni",
            Self::AVXIFMA => "avxifma",
            Self::AVXNECONVERT => "avxneconvert",
            Self::AVXVNNIINT8 => "avxvnniint8",
            Self::AVXVNNIINT16 => "avxvnniint16",
            Self::FMA => "fma",
            Self::FMA4 => "fma4",
            Self::F16C => "f16c",
            Self::BMI => "bmi",
            Self::BMI2 => "bmi2",
            Self::ADX => "adx",
            Self::ABM => "abm",
            Self::POPCNT => "popcnt",
            Self::LZCNT => "lzcnt",
            Self::AES => "aes",
            Self::PCLMUL => "pclmul",
            Self::SHA => "sha",
            Self::SM3 => "sm3",
            Self::SM4 => "sm4",
            Self::VAES => "vaes",
            Self::VPCLMULQDQ => "vpclmulqdq",
            Self::CMOV => "cmov",
            Self::CX8 => "cx8",
            Self::CX16 => "cx16",
            Self::MOVBE => "movbe",
            Self::RDRAND => "rdrand",
            Self::RDSEED => "rdseed",
            Self::FSGSBASE => "fsgsbase",
            Self::RDTSCP => "rdtscp",
            Self::SGX => "sgx",
            Self::CET => "cet",
            Self::MPX => "mpx",
            Self::RTM => "rtm",
            Self::HLE => "hle",
            Self::TSXLDTRK => "tsxldtrk",
            Self::PREFETCHW => "prefetchw",
            Self::PREFETCHWT1 => "prefetchwt1",
            Self::CLFLUSHOPT => "clflushopt",
            Self::CLWB => "clwb",
            Self::CLZERO => "clzero",
            Self::SSE4A => "sse4a",
            Self::XOP => "xop",
            Self::AMXTILE => "amxtile",
            Self::AMXINT8 => "amxint8",
            Self::AMXBF16 => "amxbf16",
            Self::AMXFP16 => "amxfp16",
            Self::GFNI => "gfni",
            Self::VPCLMUL => "vpclmul",
            Self::MOVDIRI => "movdiri",
            Self::MOVDIR64B => "movdir64b",
            Self::ENQCMD => "enqcmd",
            Self::WAITPKG => "waitpkg",
            Self::SERIALIZE => "serialize",
            Self::PCONFIG => "pconfig",
            Self::PTWRITE => "ptwrite",
            Self::INVPCID => "invpcid",
            Self::AVX512VPOPCNTDQ => "avx512vpopcntdq",
            Self::AVX5124FMAPS => "avx5124fmaps",
            Self::AVX5124VNNIW => "avx5124vnniw",
        }
    }
}

// ============================================================================
// Feature Gate
// ============================================================================

/// Feature gate that controls pattern availability based on CPU features.
#[derive(Debug, Clone)]
pub struct FeatureGate {
    /// Set of enabled features.
    enabled: HashSet<X86Feature>,
    /// Whether 64-bit mode is active.
    pub is_64bit: bool,
}

impl FeatureGate {
    /// Create a new feature gate with the given features.
    pub fn new(features: &[X86Feature], is_64bit: bool) -> Self {
        Self {
            enabled: features.iter().copied().collect(),
            is_64bit,
        }
    }

    /// All known x86 features, roughly in dependency order.
    pub fn all_features() -> Vec<X86Feature> {
        use X86Feature::*;
        vec![
            X86,
            X86_64,
            CMOV,
            CX8,
            CX16, // baseline
            SSE,
            SSE2,
            SSE3,
            SSSE3,
            SSE41,
            SSE42,
            SSE4A, // SSE
            AVX,
            AVX2,
            F16C, // AVX
            FMA,
            FMA4, // FMA
            AVX512F,
            AVX512BW,
            AVX512DQ,
            AVX512VL,
            AVX512CD,
            AVX512ER,
            AVX512PF,
            AVX512VBMI,
            AVX512VBMI2,
            AVX512VNNI,
            AVX512BITALG,
            AVX512BF16,
            AVX512FP16,
            AVX512VPOPCNTDQ,
            AVX5124FMAPS,
            AVX5124VNNIW,
            AVXVNNI,
            AVXIFMA,
            AVXNECONVERT,
            AVXVNNIINT8,
            AVXVNNIINT16,
            BMI,
            BMI2,
            ADX,
            ABM,
            POPCNT,
            LZCNT,
            AES,
            PCLMUL,
            SHA,
            SM3,
            SM4,
            VAES,
            VPCLMULQDQ,
            VPCLMUL,
            GFNI,
            RDRAND,
            RDSEED,
            FSGSBASE,
            RDTSCP,
            MOVBE,
            SGX,
            CET,
            MPX,
            RTM,
            HLE,
            TSXLDTRK,
            PREFETCHW,
            PREFETCHWT1,
            CLFLUSHOPT,
            CLWB,
            CLZERO,
            XOP,
            AMXTILE,
            AMXINT8,
            AMXBF16,
            AMXFP16,
            MOVDIRI,
            MOVDIR64B,
            ENQCMD,
            WAITPKG,
            SERIALIZE,
            PCONFIG,
            PTWRITE,
            INVPCID,
        ]
    }

    /// Check if the given feature is enabled.
    pub fn has(&self, feature: X86Feature) -> bool {
        self.enabled.contains(&feature)
    }

    /// Check if any of the given features are enabled.
    pub fn has_any(&self, features: &[X86Feature]) -> bool {
        features.iter().any(|f| self.enabled.contains(f))
    }

    /// Check if all of the given features are enabled.
    pub fn has_all(&self, features: &[X86Feature]) -> bool {
        features.iter().all(|f| self.enabled.contains(f))
    }

    /// Enable a feature.
    pub fn enable(&mut self, feature: X86Feature) {
        self.enabled.insert(feature);
    }

    /// Disable a feature.
    pub fn disable(&mut self, feature: X86Feature) {
        self.enabled.remove(&feature);
    }

    /// Enable features implied by the given feature (e.g., AVX2 implies AVX, SSE2 implies SSE).
    pub fn enable_with_implications(&mut self, feature: X86Feature) {
        self.enabled.insert(feature);
        // Implication chain
        match feature {
            X86Feature::X86_64 => {
                self.enabled.insert(X86Feature::X86);
                self.enabled.insert(X86Feature::CMOV);
                self.enabled.insert(X86Feature::SSE);
                self.enabled.insert(X86Feature::SSE2);
            }
            X86Feature::SSE2 => {
                self.enabled.insert(X86Feature::SSE);
            }
            X86Feature::SSE3 | X86Feature::SSSE3 => {
                self.enabled.insert(X86Feature::SSE2);
                self.enabled.insert(X86Feature::SSE);
            }
            X86Feature::SSE41 | X86Feature::SSE42 => {
                self.enabled.insert(X86Feature::SSSE3);
                self.enabled.insert(X86Feature::SSE3);
                self.enabled.insert(X86Feature::SSE2);
                self.enabled.insert(X86Feature::SSE);
            }
            X86Feature::AVX => {
                self.enabled.insert(X86Feature::SSE42);
                self.enabled.insert(X86Feature::SSE41);
                self.enabled.insert(X86Feature::SSSE3);
                self.enabled.insert(X86Feature::SSE3);
                self.enabled.insert(X86Feature::SSE2);
                self.enabled.insert(X86Feature::SSE);
            }
            X86Feature::AVX2 => {
                self.enable_with_implications(X86Feature::AVX);
                self.enabled.insert(X86Feature::FMA);
                self.enabled.insert(X86Feature::BMI);
                self.enabled.insert(X86Feature::BMI2);
            }
            X86Feature::FMA => {
                self.enabled.insert(X86Feature::AVX);
            }
            X86Feature::BMI2 => {
                self.enabled.insert(X86Feature::BMI);
            }
            X86Feature::AVX512F => {
                self.enable_with_implications(X86Feature::AVX2);
                self.enabled.insert(X86Feature::F16C);
            }
            X86Feature::AVX512BW
            | X86Feature::AVX512DQ
            | X86Feature::AVX512VL
            | X86Feature::AVX512CD
            | X86Feature::AVX512ER
            | X86Feature::AVX512PF => {
                self.enable_with_implications(X86Feature::AVX512F);
            }
            X86Feature::AVX512VBMI
            | X86Feature::AVX512VBMI2
            | X86Feature::AVX512VNNI
            | X86Feature::AVX512BITALG
            | X86Feature::AVX512VPOPCNTDQ => {
                self.enable_with_implications(X86Feature::AVX512BW);
            }
            X86Feature::AVX512BF16 | X86Feature::AVX512FP16 => {
                self.enable_with_implications(X86Feature::AVX512BW);
                self.enabled.insert(X86Feature::AVX512DQ);
            }
            _ => {}
        }
    }

    /// Get list of enabled features, sorted canonically.
    pub fn enabled_features(&self) -> Vec<X86Feature> {
        let mut v: Vec<_> = self.enabled.iter().copied().collect();
        v.sort();
        v
    }
}

impl Default for FeatureGate {
    fn default() -> Self {
        Self {
            enabled: [X86Feature::X86, X86Feature::X86_64, X86Feature::SSE2]
                .iter()
                .copied()
                .collect(),
            is_64bit: true,
        }
    }
}

// ============================================================================
// ISelPatternRule — individual pattern rule
// ============================================================================

/// A single instruction selection pattern rule.
///
/// Maps an IR opcode (optionally with type constraints, predicate conditions,
/// and feature requirements) to an X86 opcode.
#[derive(Debug, Clone)]
pub struct ISelPatternRule {
    /// Unique name for this pattern (e.g. "add_i32_rm").
    pub name: String,

    /// The IR opcode this pattern handles.
    pub ir_opcode: Opcode,

    /// Optional condition on which IR predicate (e.g. for icmp/fcmp).
    pub ir_predicate: Option<u8>,

    /// Result type constraint.
    pub result_type: TypeConstraint,

    /// Operand type constraints (one per operand).
    pub operand_types: Vec<TypeConstraint>,

    /// The X86 opcode to emit.
    pub x86_opcode: X86Opcode,

    /// Cost: lower numbers are preferred (0 = free, 1 = single uop, etc.).
    pub cost: u32,

    /// Priority: higher values are tried first.
    /// Priority ranges encode the ISA level:
    ///   10000–9000  AVX‑512
    ///   8999–7000   AVX2 / FMA / BMI2
    ///   6999–5000   AVX / BMI / ADX
    ///   4999–3000   SSE4.x / SSSE3 / SSE3
    ///   2999–2000   SSE2 / SSE
    ///   1999–1000   Base x86‑64 / i686
    ///    999–0      Fallback
    pub priority: u16,

    /// Whether the instruction sets the flags register.
    pub sets_flags: bool,

    /// Whether this is a conditional instruction (branch, cmov, setcc).
    pub is_conditional: bool,

    /// Required CPU feature; `None` means always available.
    pub required_feature: Option<X86Feature>,

    /// Alternative features (any one is sufficient).
    pub alternative_features: Vec<X86Feature>,

    /// Whether this pattern is only valid in 64-bit mode.
    pub only_64bit: bool,

    /// Whether this pattern is only valid in 32-bit mode.
    pub only_32bit: bool,

    /// Whether this is a commutative pattern (operand order doesn't matter).
    pub is_commutative: bool,

    /// Whether the first operand must be an immediate value.
    pub requires_imm: Option<u32>,

    /// Whether the immediate must fit in this many bits.
    pub imm_fits_bits: Option<u32>,

    /// Human-readable description.
    pub description: String,
}

impl ISelPatternRule {
    /// Create a simple pattern rule.
    pub fn new(
        name: &str,
        ir_opcode: Opcode,
        result_type: TypeConstraint,
        x86_opcode: X86Opcode,
        cost: u32,
        priority: u16,
        required_feature: Option<X86Feature>,
    ) -> Self {
        Self {
            name: name.to_string(),
            ir_opcode,
            ir_predicate: None,
            result_type,
            operand_types: vec![],
            x86_opcode,
            cost,
            priority,
            sets_flags: false,
            is_conditional: false,
            required_feature,
            alternative_features: vec![],
            only_64bit: false,
            only_32bit: false,
            is_commutative: false,
            requires_imm: None,
            imm_fits_bits: None,
            description: String::new(),
        }
    }

    /// Builder-style: set operand type constraints.
    pub fn with_operands(mut self, types: Vec<TypeConstraint>) -> Self {
        self.operand_types = types;
        self
    }

    /// Builder-style: mark as commutative.
    pub fn commutative(mut self) -> Self {
        self.is_commutative = true;
        self
    }

    /// Builder-style: set flags.
    pub fn with_flags(mut self) -> Self {
        self.sets_flags = true;
        self
    }

    /// Builder-style: mark as conditional.
    pub fn conditional(mut self) -> Self {
        self.is_conditional = true;
        self
    }

    /// Builder-style: 64-bit only.
    pub fn only64(mut self) -> Self {
        self.only_64bit = true;
        self
    }

    /// Builder-style: 32-bit only.
    pub fn only32(mut self) -> Self {
        self.only_32bit = true;
        self
    }

    /// Builder-style: set alternative features.
    pub fn with_alternatives(mut self, features: Vec<X86Feature>) -> Self {
        self.alternative_features = features;
        self
    }

    /// Builder-style: set description.
    pub fn with_desc(mut self, desc: &str) -> Self {
        self.description = desc.to_string();
        self
    }

    /// Builder-style: set IR predicate (for comparison patterns).
    pub fn with_pred(mut self, pred: u8) -> Self {
        self.ir_predicate = Some(pred);
        self
    }

    /// Builder-style: require an immediate of a given width.
    pub fn with_imm(mut self, bits: u32) -> Self {
        self.requires_imm = Some(bits);
        self.imm_fits_bits = Some(bits);
        self
    }

    /// Check if this pattern is available given a feature gate.
    pub fn is_available(&self, gate: &FeatureGate) -> bool {
        if self.only_64bit && !gate.is_64bit {
            return false;
        }
        if self.only_32bit && gate.is_64bit {
            return false;
        }
        match &self.required_feature {
            None => true,
            Some(f) => gate.has(*f) || gate.has_any(&self.alternative_features),
        }
    }
}

// ============================================================================
// ISelComplexPattern — multi-instruction patterns
// ============================================================================

/// A complex pattern that may require multiple X86 instructions to implement.
///
/// Examples:
/// - `(add (mul x, c1), c2)` → LEA
/// - `i64 mul` on 32‑bit → mul sequence with EDX:EAX
/// - `f64 sin` → x87 fsin / libcall
/// - Atomic read-modify-write sequences
#[derive(Debug, Clone)]
pub struct ISelComplexPattern {
    /// Name of the pattern.
    pub name: String,

    /// The root IR opcode.
    pub ir_opcode: Opcode,

    /// Nested pattern tree (simplified as a list of sub-opcodes).
    pub sub_instructions: Vec<Opcode>,

    /// The X86 opcodes to emit, in sequence.
    pub x86_sequence: Vec<X86Opcode>,

    /// Total cost of this complex pattern.
    pub cost: u32,

    /// Priority of this pattern.
    pub priority: u16,

    /// Required feature.
    pub required_feature: Option<X86Feature>,

    /// Whether this pattern is 64‑bit only.
    pub only_64bit: bool,

    /// Whether this pattern is 32‑bit only.
    pub only_32bit: bool,

    /// Human-readable description.
    pub description: String,
}

impl ISelComplexPattern {
    pub fn is_available(&self, gate: &FeatureGate) -> bool {
        if self.only_64bit && !gate.is_64bit {
            return false;
        }
        if self.only_32bit && gate.is_64bit {
            return false;
        }
        match &self.required_feature {
            None => true,
            Some(f) => gate.has(*f),
        }
    }
}

// ============================================================================
// IR Coverage Tracker
// ============================================================================

/// Tracks which IR opcodes have ISel patterns and reports coverage.
#[derive(Debug, Clone)]
pub struct ISelCoverageTracker {
    /// Map from IR opcode name to the number of patterns that cover it.
    pattern_counts: HashMap<String, usize>,
    /// Map from IR opcode name to the set of X86 features that cover it.
    feature_coverage: HashMap<String, HashSet<X86Feature>>,
    /// Set of opcodes that have at least one non-fallback pattern.
    covered: HashSet<String>,
    /// Total patterns registered.
    total_patterns: usize,
    /// Total complex patterns registered.
    total_complex: usize,
}

impl ISelCoverageTracker {
    pub fn new() -> Self {
        Self {
            pattern_counts: HashMap::new(),
            feature_coverage: HashMap::new(),
            covered: HashSet::new(),
            total_patterns: 0,
            total_complex: 0,
        }
    }

    /// Register a pattern for an IR opcode.
    pub fn register_pattern(&mut self, ir_op: &str, feature: Option<X86Feature>) {
        *self.pattern_counts.entry(ir_op.to_string()).or_insert(0) += 1;
        self.total_patterns += 1;
        if let Some(f) = feature {
            self.feature_coverage
                .entry(ir_op.to_string())
                .or_default()
                .insert(f);
        }
        self.covered.insert(ir_op.to_string());
    }

    /// Register a complex pattern.
    pub fn register_complex(&mut self, _ir_op: &str) {
        self.total_complex += 1;
    }

    /// Get pattern count for an IR opcode.
    pub fn count_for(&self, ir_op: &str) -> usize {
        self.pattern_counts.get(ir_op).copied().unwrap_or(0)
    }

    /// Check if an IR opcode is covered.
    pub fn is_covered(&self, ir_op: &str) -> bool {
        self.covered.contains(ir_op)
    }

    /// Get all covered IR opcodes.
    pub fn covered_opcodes(&self) -> Vec<String> {
        let mut v: Vec<_> = self.covered.iter().cloned().collect();
        v.sort();
        v
    }

    /// Get gaps: IR opcodes that we know about but have no pattern.
    pub fn gaps(&self) -> Vec<&'static str> {
        let all_ops: &[&str] = &[
            "add",
            "sub",
            "mul",
            "udiv",
            "sdiv",
            "urem",
            "srem",
            "fadd",
            "fsub",
            "fmul",
            "fdiv",
            "frem",
            "shl",
            "lshr",
            "ashr",
            "and",
            "or",
            "xor",
            "icmp",
            "fcmp",
            "load",
            "store",
            "alloca",
            "getelementptr",
            "trunc",
            "zext",
            "sext",
            "fptrunc",
            "fpext",
            "fptoui",
            "fptosi",
            "uitofp",
            "sitofp",
            "ptrtoint",
            "inttoptr",
            "bitcast",
            "br",
            "switch",
            "indirectbr",
            "invoke",
            "callbr",
            "call",
            "ret",
            "resume",
            "unreachable",
            "phi",
            "select",
            "freeze",
            "extractelement",
            "insertelement",
            "shufflevector",
            "extractvalue",
            "insertvalue",
            "fence",
            "cmpxchg",
            "atomicrmw",
            "va_arg",
            "landingpad",
        ];
        all_ops
            .iter()
            .filter(|op| !self.covered.contains(**op))
            .copied()
            .collect()
    }

    /// Coverage percentage.
    pub fn coverage_pct(&self) -> f64 {
        let total_known = [
            "add",
            "sub",
            "mul",
            "udiv",
            "sdiv",
            "urem",
            "srem",
            "fadd",
            "fsub",
            "fmul",
            "fdiv",
            "frem",
            "shl",
            "lshr",
            "ashr",
            "and",
            "or",
            "xor",
            "icmp",
            "fcmp",
            "load",
            "store",
            "alloca",
            "getelementptr",
            "trunc",
            "zext",
            "sext",
            "fptrunc",
            "fpext",
            "fptoui",
            "fptosi",
            "uitofp",
            "sitofp",
            "ptrtoint",
            "inttoptr",
            "bitcast",
            "br",
            "switch",
            "indirectbr",
            "invoke",
            "callbr",
            "call",
            "ret",
            "resume",
            "unreachable",
            "phi",
            "select",
            "freeze",
            "extractelement",
            "insertelement",
            "shufflevector",
            "extractvalue",
            "insertvalue",
            "fence",
            "cmpxchg",
            "atomicrmw",
            "va_arg",
            "landingpad",
        ]
        .len() as f64;

        if total_known == 0.0 {
            return 100.0;
        }
        (self.covered.len() as f64 / total_known) * 100.0
    }

    /// Summary statistics.
    pub fn summary(&self) -> String {
        format!(
            "ISel Coverage: {}/{} IR ops covered ({:.1}%), {} patterns, {} complex",
            self.covered.len(),
            48, // known opcodes
            self.coverage_pct(),
            self.total_patterns,
            self.total_complex,
        )
    }

    /// Report all gaps.
    pub fn gap_report(&self) -> String {
        let gaps = self.gaps();
        if gaps.is_empty() {
            "FULL COVERAGE: all IR opcodes have at least one pattern.".to_string()
        } else {
            let mut s = format!("GAP REPORT: {} IR opcodes missing patterns:\n", gaps.len());
            for g in &gaps {
                s.push_str(&format!("  - {}\n", g));
            }
            s
        }
    }
}

impl Default for ISelCoverageTracker {
    fn default() -> Self {
        Self::new()
    }
}

// ============================================================================
// ISelPatternMatcher — pattern matching engine
// ============================================================================

/// Result of matching an ISel pattern.
#[derive(Debug, Clone)]
pub struct MatchResult {
    /// The matched pattern rule.
    pub rule: ISelPatternRule,
    /// How the operands map to X86 operands.
    pub operand_mapping: Vec<usize>,
    /// Whether this was a complex (multi-instruction) match.
    pub is_complex: bool,
    /// If complex, the full sequence of X86 opcodes.
    pub complex_sequence: Option<Vec<X86Opcode>>,
}

/// The pattern matching engine.
pub struct ISelPatternMatcher {
    /// All simple pattern rules, sorted by descending priority.
    rules: Vec<ISelPatternRule>,
    /// All complex patterns.
    complex_patterns: Vec<ISelComplexPattern>,
    /// Coverage tracker.
    pub coverage: ISelCoverageTracker,
    /// Feature gate for pattern availability.
    pub features: FeatureGate,
    /// Statistics.
    pub stats: MatchStats,
}

/// Match statistics.
#[derive(Debug, Default, Clone)]
pub struct MatchStats {
    /// Total lookup requests.
    pub total_queries: usize,
    /// Successful matches.
    pub matches: usize,
    /// Failed matches (no pattern found).
    pub misses: usize,
    /// Complex pattern matches.
    pub complex_matches: usize,
    /// Per-opcode hit counts.
    pub opcode_hits: HashMap<String, usize>,
}

impl MatchStats {
    pub fn new() -> Self {
        Self::default()
    }

    pub fn record_hit(&mut self, ir_op: &str) {
        self.total_queries += 1;
        self.matches += 1;
        *self.opcode_hits.entry(ir_op.to_string()).or_insert(0) += 1;
    }

    pub fn record_miss(&mut self) {
        self.total_queries += 1;
        self.misses += 1;
    }

    pub fn record_complex(&mut self, ir_op: &str) {
        self.complex_matches += 1;
        *self.opcode_hits.entry(ir_op.to_string()).or_insert(0) += 1;
    }

    pub fn hit_rate(&self) -> f64 {
        if self.total_queries == 0 {
            return 0.0;
        }
        (self.matches as f64 / self.total_queries as f64) * 100.0
    }
}

impl ISelPatternMatcher {
    /// Create a new matcher with all built-in patterns.
    pub fn new(features: FeatureGate) -> Self {
        let rules = build_golden_rules();
        let complex = build_complex_patterns();
        let mut coverage = ISelCoverageTracker::new();

        for rule in &rules {
            let op_name = rule.ir_opcode.name().to_string();
            coverage.register_pattern(&op_name, rule.required_feature);
        }
        for cp in &complex {
            let op_name = cp.ir_opcode.name().to_string();
            coverage.register_complex(&op_name);
        }

        Self {
            rules,
            complex_patterns: complex,
            coverage,
            features,
            stats: MatchStats::new(),
        }
    }

    /// Find the best matching pattern for an IR opcode, given result type.
    pub fn find_best(
        &mut self,
        ir_opcode: Opcode,
        _result_type: &TypeConstraint,
    ) -> Option<MatchResult> {
        let op_name = ir_opcode.name().to_string();

        // Collect candidates
        let candidates: Vec<&ISelPatternRule> = self
            .rules
            .iter()
            .filter(|r| r.ir_opcode == ir_opcode && r.is_available(&self.features))
            .collect();

        if candidates.is_empty() {
            self.stats.record_miss();
            return None;
        }

        // Pick highest priority
        let best = candidates.iter().max_by_key(|r| r.priority).unwrap();

        self.stats.record_hit(&op_name);

        Some(MatchResult {
            rule: (*best).clone(),
            operand_mapping: (0..best.operand_types.len()).collect(),
            is_complex: false,
            complex_sequence: None,
        })
    }

    /// Find a complex pattern match.
    pub fn find_complex(&mut self, ir_opcode: Opcode) -> Option<&ISelComplexPattern> {
        let candidates: Vec<&ISelComplexPattern> = self
            .complex_patterns
            .iter()
            .filter(|cp| cp.ir_opcode == ir_opcode && cp.is_available(&self.features))
            .collect();

        if let Some(best) = candidates.iter().max_by_key(|cp| cp.priority) {
            self.stats.record_complex(&ir_opcode.name().to_string());
            Some(best)
        } else {
            None
        }
    }

    /// Look up all patterns for a given IR opcode.
    pub fn patterns_for(&self, ir_opcode: Opcode) -> Vec<&ISelPatternRule> {
        self.rules
            .iter()
            .filter(|r| r.ir_opcode == ir_opcode && r.is_available(&self.features))
            .collect()
    }

    /// Look up all complex patterns for a given IR opcode.
    pub fn complex_for(&self, ir_opcode: Opcode) -> Vec<&ISelComplexPattern> {
        self.complex_patterns
            .iter()
            .filter(|cp| cp.ir_opcode == ir_opcode && cp.is_available(&self.features))
            .collect()
    }

    /// Total number of patterns available.
    pub fn total_patterns(&self) -> usize {
        self.rules
            .iter()
            .filter(|r| r.is_available(&self.features))
            .count()
    }
}

// ============================================================================
// Pattern builders — helper macros for concise pattern definitions
// ============================================================================

macro_rules! rule {
    ($name:expr_2021, $ir:expr_2021, $type:expr_2021, $x86:expr_2021, $cost:expr_2021, $pri:expr_2021, $feat:expr_2021) => {
        ISelPatternRule::new($name, $ir, $type, $x86, $cost, $pri, $feat)
    };
}

// ============================================================================
// Build golden rules — the complete pattern database
// ============================================================================

/// Build all golden ISel pattern rules.
///
/// Patterns are organized by category and sorted by descending priority.
/// The final vector is sorted once at the end.
pub fn build_golden_rules() -> Vec<ISelPatternRule> {
    let mut rules: Vec<ISelPatternRule> = Vec::new();

    // ========================================================================
    // Priority: 10000–9000  —  AVX‑512 patterns
    // ========================================================================

    // --- AVX-512 Integer Arithmetic (i32/i64 with ZMM) ---
    rules.push(
        rule!(
            "avx512_add_i32",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPADDD_Z,
            1,
            10000,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 add → VPADDD (ZMM, AVX-512F)"),
    );
    rules.push(
        rule!(
            "avx512_add_i64",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::VPADDQ_Z,
            1,
            9999,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I64),
            TypeConstraint::Int(IntWidth::I64),
        ])
        .with_desc("i64 add → VPADDQ (ZMM, AVX-512F)"),
    );
    rules.push(
        rule!(
            "avx512_add_i8",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::VPADDB_Z,
            1,
            9998,
            Some(X86Feature::AVX512BW)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I8),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("i8 add → VPADDB (ZMM, AVX-512BW)"),
    );
    rules.push(
        rule!(
            "avx512_add_i16",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::VPADDW_Z,
            1,
            9997,
            Some(X86Feature::AVX512BW)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I16),
            TypeConstraint::Int(IntWidth::I16),
        ])
        .with_desc("i16 add → VPADDW (ZMM, AVX-512BW)"),
    );

    rules.push(
        rule!(
            "avx512_sub_i32",
            Opcode::Sub,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPSUBD_Z,
            1,
            9996,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 sub → VPSUBD (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_sub_i64",
            Opcode::Sub,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::VPSUBQ_Z,
            1,
            9995,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I64),
            TypeConstraint::Int(IntWidth::I64),
        ])
        .with_desc("i64 sub → VPSUBQ (ZMM)"),
    );

    rules.push(
        rule!(
            "avx512_mul_i32",
            Opcode::Mul,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPMULLD_Z,
            1,
            9994,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 mul → VPMULLD (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_mul_i64",
            Opcode::Mul,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::VPMULLQ_Z,
            1,
            9993,
            Some(X86Feature::AVX512DQ)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I64),
            TypeConstraint::Int(IntWidth::I64),
        ])
        .with_desc("i64 mul → VPMULLQ (ZMM, AVX-512DQ)"),
    );

    rules.push(
        rule!(
            "avx512_and_i32",
            Opcode::And,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPANDD_Z,
            1,
            9992,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 and → VPANDD (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_or_i32",
            Opcode::Or,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPORD_Z,
            1,
            9991,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 or → VPORD (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_xor_i32",
            Opcode::Xor,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPXORD_Z,
            1,
            9990,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 xor → VPXORD (ZMM)"),
    );

    // --- AVX-512 FP Arithmetic (f32/f64 with ZMM) ---
    rules.push(
        rule!(
            "avx512_fadd_f32",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VADDPS_Z,
            1,
            9989,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 fadd → VADDPS (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_fadd_f64",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VADDPD_Z,
            1,
            9988,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 fadd → VADDPD (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_fsub_f32",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VSUBPS,
            1,
            9987,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 fsub → VSUBPS (AVX)"),
    );
    rules.push(
        rule!(
            "avx512_fsub_f64",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VSUBPD,
            1,
            9986,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 fsub → VSUBPD (AVX)"),
    );
    rules.push(
        rule!(
            "avx512_fmul_f32",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VMULPS_Z,
            1,
            9985,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 fmul → VMULPS (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_fmul_f64",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VMULPD_Z,
            1,
            9984,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 fmul → VMULPD (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_fdiv_f32",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VDIVPS_Z,
            1,
            9983,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 fdiv → VDIVPS (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_fdiv_f64",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VDIVPD_Z,
            1,
            9982,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 fdiv → VDIVPD (ZMM)"),
    );

    // --- AVX-512 Scalar FP ---
    rules.push(
        rule!(
            "avx512_fadd_f32_scalar",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VADDSS_Z,
            1,
            9981,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 fadd scalar → VADDSS (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_fadd_f64_scalar",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VADDSD_Z,
            1,
            9980,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 fadd scalar → VADDSD (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_fmul_f32_scalar",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VMULSS_Z,
            1,
            9979,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 fmul scalar → VMULSS (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_fmul_f64_scalar",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VMULSD_Z,
            1,
            9978,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 fmul scalar → VMULSD (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_fdiv_f32_scalar",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VDIVSS_Z,
            1,
            9977,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 fdiv scalar → VDIVSS (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_fdiv_f64_scalar",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VDIVSD_Z,
            1,
            9976,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 fdiv scalar → VDIVSD (ZMM)"),
    );

    // --- AVX-512 FP16 ---
    rules.push(
        rule!(
            "avx512_fp16_fadd",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F16),
            X86Opcode::VADDPH_Z,
            1,
            9975,
            Some(X86Feature::AVX512FP16)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F16),
            TypeConstraint::Float(FloatWidth::F16),
        ])
        .with_desc("f16 fadd → VADDPH (ZMM, AVX-512 FP16)"),
    );
    rules.push(
        rule!(
            "avx512_fp16_fmul",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F16),
            X86Opcode::VMULPH_Z,
            1,
            9974,
            Some(X86Feature::AVX512FP16)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F16),
            TypeConstraint::Float(FloatWidth::F16),
        ])
        .with_desc("f16 fmul → VMULPH (ZMM, AVX-512 FP16)"),
    );
    rules.push(
        rule!(
            "avx512_fp16_fdiv",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F16),
            X86Opcode::VDIVPH_Z,
            1,
            9973,
            Some(X86Feature::AVX512FP16)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F16),
            TypeConstraint::Float(FloatWidth::F16),
        ])
        .with_desc("f16 fdiv → VDIVPH (ZMM, AVX-512 FP16)"),
    );

    // --- AVX-512 FMA ---
    rules.push(
        rule!(
            "avx512_fma_faddmul_f32_132",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMADD132PS,
            1,
            9972,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma f32 a*b+c → VFMADD132PS (3‑op, ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_fma_faddmul_f64_132",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VFMADD132PD,
            1,
            9971,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("fma f64 a*b+c → VFMADD132PD (3‑op, ZMM)"),
    );

    // --- AVX-512 FMA FP16 ---
    rules.push(
        rule!(
            "avx512_fp16_fma_132",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F16),
            X86Opcode::VFMADD132PH_Z,
            1,
            9970,
            Some(X86Feature::AVX512FP16)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F16),
            TypeConstraint::Float(FloatWidth::F16),
            TypeConstraint::Float(FloatWidth::F16),
        ])
        .with_desc("fma f16 a*b+c → VFMADD132PH (ZMM, AVX-512 FP16)"),
    );

    // --- AVX-512 FMA sub variants ---
    rules.push(
        rule!(
            "avx512_fmsub_f32_132",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMSUB132PS,
            1,
            9969,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma f32 a*b-c → VFMSUB132PS (3‑op, ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_fnmsub_f32_132",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFNMSUB132PS,
            1,
            9968,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma f32 -(a*b)+c → VFNMSUB132PS (3‑op, ZMM)"),
    );

    // --- AVX-512 Mask register ops (for vector icmp) ---
    rules.push(
        rule!(
            "avx512_kand",
            Opcode::And,
            TypeConstraint::AnyInt,
            X86Opcode::KANDW_Z,
            1,
            9967,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![TypeConstraint::AnyInt, TypeConstraint::AnyInt])
        .with_desc("mask and → KANDW (AVX-512F)"),
    );
    rules.push(
        rule!(
            "avx512_kor",
            Opcode::Or,
            TypeConstraint::AnyInt,
            X86Opcode::KORW_Z,
            1,
            9966,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![TypeConstraint::AnyInt, TypeConstraint::AnyInt])
        .with_desc("mask or → KORW (AVX-512F)"),
    );
    rules.push(
        rule!(
            "avx512_kxor",
            Opcode::Xor,
            TypeConstraint::AnyInt,
            X86Opcode::KXORW_Z,
            1,
            9965,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![TypeConstraint::AnyInt, TypeConstraint::AnyInt])
        .with_desc("mask xor → KXORW (AVX-512F)"),
    );
    rules.push(
        rule!(
            "avx512_knot",
            Opcode::Xor,
            TypeConstraint::AnyInt,
            X86Opcode::KNOTW_Z,
            1,
            9964,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![TypeConstraint::AnyInt, TypeConstraint::AnyInt])
        .with_desc("mask not → KNOTW (AVX-512F)"),
    );

    // --- AVX-512 BF16 ---
    rules.push(
        rule!(
            "avx512_bf16_cvt",
            Opcode::FPTrunc,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VCVTNEEBF162PS_Z,
            1,
            9963,
            Some(X86Feature::AVX512BF16)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("bf16 convert → VCVTNEEBF162PS (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_bf16_dp",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VDPBF16PS_Z,
            1,
            9962,
            Some(X86Feature::AVX512BF16)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("bf16 dot-product → VDPBF16PS (ZMM)"),
    );

    // --- AVX-512 convert ---
    rules.push(
        rule!(
            "avx512_cvtdq2ps",
            Opcode::SIToFP,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VCVTDQ2PS_Z,
            1,
            9961,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32→f32 packed → VCVTDQ2PS (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_cvtps2dq",
            Opcode::FPToSI,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VCVTPS2DQ_Z,
            1,
            9960,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32→i32 packed → VCVTPS2DQ (ZMM)"),
    );

    // --- AVX-512 min/max ---
    rules.push(
        rule!(
            "avx512_min_f32",
            Opcode::FCmp,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VMINPS_Z,
            1,
            9959,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 min → VMINPS (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_max_f32",
            Opcode::FCmp,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VMAXPS_Z,
            1,
            9958,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 max → VMAXPS (ZMM)"),
    );

    // --- AVX-512 variable shift/rotate ---
    rules.push(
        rule!(
            "avx512_shlv_i32",
            Opcode::Shl,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPROLVD_Z,
            1,
            9957,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 variable shift-left → VPROLVD (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_shrv_i32",
            Opcode::LShr,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPRORVD_Z,
            1,
            9956,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 variable shift-right → VPRORVD (ZMM)"),
    );

    // --- AVX-512 compress/expand ---
    rules.push(
        rule!(
            "avx512_compress_f32",
            Opcode::Store,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VCOMPRESSPS_Z,
            2,
            9955,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 compress store → VCOMPRESSPS (ZMM)"),
    );
    rules.push(
        rule!(
            "avx512_expand_f32",
            Opcode::Load,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VEXPANDPS_Z,
            2,
            9954,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("f32 expand load → VEXPANDPS (ZMM)"),
    );

    // --- AVX-512 gather ---
    rules.push(
        rule!(
            "avx512_gather_dd",
            Opcode::Load,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPGATHERDD_Z,
            3,
            9953,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 gather → VPGATHERDD (ZMM)"),
    );

    // --- AVX-512 broadcast ---
    rules.push(
        rule!(
            "avx512_broadcast_f32",
            Opcode::Load,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VBROADCASTSS_Z,
            2,
            9952,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("f32 broadcast load → VBROADCASTSS (ZMM)"),
    );

    // --- AVX-512 test ---
    rules.push(
        rule!(
            "avx512_vptest",
            Opcode::ICmp,
            TypeConstraint::AnyInt,
            X86Opcode::VCMPD_Z,
            1,
            9951,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![TypeConstraint::AnyInt, TypeConstraint::AnyInt])
        .with_desc("vector icmp → VCMPD (ZMM, mask)"),
    );

    // ========================================================================
    // Priority: 8999–7000  — AVX2 / FMA / BMI2 patterns
    // ========================================================================

    // --- AVX2 Integer Arithmetic ---
    rules.push(
        rule!(
            "avx2_add_i8",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::VPADDB,
            1,
            8999,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I8),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("i8 add → VPADDB (YMM, AVX2)"),
    );
    rules.push(
        rule!(
            "avx2_add_i16",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::VPADDW,
            1,
            8998,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I16),
            TypeConstraint::Int(IntWidth::I16),
        ])
        .with_desc("i16 add → VPADDW (YMM, AVX2)"),
    );
    rules.push(
        rule!(
            "avx2_add_i32",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPADDD,
            1,
            8997,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 add → VPADDD (YMM, AVX2)"),
    );
    rules.push(
        rule!(
            "avx2_add_i64",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::VPADDQ,
            1,
            8996,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I64),
            TypeConstraint::Int(IntWidth::I64),
        ])
        .with_desc("i64 add → VPADDQ (YMM, AVX2)"),
    );

    rules.push(
        rule!(
            "avx2_sub_i8",
            Opcode::Sub,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::VPSUBB,
            1,
            8995,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I8),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("i8 sub → VPSUBB (YMM, AVX2)"),
    );
    rules.push(
        rule!(
            "avx2_sub_i16",
            Opcode::Sub,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::VPSUBW,
            1,
            8994,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I16),
            TypeConstraint::Int(IntWidth::I16),
        ])
        .with_desc("i16 sub → VPSUBW (YMM, AVX2)"),
    );
    rules.push(
        rule!(
            "avx2_sub_i32",
            Opcode::Sub,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPSUBD,
            1,
            8993,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 sub → VPSUBD (YMM, AVX2)"),
    );
    rules.push(
        rule!(
            "avx2_sub_i64",
            Opcode::Sub,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::VPSUBQ,
            1,
            8992,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I64),
            TypeConstraint::Int(IntWidth::I64),
        ])
        .with_desc("i64 sub → VPSUBQ (YMM, AVX2)"),
    );

    rules.push(
        rule!(
            "avx2_mul_i32",
            Opcode::Mul,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPMULLD,
            1,
            8991,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 mul → VPMULLD (YMM, AVX2)"),
    );

    // --- AVX2 Integer min/max ---
    rules.push(
        rule!(
            "avx2_minsb",
            Opcode::ICmp,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::VPMINSB,
            1,
            8980,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I8),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("i8 signed min → VPMINSB (YMM, AVX2)"),
    );
    rules.push(
        rule!(
            "avx2_maxsb",
            Opcode::ICmp,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::VPMAXSB,
            1,
            8979,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I8),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("i8 signed max → VPMAXSB (YMM, AVX2)"),
    );
    rules.push(
        rule!(
            "avx2_minsd",
            Opcode::ICmp,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPMINSD,
            1,
            8978,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 signed min → VPMINSD (YMM, AVX2)"),
    );
    rules.push(
        rule!(
            "avx2_maxsd",
            Opcode::ICmp,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPMAXSD,
            1,
            8977,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 signed max → VPMAXSD (YMM, AVX2)"),
    );

    // --- FMA patterns ---
    rules.push(
        rule!(
            "fma132_ps",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMADD132PS,
            1,
            8970,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma f32 a*b+c → VFMADD132PS (3‑op, FMA)"),
    );
    rules.push(
        rule!(
            "fma213_ps",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMADD213PS,
            1,
            8969,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma f32 b*a+c → VFMADD213PS (3‑op, FMA)"),
    );
    rules.push(
        rule!(
            "fma231_ps",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMADD231PS,
            1,
            8968,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma f32 c+a*b → VFMADD231PS (3‑op, FMA)"),
    );
    rules.push(
        rule!(
            "fma132_pd",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMADD132PD,
            1,
            8967,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma f64 a*b+c → VFMADD132PD (3‑op, FMA)"),
    );

    // FMA scalar (SS/SD)
    rules.push(
        rule!(
            "fma132_ss",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMADD132SS,
            1,
            8966,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma f32 a*b+c scalar → VFMADD132SS (FMA)"),
    );
    rules.push(
        rule!(
            "fma132_sd",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMADD132SD,
            1,
            8965,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma f64 a*b+c scalar → VFMADD132SD (FMA)"),
    );

    rules.push(
        rule!(
            "fma231_ss",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMADD231SS,
            1,
            8964,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma f32 c+a*b scalar → VFMADD231SS (FMA)"),
    );
    rules.push(
        rule!(
            "fma231_sd",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMADD231SD,
            1,
            8963,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma f64 c+a*b scalar → VFMADD231SD (FMA)"),
    );

    // FMA sub variants
    rules.push(
        rule!(
            "fms132_ps",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMSUB132PS,
            1,
            8962,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma sub f32 a*b-c → VFMSUB132PS (FMA)"),
    );
    rules.push(
        rule!(
            "fnms132_ps",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFNMSUB132PS,
            1,
            8961,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma neg-sub f32 -(a*b)+c → VFNMSUB132PS (FMA)"),
    );
    rules.push(
        rule!(
            "fms213_ps",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMSUB213PS,
            1,
            8960,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma sub f32 b*a-c → VFMSUB213PS (FMA)"),
    );
    rules.push(
        rule!(
            "fms231_ps",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMSUB231PS,
            1,
            8959,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma sub f32 c-a*b → VFMSUB231PS (FMA)"),
    );

    // FMA scalar sub
    rules.push(
        rule!(
            "fms132_ss",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMSUB132SS,
            1,
            8958,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma sub f32 a*b-c scalar → VFMSUB132SS (FMA)"),
    );
    rules.push(
        rule!(
            "fms132_sd",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMSUB132SD,
            1,
            8957,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma sub f64 a*b-c scalar → VFMSUB132SD (FMA)"),
    );

    // FMA negative multiply-add
    rules.push(
        rule!(
            "fnmadd132_ps",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFNMADD132PS,
            1,
            8956,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma -(a*b)+c → VFNMADD132PS (FMA)"),
    );
    rules.push(
        rule!(
            "fnmadd213_ps",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFNMADD213PS,
            1,
            8955,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma -(b*a)+c → VFNMADD213PS (FMA)"),
    );
    rules.push(
        rule!(
            "fnmadd231_ps",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFNMADD231PS,
            1,
            8954,
            Some(X86Feature::FMA)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma c-(a*b) → VFNMADD231PS (FMA)"),
    );

    // FMA addsub (alternating add/sub)
    rules.push(
        rule!(
            "fmaddsub132_ps",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VFMADDSUB132PS,
            1,
            8953,
            Some(X86Feature::AVX512F)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fma add-sub f32 → VFMADDSUB132PS (AVX-512F)"),
    );

    // --- BMI2 patterns ---
    rules.push(
        rule!(
            "bmi2_bzhi",
            Opcode::And,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI2_BZHI,
            1,
            8940,
            Some(X86Feature::BMI2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("zero high bits → BZHI (BMI2)"),
    );
    rules.push(
        rule!(
            "bmi2_mulx",
            Opcode::Mul,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI2_MULX,
            1,
            8939,
            Some(X86Feature::BMI2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("mul no-flags → MULX (BMI2)"),
    );
    rules.push(
        rule!(
            "bmi2_pdep",
            Opcode::Or,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI2_PDEP,
            1,
            8938,
            Some(X86Feature::BMI2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("parallel bit deposit → PDEP (BMI2)"),
    );
    rules.push(
        rule!(
            "bmi2_pext",
            Opcode::And,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI2_PEXT,
            1,
            8937,
            Some(X86Feature::BMI2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("parallel bit extract → PEXT (BMI2)"),
    );
    rules.push(
        rule!(
            "bmi2_rorx",
            Opcode::LShr,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI2_RORX,
            1,
            8936,
            Some(X86Feature::BMI2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("rotate right no-flags → RORX (BMI2)"),
    );
    rules.push(
        rule!(
            "bmi2_shlx",
            Opcode::Shl,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI2_SHLX,
            1,
            8935,
            Some(X86Feature::BMI2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("shift left no-flags → SHLX (BMI2)"),
    );
    rules.push(
        rule!(
            "bmi2_shrx",
            Opcode::LShr,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI2_SHRX,
            1,
            8934,
            Some(X86Feature::BMI2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("shift right no-flags → SHRX (BMI2)"),
    );
    rules.push(
        rule!(
            "bmi2_sarx",
            Opcode::AShr,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI2_SARX,
            1,
            8933,
            Some(X86Feature::BMI2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("arith shift right no-flags → SARX (BMI2)"),
    );

    // --- AVX2 permute ---
    rules.push(
        rule!(
            "avx2_vpermd",
            Opcode::ShuffleVector,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPERMQ,
            1,
            8920,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 permute → VPERMQ (YMM, AVX2)"),
    );
    rules.push(
        rule!(
            "avx2_vpermpd",
            Opcode::ShuffleVector,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VPERMPD,
            1,
            8919,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("f64 permute → VPERMPD (YMM, AVX2)"),
    );

    // --- AVX2 gather ---
    rules.push(
        rule!(
            "avx2_gather_dd",
            Opcode::Load,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPGATHERDD,
            3,
            8910,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 gather → VPGATHERDD (YMM, AVX2)"),
    );

    // --- AVX2 broadcast ---
    rules.push(
        rule!(
            "avx2_broadcast_b",
            Opcode::Load,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::VPBROADCASTB,
            2,
            8900,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("i8 broadcast → VPBROADCASTB (YMM, AVX2)"),
    );
    rules.push(
        rule!(
            "avx2_broadcast_w",
            Opcode::Load,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::VPBROADCASTW,
            2,
            8899,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("i16 broadcast → VPBROADCASTW (YMM, AVX2)"),
    );
    rules.push(
        rule!(
            "avx2_broadcast_d",
            Opcode::Load,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::VPBROADCASTD,
            2,
            8898,
            Some(X86Feature::AVX2)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("i32 broadcast → VPBROADCASTD (YMM, AVX2)"),
    );

    // ========================================================================
    // Priority: 6999–5000  — AVX / BMI / ADX patterns
    // ========================================================================

    // --- AVX FP Arithmetic ---
    rules.push(
        rule!(
            "avx_fadd_ps",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VADDPS,
            1,
            6999,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 fadd → VADDPS (YMM, AVX)"),
    );
    rules.push(
        rule!(
            "avx_fadd_pd",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VADDPD,
            1,
            6998,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 fadd → VADDPD (YMM, AVX)"),
    );
    rules.push(
        rule!(
            "avx_fsub_ps",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VSUBPS,
            1,
            6997,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 fsub → VSUBPS (YMM, AVX)"),
    );
    rules.push(
        rule!(
            "avx_fsub_pd",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VSUBPD,
            1,
            6996,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 fsub → VSUBPD (YMM, AVX)"),
    );
    rules.push(
        rule!(
            "avx_fmul_ps",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VMULPS,
            1,
            6995,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 fmul → VMULPS (YMM, AVX)"),
    );
    rules.push(
        rule!(
            "avx_fmul_pd",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VMULPD,
            1,
            6994,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 fmul → VMULPD (YMM, AVX)"),
    );
    rules.push(
        rule!(
            "avx_fdiv_ps",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VDIVPS,
            1,
            6993,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 fdiv → VDIVPS (YMM, AVX)"),
    );
    rules.push(
        rule!(
            "avx_fdiv_pd",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VDIVPD,
            1,
            6992,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 fdiv → VDIVPD (YMM, AVX)"),
    );

    // --- AVX Scalar FP ---
    rules.push(
        rule!(
            "avx_addss",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VADDSS,
            1,
            6991,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 add scalar → VADDSS (AVX)"),
    );
    rules.push(
        rule!(
            "avx_addsd",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VADDSD,
            1,
            6990,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 add scalar → VADDSD (AVX)"),
    );
    rules.push(
        rule!(
            "avx_subss",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VSUBSS,
            1,
            6989,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 sub scalar → VSUBSS (AVX)"),
    );
    rules.push(
        rule!(
            "avx_subsd",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VSUBSD,
            1,
            6988,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 sub scalar → VSUBSD (AVX)"),
    );
    rules.push(
        rule!(
            "avx_mulss",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VMULSS,
            1,
            6987,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 mul scalar → VMULSS (AVX)"),
    );
    rules.push(
        rule!(
            "avx_mulsd",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VMULSD,
            1,
            6986,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 mul scalar → VMULSD (AVX)"),
    );
    rules.push(
        rule!(
            "avx_divss",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VDIVSS,
            1,
            6985,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 div scalar → VDIVSS (AVX)"),
    );
    rules.push(
        rule!(
            "avx_divsd",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::VDIVSD,
            1,
            6984,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 div scalar → VDIVSD (AVX)"),
    );

    // --- AVX bitwise ---
    rules.push(
        rule!(
            "avx_andps",
            Opcode::And,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VANDPS,
            1,
            6980,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 bitwise and → VANDPS (AVX)"),
    );
    rules.push(
        rule!(
            "avx_orps",
            Opcode::Or,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VORPS,
            1,
            6979,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 bitwise or → VORPS (AVX)"),
    );
    rules.push(
        rule!(
            "avx_xorps",
            Opcode::Xor,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VXORPS,
            1,
            6978,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 bitwise xor → VXORPS (AVX)"),
    );
    rules.push(
        rule!(
            "avx_andnps",
            Opcode::And,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VANDNPS,
            1,
            6977,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 bitwise andn → VANDNPS (AVX)"),
    );

    // --- AVX convert ---
    rules.push(
        rule!(
            "avx_cvtsi2ss",
            Opcode::SIToFP,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::CVTSI2SS,
            1,
            6970,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32→f32 scalar → CVTSI2SS (AVX)"),
    );
    rules.push(
        rule!(
            "avx_cvtsi2sd",
            Opcode::SIToFP,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::CVTSI2SD,
            1,
            6969,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32→f64 scalar → CVTSI2SD (AVX)"),
    );
    rules.push(
        rule!(
            "avx_cvtss2si",
            Opcode::FPToSI,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::CVTSS2SI,
            1,
            6968,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32→i32 scalar → CVTSS2SI (AVX)"),
    );
    rules.push(
        rule!(
            "avx_cvtsd2si",
            Opcode::FPToSI,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::CVTSD2SI,
            1,
            6967,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F64)])
        .with_desc("f64→i32 scalar → CVTSD2SI (AVX)"),
    );

    // --- AVX broadcast ---
    rules.push(
        rule!(
            "avx_vbroadcastss",
            Opcode::Load,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VBROADCASTSS,
            2,
            6960,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("f32 broadcast → VBROADCASTSS (AVX)"),
    );

    // --- AVX permute ---
    rules.push(
        rule!(
            "avx_vpermilps",
            Opcode::ShuffleVector,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VPERMILPS,
            1,
            6950,
            Some(X86Feature::AVX)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("f32 permute imm → VPERMILPS (AVX)"),
    );

    // --- BMI patterns ---
    rules.push(
        rule!(
            "bmi_andn",
            Opcode::And,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI_ANDN,
            1,
            6900,
            Some(X86Feature::BMI)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("(~a)&b → ANDN (BMI)"),
    );
    rules.push(
        rule!(
            "bmi_bextr",
            Opcode::And,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI_BEXTR,
            1,
            6899,
            Some(X86Feature::BMI)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("bit field extract → BEXTR (BMI)"),
    );
    rules.push(
        rule!(
            "bmi_blsi",
            Opcode::And,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI_BLSI,
            1,
            6898,
            Some(X86Feature::BMI)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("isolate lowest set bit → BLSI (BMI)"),
    );
    rules.push(
        rule!(
            "bmi_blsmsk",
            Opcode::Or,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI_BLSMSK,
            1,
            6897,
            Some(X86Feature::BMI)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("mask up to lowest set bit → BLSMSK (BMI)"),
    );
    rules.push(
        rule!(
            "bmi_blsr",
            Opcode::Sub,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BMI_BLSR,
            1,
            6896,
            Some(X86Feature::BMI)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("reset lowest set bit → BLSR (BMI)"),
    );

    // --- ADX patterns ---
    rules.push(
        rule!(
            "adx_adcx",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::ADCX,
            1,
            6850,
            Some(X86Feature::ADX)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("add with carry (CF-only) → ADCX (ADX)"),
    );
    rules.push(
        rule!(
            "adx_adox",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::ADOX,
            1,
            6849,
            Some(X86Feature::ADX)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("add with carry (OF-only) → ADOX (ADX)"),
    );

    // ========================================================================
    // Priority: 4999–3000  — SSE4.x / SSSE3 / SSE3 patterns
    // ========================================================================

    // --- SSE4.2 ---
    rules.push(
        rule!(
            "sse42_crc32b",
            Opcode::Xor,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::CRC32B,
            1,
            4999,
            Some(X86Feature::SSE42)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("crc32 byte → CRC32B (SSE4.2)"),
    );

    // --- SSE4.1 min/max ---
    rules.push(
        rule!(
            "sse41_pminsb",
            Opcode::ICmp,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::PMINSB,
            1,
            4980,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I8),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("i8 signed min → PMINSB (SSE4.1)"),
    );
    rules.push(
        rule!(
            "sse41_pmaxsb",
            Opcode::ICmp,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::PMAXSB,
            1,
            4979,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I8),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("i8 signed max → PMAXSB (SSE4.1)"),
    );
    rules.push(
        rule!(
            "sse41_pminsd",
            Opcode::ICmp,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PMINSD,
            1,
            4978,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 signed min → PMINSD (SSE4.1)"),
    );
    rules.push(
        rule!(
            "sse41_pmaxsd",
            Opcode::ICmp,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PMAXSD,
            1,
            4977,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 signed max → PMAXSD (SSE4.1)"),
    );
    rules.push(
        rule!(
            "sse41_pminud",
            Opcode::ICmp,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PMINUD,
            1,
            4976,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 unsigned min → PMINUD (SSE4.1)"),
    );
    rules.push(
        rule!(
            "sse41_pmaxud",
            Opcode::ICmp,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PMAXUD,
            1,
            4975,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 unsigned max → PMAXUD (SSE4.1)"),
    );

    // --- SSE4.1 mul ---
    rules.push(
        rule!(
            "sse41_pmulld",
            Opcode::Mul,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PMULLD,
            1,
            4970,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 packed mul → PMULLD (SSE4.1)"),
    );

    // --- SSE4.1 blend ---
    rules.push(
        rule!(
            "sse41_blendps",
            Opcode::Select,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::BLENDPS,
            1,
            4960,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("f32 blend → BLENDPS (SSE4.1)"),
    );
    rules.push(
        rule!(
            "sse41_blendpd",
            Opcode::Select,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::BLENDPD,
            1,
            4959,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("f64 blend → BLENDPD (SSE4.1)"),
    );

    // --- SSE4.1 extract/insert ---
    rules.push(
        rule!(
            "sse41_extractps",
            Opcode::ExtractElement,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::EXTRACTPS,
            1,
            4950,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("extract f32 lane → EXTRACTPS (SSE4.1)"),
    );
    rules.push(
        rule!(
            "sse41_insertps",
            Opcode::InsertElement,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::INSERTPS,
            1,
            4949,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("insert f32 lane → INSERTPS (SSE4.1)"),
    );

    // --- SSSE3 ---
    rules.push(
        rule!(
            "ssse3_phaddw",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::PHADDW,
            1,
            4900,
            Some(X86Feature::SSSE3)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I16),
            TypeConstraint::Int(IntWidth::I16),
        ])
        .with_desc("i16 horizontal add → PHADDW (SSSE3)"),
    );
    rules.push(
        rule!(
            "ssse3_phaddd",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PHADDD,
            1,
            4899,
            Some(X86Feature::SSSE3)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 horizontal add → PHADDD (SSSE3)"),
    );
    rules.push(
        rule!(
            "ssse3_phsubw",
            Opcode::Sub,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::PHSUBW,
            1,
            4898,
            Some(X86Feature::SSSE3)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I16),
            TypeConstraint::Int(IntWidth::I16),
        ])
        .with_desc("i16 horizontal sub → PHSUBW (SSSE3)"),
    );
    rules.push(
        rule!(
            "ssse3_phsubd",
            Opcode::Sub,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PHSUBD,
            1,
            4897,
            Some(X86Feature::SSSE3)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 horizontal sub → PHSUBD (SSSE3)"),
    );
    rules.push(
        rule!(
            "ssse3_pabsb",
            Opcode::Sub,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::PABSB,
            1,
            4896,
            Some(X86Feature::SSSE3)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I8)])
        .with_desc("i8 abs → PABSB (SSSE3)"),
    );
    rules.push(
        rule!(
            "ssse3_pabsw",
            Opcode::Sub,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::PABSW,
            1,
            4895,
            Some(X86Feature::SSSE3)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I16)])
        .with_desc("i16 abs → PABSW (SSSE3)"),
    );
    rules.push(
        rule!(
            "ssse3_pabsd",
            Opcode::Sub,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PABSD,
            1,
            4894,
            Some(X86Feature::SSSE3)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32 abs → PABSD (SSSE3)"),
    );

    // --- SSE3 horizontal ---
    rules.push(
        rule!(
            "sse3_haddps",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::HADDPS,
            1,
            4850,
            Some(X86Feature::SSE3)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 horizontal add → HADDPS (SSE3)"),
    );
    rules.push(
        rule!(
            "sse3_haddpd",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::HADDPD,
            1,
            4849,
            Some(X86Feature::SSE3)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 horizontal add → HADDPD (SSE3)"),
    );
    rules.push(
        rule!(
            "sse3_hsubps",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::HSUBPS,
            1,
            4848,
            Some(X86Feature::SSE3)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 horizontal sub → HSUBPS (SSE3)"),
    );
    rules.push(
        rule!(
            "sse3_hsubpd",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::HSUBPD,
            1,
            4847,
            Some(X86Feature::SSE3)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 horizontal sub → HSUBPD (SSE3)"),
    );
    rules.push(
        rule!(
            "sse3_addsubps",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::ADDSUBPS,
            1,
            4846,
            Some(X86Feature::SSE3)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 add-sub → ADDSUBPS (SSE3)"),
    );

    // ========================================================================
    // Priority: 2999–2000  — SSE2 / SSE patterns
    // ========================================================================

    // --- SSE2 Scalar FP ---
    rules.push(
        rule!(
            "sse2_addss",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::ADDSS,
            1,
            2999,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 add scalar → ADDSS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_addsd",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::ADDSD,
            1,
            2998,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 add scalar → ADDSD (SSE2)"),
    );
    rules.push(
        rule!(
            "sse2_subss",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::SUBSS,
            1,
            2997,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 sub scalar → SUBSS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_subsd",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::SUBSD,
            1,
            2996,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 sub scalar → SUBSD (SSE2)"),
    );
    rules.push(
        rule!(
            "sse2_mulss",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::MULSS,
            1,
            2995,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 mul scalar → MULSS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_mulsd",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::MULSD,
            1,
            2994,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 mul scalar → MULSD (SSE2)"),
    );
    rules.push(
        rule!(
            "sse2_divss",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::DIVSS,
            1,
            2993,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 div scalar → DIVSS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_divsd",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::DIVSD,
            1,
            2992,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 div scalar → DIVSD (SSE2)"),
    );

    // --- SSE2 Packed FP ---
    rules.push(
        rule!(
            "sse2_addps",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::ADDPS,
            1,
            2990,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 add packed → ADDPS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_addpd",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::ADDPD,
            1,
            2989,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 add packed → ADDPD (SSE2)"),
    );
    rules.push(
        rule!(
            "sse2_subps",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::SUBPS,
            1,
            2988,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 sub packed → SUBPS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_subpd",
            Opcode::FSub,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::SUBPD,
            1,
            2987,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 sub packed → SUBPD (SSE2)"),
    );
    rules.push(
        rule!(
            "sse2_mulps",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::MULPS,
            1,
            2986,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 mul packed → MULPS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_mulpd",
            Opcode::FMul,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::MULPD,
            1,
            2985,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 mul packed → MULPD (SSE2)"),
    );
    rules.push(
        rule!(
            "sse2_divps",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::DIVPS,
            1,
            2984,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 div packed → DIVPS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_divpd",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::DIVPD,
            1,
            2983,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 div packed → DIVPD (SSE2)"),
    );

    // --- SSE2 Bitwise ---
    rules.push(
        rule!(
            "sse2_andps",
            Opcode::And,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::ANDPS,
            1,
            2980,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 and → ANDPS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_orps",
            Opcode::Or,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::ORPS,
            1,
            2979,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 or → ORPS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_xorps",
            Opcode::Xor,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::XORPS,
            1,
            2978,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 xor → XORPS (SSE)"),
    );

    // --- SSE2 Scalar FP compare ---
    rules.push(
        rule!(
            "sse2_cmpss",
            Opcode::FCmp,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::CMPSS,
            1,
            2970,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fcmp f32 → CMPSS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_cmpsd",
            Opcode::FCmp,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::CMPSD,
            1,
            2969,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("fcmp f64 → CMPSD (SSE2)"),
    );

    // --- SSE2 Scalar min/max ---
    rules.push(
        rule!(
            "sse2_minss",
            Opcode::FCmp,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::MINSS,
            1,
            2965,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 min scalar → MINSS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_maxss",
            Opcode::FCmp,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::MAXSS,
            1,
            2964,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 max scalar → MAXSS (SSE)"),
    );

    // --- SSE2 sqrt ---
    rules.push(
        rule!(
            "sse2_sqrtss",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::SQRTSS,
            2,
            2960,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32 sqrt scalar → SQRTSS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_sqrtsd",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::SQRTSD,
            2,
            2959,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F64)])
        .with_desc("f64 sqrt scalar → SQRTSD (SSE2)"),
    );
    rules.push(
        rule!(
            "sse2_sqrtps",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::SQRTPS,
            2,
            2958,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32 sqrt packed → SQRTPS (SSE)"),
    );

    // --- SSE2 convert ---
    rules.push(
        rule!(
            "sse2_cvtsi2ss",
            Opcode::SIToFP,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::CVTSI2SS,
            1,
            2950,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32→f32 scalar → CVTSI2SS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_cvtsi2sd",
            Opcode::SIToFP,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::CVTSI2SD,
            1,
            2949,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32→f64 scalar → CVTSI2SD (SSE2)"),
    );
    rules.push(
        rule!(
            "sse2_cvtss2si",
            Opcode::FPToSI,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::CVTSS2SI,
            1,
            2948,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32→i32 scalar → CVTSS2SI (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_cvtsd2si",
            Opcode::FPToSI,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::CVTSD2SI,
            1,
            2947,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F64)])
        .with_desc("f64→i32 scalar → CVTSD2SI (SSE2)"),
    );

    // --- SSE2 trunc convert ---
    rules.push(
        rule!(
            "sse2_cvttss2si",
            Opcode::FPToSI,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::CVTTSS2SI,
            1,
            2945,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32→i32 trunc → CVTTSS2SI (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_cvttsd2si",
            Opcode::FPToSI,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::CVTTSD2SI,
            1,
            2944,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F64)])
        .with_desc("f64→i32 trunc → CVTTSD2SI (SSE2)"),
    );

    // --- SSE2 inter-convert ---
    rules.push(
        rule!(
            "sse2_cvtss2sd",
            Opcode::FPExt,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::CVTSS2SD,
            1,
            2940,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32→f64 → CVTSS2SD (SSE2)"),
    );
    rules.push(
        rule!(
            "sse2_cvtsd2ss",
            Opcode::FPTrunc,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::CVTSD2SS,
            1,
            2939,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F64)])
        .with_desc("f64→f32 → CVTSD2SS (SSE2)"),
    );

    // --- SSE2 shuffle ---
    rules.push(
        rule!(
            "sse2_shufps",
            Opcode::ShuffleVector,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::SHUFPS,
            1,
            2930,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("f32 shuffle → SHUFPS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_shufpd",
            Opcode::ShuffleVector,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::SHUFPD,
            1,
            2929,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("f64 shuffle → SHUFPD (SSE2)"),
    );
    rules.push(
        rule!(
            "sse2_pshufd",
            Opcode::ShuffleVector,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PSHUFD,
            1,
            2928,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("i32 shuffle → PSHUFD (SSE2)"),
    );

    // --- SSE2 unpack ---
    rules.push(
        rule!(
            "sse2_unpcklps",
            Opcode::ShuffleVector,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::UNPCKLPS,
            1,
            2920,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 unpack low → UNPCKLPS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_unpckhps",
            Opcode::ShuffleVector,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::UNPCKHPS,
            1,
            2919,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 unpack high → UNPCKHPS (SSE)"),
    );

    // --- SSE2 integer unpack ---
    rules.push(
        rule!(
            "sse2_punpcklbw",
            Opcode::ShuffleVector,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::PUNPCKLBW,
            1,
            2910,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I8),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("i8 unpack low → PUNPCKLBW (SSE2)"),
    );
    rules.push(
        rule!(
            "sse2_punpcklwd",
            Opcode::ShuffleVector,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::PUNPCKLWD,
            1,
            2909,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I16),
            TypeConstraint::Int(IntWidth::I16),
        ])
        .with_desc("i16 unpack low → PUNPCKLWD (SSE2)"),
    );
    rules.push(
        rule!(
            "sse2_punpckldq",
            Opcode::ShuffleVector,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PUNPCKLDQ,
            1,
            2908,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 unpack low → PUNPCKLDQ (SSE2)"),
    );

    // --- SSE2 rcp/rsqrt ---
    rules.push(
        rule!(
            "sse2_rcpps",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::RCPPS,
            2,
            2900,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32 reciprocal → RCPPS (SSE)"),
    );
    rules.push(
        rule!(
            "sse2_rsqrtps",
            Opcode::FDiv,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::RSQRTPS,
            2,
            2899,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32 rsqrt → RSQRTPS (SSE)"),
    );

    // --- SSE2 integer compare ---
    rules.push(
        rule!(
            "sse2_pcmpeqq",
            Opcode::ICmp,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::PCMPEQQ,
            1,
            2890,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I64),
            TypeConstraint::Int(IntWidth::I64),
        ])
        .with_desc("i64 cmpeq → PCMPEQQ (SSE4.1)"),
    );

    // ========================================================================
    // Priority: 1999–1000  — Base x86‑64 / i686 arithmetic patterns
    // ========================================================================

    // --- Integer Arithmetic (all widths) ---
    for width in &[IntWidth::I8, IntWidth::I16, IntWidth::I32, IntWidth::I64] {
        let w = *width;
        // Add
        rules.push(
            ISelPatternRule::new(
                &format!("add_{}", w),
                Opcode::Add,
                TypeConstraint::Int(w),
                X86Opcode::ADD,
                1,
                1990 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w), TypeConstraint::Int(w)])
            .commutative()
            .with_desc(&format!("{} add → ADD", w)),
        );

        // Sub
        rules.push(
            ISelPatternRule::new(
                &format!("sub_{}", w),
                Opcode::Sub,
                TypeConstraint::Int(w),
                X86Opcode::SUB,
                1,
                1980 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w), TypeConstraint::Int(w)])
            .with_desc(&format!("{} sub → SUB", w)),
        );

        // IMUL (signed)
        rules.push(
            ISelPatternRule::new(
                &format!("imul_{}", w),
                Opcode::Mul,
                TypeConstraint::Int(w),
                X86Opcode::IMUL,
                3,
                1970 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w), TypeConstraint::Int(w)])
            .commutative()
            .with_desc(&format!("{} signed mul → IMUL", w)),
        );

        // DIV (unsigned)
        rules.push(
            ISelPatternRule::new(
                &format!("div_{}", w),
                Opcode::UDiv,
                TypeConstraint::Int(w),
                X86Opcode::DIV,
                20,
                1960 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w), TypeConstraint::Int(w)])
            .with_desc(&format!("{} unsigned div → DIV", w)),
        );

        // IDIV (signed)
        rules.push(
            ISelPatternRule::new(
                &format!("idiv_{}", w),
                Opcode::SDiv,
                TypeConstraint::Int(w),
                X86Opcode::IDIV,
                20,
                1950 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w), TypeConstraint::Int(w)])
            .with_desc(&format!("{} signed div → IDIV", w)),
        );

        // And
        rules.push(
            ISelPatternRule::new(
                &format!("and_{}", w),
                Opcode::And,
                TypeConstraint::Int(w),
                X86Opcode::AND,
                1,
                1940 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w), TypeConstraint::Int(w)])
            .commutative()
            .with_desc(&format!("{} and → AND", w)),
        );

        // Or
        rules.push(
            ISelPatternRule::new(
                &format!("or_{}", w),
                Opcode::Or,
                TypeConstraint::Int(w),
                X86Opcode::OR,
                1,
                1930 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w), TypeConstraint::Int(w)])
            .commutative()
            .with_desc(&format!("{} or → OR", w)),
        );

        // Xor
        rules.push(
            ISelPatternRule::new(
                &format!("xor_{}", w),
                Opcode::Xor,
                TypeConstraint::Int(w),
                X86Opcode::XOR,
                1,
                1920 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w), TypeConstraint::Int(w)])
            .commutative()
            .with_desc(&format!("{} xor → XOR", w)),
        );

        // Shifts (all three kinds)
        rules.push(
            ISelPatternRule::new(
                &format!("shl_{}", w),
                Opcode::Shl,
                TypeConstraint::Int(w),
                X86Opcode::SHL,
                1,
                1910 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![
                TypeConstraint::Int(w),
                TypeConstraint::Int(IntWidth::I8),
            ])
            .with_desc(&format!("{} shift left → SHL", w)),
        );

        rules.push(
            ISelPatternRule::new(
                &format!("shr_{}", w),
                Opcode::LShr,
                TypeConstraint::Int(w),
                X86Opcode::SHR,
                1,
                1900 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![
                TypeConstraint::Int(w),
                TypeConstraint::Int(IntWidth::I8),
            ])
            .with_desc(&format!("{} logical shift right → SHR", w)),
        );

        rules.push(
            ISelPatternRule::new(
                &format!("sar_{}", w),
                Opcode::AShr,
                TypeConstraint::Int(w),
                X86Opcode::SAR,
                1,
                1890 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![
                TypeConstraint::Int(w),
                TypeConstraint::Int(IntWidth::I8),
            ])
            .with_desc(&format!("{} arithmetic shift right → SAR", w)),
        );

        // Rotates
        rules.push(
            ISelPatternRule::new(
                &format!("rol_{}", w),
                Opcode::Shl,
                TypeConstraint::Int(w),
                X86Opcode::ROL,
                1,
                1880 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![
                TypeConstraint::Int(w),
                TypeConstraint::Int(IntWidth::I8),
            ])
            .with_desc(&format!("{} rotate left → ROL", w)),
        );

        rules.push(
            ISelPatternRule::new(
                &format!("ror_{}", w),
                Opcode::LShr,
                TypeConstraint::Int(w),
                X86Opcode::ROR,
                1,
                1870 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![
                TypeConstraint::Int(w),
                TypeConstraint::Int(IntWidth::I8),
            ])
            .with_desc(&format!("{} rotate right → ROR", w)),
        );

        // INC / DEC
        rules.push(
            ISelPatternRule::new(
                &format!("inc_{}", w),
                Opcode::Add,
                TypeConstraint::Int(w),
                X86Opcode::INC,
                1,
                1860 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w)])
            .with_desc(&format!("{} inc → INC", w)),
        );

        rules.push(
            ISelPatternRule::new(
                &format!("dec_{}", w),
                Opcode::Sub,
                TypeConstraint::Int(w),
                X86Opcode::DEC,
                1,
                1850 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w)])
            .with_desc(&format!("{} dec → DEC", w)),
        );

        // NEG / NOT
        rules.push(
            ISelPatternRule::new(
                &format!("neg_{}", w),
                Opcode::Sub,
                TypeConstraint::Int(w),
                X86Opcode::NEG,
                1,
                1840 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w)])
            .with_desc(&format!("{} neg → NEG", w)),
        );

        rules.push(
            ISelPatternRule::new(
                &format!("not_{}", w),
                Opcode::Xor,
                TypeConstraint::Int(w),
                X86Opcode::NOT,
                1,
                1830 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w)])
            .with_desc(&format!("{} not → NOT", w)),
        );
    }

    // --- ADC / SBB (with carry) ---
    for width in &[IntWidth::I32, IntWidth::I64] {
        let w = *width;
        rules.push(
            ISelPatternRule::new(
                &format!("adc_{}", w),
                Opcode::Add,
                TypeConstraint::Int(w),
                X86Opcode::ADC,
                1,
                1820,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w), TypeConstraint::Int(w)])
            .with_desc(&format!("{} add with carry → ADC", w)),
        );

        rules.push(
            ISelPatternRule::new(
                &format!("sbb_{}", w),
                Opcode::Sub,
                TypeConstraint::Int(w),
                X86Opcode::SBB,
                1,
                1819,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w), TypeConstraint::Int(w)])
            .with_desc(&format!("{} sub with borrow → SBB", w)),
        );
    }

    // ========================================================================
    // Comparison operations → condition codes (1999–1000)
    // ========================================================================

    // CMP (integer compare, sets flags for subsequent Jcc/SETcc/CMOVcc)
    for width in &[IntWidth::I8, IntWidth::I16, IntWidth::I32, IntWidth::I64] {
        let w = *width;
        rules.push(
            ISelPatternRule::new(
                &format!("cmp_{}", w),
                Opcode::ICmp,
                TypeConstraint::Int(w),
                X86Opcode::CMP,
                1,
                1790 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w), TypeConstraint::Int(w)])
            .with_desc(&format!("{} icmp → CMP", w)),
        );
    }

    // TEST (bitwise test, sets ZF)
    for width in &[IntWidth::I32, IntWidth::I64] {
        let w = *width;
        rules.push(
            ISelPatternRule::new(
                &format!("test_{}", w),
                Opcode::And,
                TypeConstraint::Int(w),
                X86Opcode::TEST,
                1,
                1780,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w), TypeConstraint::Int(w)])
            .with_desc(&format!("{} test → TEST", w)),
        );
    }

    // UCOMISS / UCOMISD (unordered float compare, sets EFLAGS)
    rules.push(
        rule!(
            "ucomiss",
            Opcode::FCmp,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::UCOMISS,
            1,
            1770,
            Some(X86Feature::SSE)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("fcmp f32 → UCOMISS (SSE)"),
    );
    rules.push(
        rule!(
            "ucomisd",
            Opcode::FCmp,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::UCOMISD,
            1,
            1769,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("fcmp f64 → UCOMISD (SSE2)"),
    );

    // ========================================================================
    // Memory operations (load/store/alloca) with all addressing modes
    // ========================================================================

    // MOV (load) — general-purpose register
    for width in &[IntWidth::I8, IntWidth::I16, IntWidth::I32, IntWidth::I64] {
        let w = *width;
        rules.push(
            ISelPatternRule::new(
                &format!("mov_load_{}", w),
                Opcode::Load,
                TypeConstraint::Int(w),
                X86Opcode::MOV,
                2,
                1750 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Pointer])
            .with_desc(&format!("{} load → MOV ({})", w, w)),
        );
    }

    // MOV (store) — general-purpose register
    for width in &[IntWidth::I8, IntWidth::I16, IntWidth::I32, IntWidth::I64] {
        let w = *width;
        rules.push(
            ISelPatternRule::new(
                &format!("mov_store_{}", w),
                Opcode::Store,
                TypeConstraint::Int(w),
                X86Opcode::MOV,
                2,
                1740 + (4 - (w.bits() as u16 / 16)) * 10,
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Pointer, TypeConstraint::Int(w)])
            .with_desc(&format!("{} store → MOV ({})", w, w)),
        );
    }

    // SSE scalar load/store
    rules.push(
        rule!(
            "movss_load",
            Opcode::Load,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::MOVSS,
            2,
            1720,
            Some(X86Feature::SSE)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("f32 load → MOVSS (SSE)"),
    );
    rules.push(
        rule!(
            "movsd_load",
            Opcode::Load,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::MOVSD,
            2,
            1719,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("f64 load → MOVSD (SSE2)"),
    );
    rules.push(
        rule!(
            "movss_store",
            Opcode::Store,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::MOVSS,
            2,
            1718,
            Some(X86Feature::SSE)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("f32 store → MOVSS (SSE)"),
    );
    rules.push(
        rule!(
            "movsd_store",
            Opcode::Store,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::MOVSD,
            2,
            1717,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("f64 store → MOVSD (SSE2)"),
    );

    // MOVD / MOVQ (XMM <-> GPR)
    rules.push(
        rule!(
            "movd_load",
            Opcode::Load,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::MOVD,
            2,
            1716,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("i32 xmm load → MOVD (SSE2)"),
    );
    rules.push(
        rule!(
            "movq_load",
            Opcode::Load,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::MOVQ,
            2,
            1715,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("i64 xmm load → MOVQ (SSE2)"),
    );

    // MOVSX (sign-extending load)
    rules.push(
        rule!(
            "movsx_i8_i32",
            Opcode::SExt,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::MOVSX,
            2,
            1710,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I8)])
        .with_desc("i8→i32 sign-extend → MOVSX"),
    );
    rules.push(
        rule!(
            "movsx_i16_i32",
            Opcode::SExt,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::MOVSX,
            2,
            1709,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I16)])
        .with_desc("i16→i32 sign-extend → MOVSX"),
    );

    // MOVZX (zero-extending load)
    rules.push(
        rule!(
            "movzx_i8_i32",
            Opcode::ZExt,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::MOVZX,
            2,
            1705,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I8)])
        .with_desc("i8→i32 zero-extend → MOVZX"),
    );
    rules.push(
        rule!(
            "movzx_i16_i32",
            Opcode::ZExt,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::MOVZX,
            2,
            1704,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I16)])
        .with_desc("i16→i32 zero-extend → MOVZX"),
    );

    // MOVZX 64-bit variants
    rules.push(
        rule!(
            "movzx_i8_i64",
            Opcode::ZExt,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::MOVZX,
            2,
            1703,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I8)])
        .with_desc("i8→i64 zero-extend → MOVZX")
        .only64(),
    );
    rules.push(
        rule!(
            "movzx_i16_i64",
            Opcode::ZExt,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::MOVZX,
            2,
            1702,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I16)])
        .with_desc("i16→i64 zero-extend → MOVZX")
        .only64(),
    );
    rules.push(
        rule!(
            "movzx_i32_i64",
            Opcode::ZExt,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::MOVZX,
            2,
            1701,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32→i64 zero-extend → MOVZX")
        .only64(),
    );

    // MOVSXD (64-bit sign extend from 32-bit, only in 64-bit mode)
    rules.push(
        rule!(
            "movsxd_i32_i64",
            Opcode::SExt,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::MOVSXD,
            2,
            1700,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32→i64 sign-extend → MOVSXD")
        .only64(),
    );

    // ALLOCA (stack allocation)
    rules.push(
        rule!(
            "alloca",
            Opcode::Alloca,
            TypeConstraint::Pointer,
            X86Opcode::SUB,
            1,
            1690,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("alloca → SUB (RSP)")
        .only64(),
    );

    // ========================================================================
    // Control flow (br, switch, call, ret, invoke)
    // ========================================================================

    // JMP (unconditional branch)
    rules.push(
        rule!(
            "br_jmp",
            Opcode::Br,
            TypeConstraint::Any,
            X86Opcode::JMP,
            1,
            1680,
            Some(X86Feature::X86)
        )
        .conditional()
        .with_desc("uncond br → JMP"),
    );

    // Conditional branches — all 16 Jcc variants
    let jcc_rules: [(&str, X86Opcode, u8); 16] = [
        ("br_jo", X86Opcode::JO, 0),
        ("br_jno", X86Opcode::JNO, 1),
        ("br_jb", X86Opcode::JB, 2),
        ("br_jae", X86Opcode::JAE, 3),
        ("br_je", X86Opcode::JE, 4),
        ("br_jne", X86Opcode::JNE, 5),
        ("br_jbe", X86Opcode::JBE, 6),
        ("br_ja", X86Opcode::JA, 7),
        ("br_js", X86Opcode::JS, 8),
        ("br_jns", X86Opcode::JNS, 9),
        ("br_jp", X86Opcode::JP, 10),
        ("br_jnp", X86Opcode::JNP, 11),
        ("br_jl", X86Opcode::JL, 12),
        ("br_jge", X86Opcode::JGE, 13),
        ("br_jle", X86Opcode::JLE, 14),
        ("br_jg", X86Opcode::JG, 15),
    ];
    for (name, opcode, _pred) in &jcc_rules {
        rules.push(
            ISelPatternRule::new(
                name,
                Opcode::Br,
                TypeConstraint::Any,
                *opcode,
                1,
                1679,
                Some(X86Feature::X86),
            )
            .conditional()
            .with_desc(&format!("cond br → {}", name)),
        );
    }

    // RET
    rules.push(
        rule!(
            "ret_void",
            Opcode::Ret,
            TypeConstraint::Any,
            X86Opcode::RET,
            1,
            1670,
            Some(X86Feature::X86)
        )
        .with_desc("ret void → RET"),
    );
    rules.push(
        rule!(
            "ret_val_i32",
            Opcode::Ret,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::RET,
            1,
            1669,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("ret i32 → RET"),
    );

    // CALL
    rules.push(
        rule!(
            "call",
            Opcode::Call,
            TypeConstraint::Any,
            X86Opcode::CALL,
            1,
            1660,
            Some(X86Feature::X86)
        )
        .with_desc("call → CALL"),
    );

    // SWITCH (indirect jump via table)
    rules.push(
        rule!(
            "switch_jmp_indirect",
            Opcode::Switch,
            TypeConstraint::Any,
            X86Opcode::JMP,
            3,
            1650,
            Some(X86Feature::X86)
        )
        .with_desc("switch → JMP (indirect)"),
    );

    // ========================================================================
    // Phi / Select lowering
    // ========================================================================

    // PHI (lowered to MOV in predecessor blocks during SSA destruction)
    rules.push(
        rule!(
            "phi_mov",
            Opcode::Phi,
            TypeConstraint::Any,
            X86Opcode::MOV,
            0,
            1640,
            Some(X86Feature::X86)
        )
        .with_desc("phi → MOV (placeholder, SSA destruction)"),
    );

    // SELECT → CMOVcc
    let select_rules: [(&str, X86Opcode); 16] = [
        ("select_cmovo", X86Opcode::CMOVO),
        ("select_cmovno", X86Opcode::CMOVNO),
        ("select_cmovb", X86Opcode::CMOVB),
        ("select_cmovae", X86Opcode::CMOVAE),
        ("select_cmove", X86Opcode::CMOVE),
        ("select_cmovne", X86Opcode::CMOVNE),
        ("select_cmovbe", X86Opcode::CMOVBE),
        ("select_cmova", X86Opcode::CMOVA),
        ("select_cmovs", X86Opcode::CMOVS),
        ("select_cmovns", X86Opcode::CMOVNS),
        ("select_cmovp", X86Opcode::CMOVP),
        ("select_cmovnp", X86Opcode::CMOVNP),
        ("select_cmovl", X86Opcode::CMOVL),
        ("select_cmovge", X86Opcode::CMOVGE),
        ("select_cmovle", X86Opcode::CMOVLE),
        ("select_cmovg", X86Opcode::CMOVG),
    ];
    for (name, opcode) in &select_rules {
        rules.push(
            ISelPatternRule::new(
                name,
                Opcode::Select,
                TypeConstraint::AnyInt,
                *opcode,
                2,
                1630,
                Some(X86Feature::CMOV),
            )
            .conditional()
            .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
            .with_desc(&format!("select → {}", name)),
        );
    }

    // SETcc (conditional set byte, icmp result → i8)
    let setcc_rules: [(&str, X86Opcode); 16] = [
        ("setcc_o", X86Opcode::SETO),
        ("setcc_no", X86Opcode::SETNO),
        ("setcc_b", X86Opcode::SETB),
        ("setcc_ae", X86Opcode::SETAE),
        ("setcc_e", X86Opcode::SETE),
        ("setcc_ne", X86Opcode::SETNE),
        ("setcc_be", X86Opcode::SETBE),
        ("setcc_a", X86Opcode::SETA),
        ("setcc_s", X86Opcode::SETS),
        ("setcc_ns", X86Opcode::SETNS),
        ("setcc_p", X86Opcode::SETP),
        ("setcc_np", X86Opcode::SETNP),
        ("setcc_l", X86Opcode::SETL),
        ("setcc_ge", X86Opcode::SETGE),
        ("setcc_le", X86Opcode::SETLE),
        ("setcc_g", X86Opcode::SETG),
    ];
    for (name, opcode) in &setcc_rules {
        rules.push(
            ISelPatternRule::new(
                name,
                Opcode::ICmp,
                TypeConstraint::Int(IntWidth::I8),
                *opcode,
                1,
                1620,
                Some(X86Feature::X86),
            )
            .conditional()
            .with_desc(&format!("icmp → {}", name)),
        );
    }

    // ========================================================================
    // Sign/zero extension patterns
    // ========================================================================

    rules.push(
        rule!(
            "zext_i8_i16",
            Opcode::ZExt,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::MOVZX,
            1,
            1610,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I8)])
        .with_desc("i8→i16 zext → MOVZX"),
    );
    rules.push(
        rule!(
            "zext_i8_i32",
            Opcode::ZExt,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::MOVZX,
            1,
            1609,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I8)])
        .with_desc("i8→i32 zext → MOVZX"),
    );
    rules.push(
        rule!(
            "zext_i16_i32",
            Opcode::ZExt,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::MOVZX,
            1,
            1608,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I16)])
        .with_desc("i16→i32 zext → MOVZX"),
    );
    rules.push(
        rule!(
            "zext_i8_i64",
            Opcode::ZExt,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::MOVZX,
            1,
            1607,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I8)])
        .with_desc("i8→i64 zext → MOVZX")
        .only64(),
    );
    rules.push(
        rule!(
            "zext_i16_i64",
            Opcode::ZExt,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::MOVZX,
            1,
            1606,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I16)])
        .with_desc("i16→i64 zext → MOVZX")
        .only64(),
    );

    rules.push(
        rule!(
            "sext_i8_i16",
            Opcode::SExt,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::MOVSX,
            1,
            1605,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I8)])
        .with_desc("i8→i16 sext → MOVSX"),
    );
    rules.push(
        rule!(
            "sext_i8_i32",
            Opcode::SExt,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::MOVSX,
            1,
            1604,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I8)])
        .with_desc("i8→i32 sext → MOVSX"),
    );
    rules.push(
        rule!(
            "sext_i16_i32",
            Opcode::SExt,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::MOVSX,
            1,
            1603,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I16)])
        .with_desc("i16→i32 sext → MOVSX"),
    );
    rules.push(
        rule!(
            "sext_i8_i64",
            Opcode::SExt,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::MOVSX,
            1,
            1602,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I8)])
        .with_desc("i8→i64 sext → MOVSX")
        .only64(),
    );
    rules.push(
        rule!(
            "sext_i16_i64",
            Opcode::SExt,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::MOVSX,
            1,
            1601,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I16)])
        .with_desc("i16→i64 sext → MOVSX")
        .only64(),
    );

    // ========================================================================
    // Truncation patterns
    // ========================================================================

    rules.push(
        rule!(
            "trunc_i16_i8",
            Opcode::Trunc,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::MOV,
            1,
            1600,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I16)])
        .with_desc("i16→i8 trunc → MOV (AL)"),
    );
    rules.push(
        rule!(
            "trunc_i32_i8",
            Opcode::Trunc,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::MOV,
            1,
            1599,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32→i8 trunc → MOV (AL)"),
    );
    rules.push(
        rule!(
            "trunc_i32_i16",
            Opcode::Trunc,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::MOV,
            1,
            1598,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32→i16 trunc → MOV (AX)"),
    );
    rules.push(
        rule!(
            "trunc_i64_i8",
            Opcode::Trunc,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::MOV,
            1,
            1597,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I64)])
        .with_desc("i64→i8 trunc → MOV (AL)")
        .only64(),
    );
    rules.push(
        rule!(
            "trunc_i64_i16",
            Opcode::Trunc,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::MOV,
            1,
            1596,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I64)])
        .with_desc("i64→i16 trunc → MOV (AX)")
        .only64(),
    );
    rules.push(
        rule!(
            "trunc_i64_i32",
            Opcode::Trunc,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::MOV,
            1,
            1595,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I64)])
        .with_desc("i64→i32 trunc → MOV (EAX)")
        .only64(),
    );

    // ========================================================================
    // Type conversion patterns
    // ========================================================================

    // Float → Int
    rules.push(
        rule!(
            "fptosi_f32_i32",
            Opcode::FPToSI,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::CVTTSS2SI,
            2,
            1590,
            Some(X86Feature::SSE)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32→i32 sint → CVTTSS2SI (SSE)"),
    );
    rules.push(
        rule!(
            "fptosi_f64_i32",
            Opcode::FPToSI,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::CVTTSD2SI,
            2,
            1589,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F64)])
        .with_desc("f64→i32 sint → CVTTSD2SI (SSE2)"),
    );

    rules.push(
        rule!(
            "fptoui_f32_i32",
            Opcode::FPToUI,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::CVTTSS2SI,
            2,
            1588,
            Some(X86Feature::SSE)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32→i32 uint → CVTTSS2SI (SSE)"),
    );
    rules.push(
        rule!(
            "fptoui_f64_i32",
            Opcode::FPToUI,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::CVTTSD2SI,
            2,
            1587,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F64)])
        .with_desc("f64→i32 uint → CVTTSD2SI (SSE2)"),
    );

    // Int → Float
    rules.push(
        rule!(
            "sitofp_i32_f32",
            Opcode::SIToFP,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::CVTSI2SS,
            2,
            1585,
            Some(X86Feature::SSE)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32→f32 → CVTSI2SS (SSE)"),
    );
    rules.push(
        rule!(
            "sitofp_i32_f64",
            Opcode::SIToFP,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::CVTSI2SD,
            2,
            1584,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32→f64 → CVTSI2SD (SSE2)"),
    );
    rules.push(
        rule!(
            "uitofp_i32_f32",
            Opcode::UIToFP,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::CVTSI2SS,
            2,
            1583,
            Some(X86Feature::SSE)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("ui32→f32 → CVTSI2SS (SSE)"),
    );
    rules.push(
        rule!(
            "uitofp_i32_f64",
            Opcode::UIToFP,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::CVTSI2SD,
            2,
            1582,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("ui32→f64 → CVTSI2SD (SSE2)"),
    );

    // Float ↔ Float
    rules.push(
        rule!(
            "fptrunc_f64_f32",
            Opcode::FPTrunc,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::CVTSD2SS,
            2,
            1580,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F64)])
        .with_desc("f64→f32 → CVTSD2SS (SSE2)"),
    );
    rules.push(
        rule!(
            "fpext_f32_f64",
            Opcode::FPExt,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::CVTSS2SD,
            2,
            1579,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32→f64 → CVTSS2SD (SSE2)"),
    );

    // ========================================================================
    // Ptr ↔ Int conversion
    // ========================================================================

    rules.push(
        rule!(
            "ptrtoint",
            Opcode::PtrToInt,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::MOV,
            1,
            1570,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("ptr→i64 → MOV")
        .only64(),
    );
    rules.push(
        rule!(
            "inttoptr",
            Opcode::IntToPtr,
            TypeConstraint::Pointer,
            X86Opcode::MOV,
            1,
            1569,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I64)])
        .with_desc("i64→ptr → MOV")
        .only64(),
    );

    // ========================================================================
    // GetElementPtr
    // ========================================================================

    rules.push(
        rule!(
            "gep_lea",
            Opcode::GetElementPtr,
            TypeConstraint::Pointer,
            X86Opcode::LEA,
            2,
            1560,
            Some(X86Feature::X86)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("gep → LEA"),
    );

    // ========================================================================
    // BitCast
    // ========================================================================

    rules.push(
        rule!(
            "bitcast_reg",
            Opcode::BitCast,
            TypeConstraint::Any,
            X86Opcode::MOV,
            1,
            1550,
            Some(X86Feature::X86)
        )
        .with_desc("bitcast → MOV (register copy)"),
    );

    // ========================================================================
    // Bit manipulation: BSF/BSR/POPCNT/LZCNT/TZCNT
    // ========================================================================

    rules.push(
        rule!(
            "bsf_i32",
            Opcode::LShr,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BSF,
            2,
            1540,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("bit scan forward → BSF"),
    );
    rules.push(
        rule!(
            "bsr_i32",
            Opcode::LShr,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BSR,
            2,
            1539,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("bit scan reverse → BSR"),
    );

    rules.push(
        rule!(
            "popcnt_i32",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::POPCNT32,
            2,
            1535,
            Some(X86Feature::POPCNT)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("popcount → POPCNT (POPCNT)"),
    );
    rules.push(
        rule!(
            "popcnt_i64",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::POPCNT64,
            2,
            1534,
            Some(X86Feature::POPCNT)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I64)])
        .with_desc("popcount → POPCNT (POPCNT)")
        .only64(),
    );

    rules.push(
        rule!(
            "lzcnt_i32",
            Opcode::LShr,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::LZCNT32,
            2,
            1530,
            Some(X86Feature::LZCNT)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("leading zero count → LZCNT (LZCNT)"),
    );
    rules.push(
        rule!(
            "tzcnt_i32",
            Opcode::LShr,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::TZCNT32,
            2,
            1529,
            Some(X86Feature::BMI)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("trailing zero count → TZCNT (BMI)"),
    );

    // ========================================================================
    // Byte swap
    // ========================================================================

    rules.push(
        rule!(
            "bswap_i32",
            Opcode::LShr,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::BSWAP,
            1,
            1520,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("byte swap → BSWAP"),
    );
    rules.push(
        rule!(
            "bswap_i64",
            Opcode::LShr,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::BSWAP,
            1,
            1519,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I64)])
        .with_desc("byte swap → BSWAP")
        .only64(),
    );

    // ========================================================================
    // Atomic operations
    // ========================================================================

    // XCHG (atomic exchange)
    rules.push(
        rule!(
            "xchg_i32",
            Opcode::AtomicRMW,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::XCHG,
            2,
            1500,
            Some(X86Feature::X86)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("atomic xchg → XCHG (lock implied)"),
    );
    rules.push(
        rule!(
            "xchg_i64",
            Opcode::AtomicRMW,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::XCHG,
            2,
            1499,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I64),
        ])
        .with_desc("atomic xchg → XCHG (lock implied)")
        .only64(),
    );

    // CMPXCHG (compare-and-swap)
    rules.push(
        rule!(
            "cmpxchg_i32",
            Opcode::CmpXchg,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::XCHG,
            3,
            1495,
            Some(X86Feature::X86)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("cmpxchg → XCHG (compare-and-swap)"),
    );
    rules.push(
        rule!(
            "cmpxchg_i64",
            Opcode::CmpXchg,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::XCHG,
            3,
            1494,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I64),
            TypeConstraint::Int(IntWidth::I64),
        ])
        .with_desc("cmpxchg → XCHG (compare-and-swap)")
        .only64(),
    );

    // Atomic add/fetch_add via LOCK ADD / LOCK XADD
    rules.push(
        rule!(
            "atomic_add_i32",
            Opcode::AtomicRMW,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::ADD,
            2,
            1490,
            Some(X86Feature::X86)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("atomicrmw add → LOCK ADD"),
    );
    rules.push(
        rule!(
            "atomic_sub_i32",
            Opcode::AtomicRMW,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::SUB,
            2,
            1489,
            Some(X86Feature::X86)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("atomicrmw sub → LOCK SUB"),
    );
    rules.push(
        rule!(
            "atomic_and_i32",
            Opcode::AtomicRMW,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::AND,
            2,
            1488,
            Some(X86Feature::X86)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("atomicrmw and → LOCK AND"),
    );
    rules.push(
        rule!(
            "atomic_or_i32",
            Opcode::AtomicRMW,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::OR,
            2,
            1487,
            Some(X86Feature::X86)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("atomicrmw or → LOCK OR"),
    );
    rules.push(
        rule!(
            "atomic_xor_i32",
            Opcode::AtomicRMW,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::XOR,
            2,
            1486,
            Some(X86Feature::X86)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("atomicrmw xor → LOCK XOR"),
    );

    // ========================================================================
    // String operations
    // ========================================================================

    // REP MOVSB (memcpy)
    rules.push(
        rule!(
            "rep_movsb",
            Opcode::Call,
            TypeConstraint::Any,
            X86Opcode::MOVSB,
            2,
            1480,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Pointer, TypeConstraint::Pointer])
        .with_desc("memcpy → REP MOVSB"),
    );

    // REP STOSB (memset)
    rules.push(
        rule!(
            "rep_stosb",
            Opcode::Call,
            TypeConstraint::Any,
            X86Opcode::STOSB,
            2,
            1479,
            Some(X86Feature::X86)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("memset → REP STOSB"),
    );

    // MOVSB / MOVSW / MOVSD / MOVSQ (width-specific string moves)
    rules.push(
        rule!(
            "movsb",
            Opcode::Load,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::MOVSB,
            2,
            1475,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("byte string load → MOVSB"),
    );
    rules.push(
        rule!(
            "movsw",
            Opcode::Load,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::MOVSW,
            2,
            1474,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("word string load → MOVSW"),
    );
    rules.push(
        rule!(
            "movsd_str",
            Opcode::Load,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::MOVSD_STR,
            2,
            1473,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("dword string load → MOVSD"),
    );
    rules.push(
        rule!(
            "movsq",
            Opcode::Load,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::MOVSQ,
            2,
            1472,
            Some(X86Feature::X86_64)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("qword string load → MOVSQ")
        .only64(),
    );

    // STOS (store string)
    rules.push(
        rule!(
            "stosb",
            Opcode::Store,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::STOSB,
            2,
            1470,
            Some(X86Feature::X86)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("byte string store → STOSB"),
    );

    // CMPS (compare string, for memcmp)
    rules.push(
        rule!(
            "cmpsb",
            Opcode::ICmp,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::CMPSB,
            2,
            1465,
            Some(X86Feature::X86)
        )
        .with_operands(vec![TypeConstraint::Pointer, TypeConstraint::Pointer])
        .with_desc("byte string compare → CMPSB"),
    );

    // ========================================================================
    // Stack operations
    // ========================================================================

    // PUSH / POP
    for width in &[IntWidth::I32, IntWidth::I64] {
        let w = *width;
        rules.push(
            ISelPatternRule::new(
                &format!("push_{}", w),
                Opcode::Store,
                TypeConstraint::Int(w),
                X86Opcode::PUSH,
                2,
                1460 + (4 - (w.bits() as u16 / 16)),
                Some(X86Feature::X86),
            )
            .with_operands(vec![TypeConstraint::Int(w)])
            .with_desc(&format!("{} push → PUSH", w)),
        );
        rules.push(
            ISelPatternRule::new(
                &format!("pop_{}", w),
                Opcode::Load,
                TypeConstraint::Int(w),
                X86Opcode::POP,
                2,
                1450 + (4 - (w.bits() as u16 / 16)),
                Some(X86Feature::X86),
            )
            .with_desc(&format!("{} pop → POP", w)),
        );
    }

    // ========================================================================
    // FLAGS save/restore
    // ========================================================================

    rules.push(
        rule!(
            "pushf",
            Opcode::Store,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PUSHF,
            2,
            1440,
            Some(X86Feature::X86)
        )
        .with_desc("flags push → PUSHF"),
    );
    rules.push(
        rule!(
            "popf",
            Opcode::Load,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::POPF,
            2,
            1439,
            Some(X86Feature::X86)
        )
        .with_desc("flags pop → POPF"),
    );

    // ========================================================================
    // Misc: NOP, INT3, UD2, PAUSE, LFENCE, MFENCE, SFENCE
    // ========================================================================

    rules.push(
        rule!(
            "nop",
            Opcode::Add,
            TypeConstraint::Any,
            X86Opcode::NOP,
            0,
            1430,
            None
        )
        .with_desc("nop → NOP"),
    );
    rules.push(
        rule!(
            "int3",
            Opcode::Call,
            TypeConstraint::Any,
            X86Opcode::INT3,
            1,
            1429,
            Some(X86Feature::X86)
        )
        .with_desc("debug trap → INT3"),
    );
    rules.push(
        rule!(
            "ud2",
            Opcode::Unreachable,
            TypeConstraint::Any,
            X86Opcode::UD2,
            1,
            1428,
            Some(X86Feature::X86)
        )
        .with_desc("unreachable → UD2"),
    );
    rules.push(
        rule!(
            "pause",
            Opcode::Add,
            TypeConstraint::Any,
            X86Opcode::PAUSE,
            1,
            1427,
            Some(X86Feature::X86)
        )
        .with_desc("spin hint → PAUSE"),
    );
    rules.push(
        rule!(
            "lfence",
            Opcode::Fence,
            TypeConstraint::Any,
            X86Opcode::LFENCE,
            1,
            1426,
            Some(X86Feature::SSE2)
        )
        .with_desc("load fence → LFENCE"),
    );
    rules.push(
        rule!(
            "mfence",
            Opcode::Fence,
            TypeConstraint::Any,
            X86Opcode::MFENCE,
            1,
            1425,
            Some(X86Feature::SSE2)
        )
        .with_desc("full fence → MFENCE"),
    );
    rules.push(
        rule!(
            "sfence",
            Opcode::Fence,
            TypeConstraint::Any,
            X86Opcode::SFENCE,
            1,
            1424,
            Some(X86Feature::SSE)
        )
        .with_desc("store fence → SFENCE"),
    );

    // ========================================================================
    // System: SYSCALL / SYSRET
    // ========================================================================

    rules.push(
        rule!(
            "syscall",
            Opcode::Call,
            TypeConstraint::Any,
            X86Opcode::SYSCALL,
            1,
            1420,
            Some(X86Feature::X86_64)
        )
        .with_desc("syscall → SYSCALL")
        .only64(),
    );
    rules.push(
        rule!(
            "sysret",
            Opcode::Ret,
            TypeConstraint::Any,
            X86Opcode::SYSRET,
            1,
            1419,
            Some(X86Feature::X86_64)
        )
        .with_desc("sysret → SYSRET")
        .only64(),
    );

    // ========================================================================
    // Cryptography: AES-NI, SHA, PCLMUL
    // ========================================================================

    rules.push(
        rule!(
            "aesenc",
            Opcode::Xor,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::AESENC,
            1,
            1400,
            Some(X86Feature::AES)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I8),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("AES encrypt round → AESENC (AES-NI)"),
    );
    rules.push(
        rule!(
            "aesdec",
            Opcode::Xor,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::AESDEC,
            1,
            1399,
            Some(X86Feature::AES)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I8),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("AES decrypt round → AESDEC (AES-NI)"),
    );
    rules.push(
        rule!(
            "aesenclast",
            Opcode::Xor,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::AESENCLAST,
            1,
            1398,
            Some(X86Feature::AES)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I8),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("AES encrypt last → AESENCLAST (AES-NI)"),
    );
    rules.push(
        rule!(
            "aesdeclast",
            Opcode::Xor,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::AESDECLAST,
            1,
            1397,
            Some(X86Feature::AES)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I8),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("AES decrypt last → AESDECLAST (AES-NI)"),
    );
    rules.push(
        rule!(
            "aesimc",
            Opcode::Xor,
            TypeConstraint::Int(IntWidth::I8),
            X86Opcode::AESIMC,
            1,
            1396,
            Some(X86Feature::AES)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I8)])
        .with_desc("AES inv mix columns → AESIMC (AES-NI)"),
    );

    rules.push(
        rule!(
            "pclmulqdq",
            Opcode::Mul,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::PCLMULQDQ,
            1,
            1390,
            Some(X86Feature::PCLMUL)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I64),
            TypeConstraint::Int(IntWidth::I64),
        ])
        .with_desc("carry-less mul → PCLMULQDQ (PCLMUL)"),
    );

    // SHA
    rules.push(
        rule!(
            "sha1rnds4",
            Opcode::Xor,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::SHA1RNDS4,
            1,
            1385,
            Some(X86Feature::SHA)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("SHA1 round → SHA1RNDS4 (SHA)"),
    );
    rules.push(
        rule!(
            "sha256rnds2",
            Opcode::Xor,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::SHA256RNDS2,
            1,
            1384,
            Some(X86Feature::SHA)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("SHA256 round → SHA256RNDS2 (SHA)"),
    );

    // ========================================================================
    // Random numbers: RDRAND / RDSEED
    // ========================================================================

    rules.push(
        rule!(
            "rdrand_i32",
            Opcode::Call,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::RDRAND32,
            2,
            1370,
            Some(X86Feature::RDRAND)
        )
        .with_desc("random i32 → RDRAND"),
    );
    rules.push(
        rule!(
            "rdrand_i64",
            Opcode::Call,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::RDRAND64,
            2,
            1369,
            Some(X86Feature::RDRAND)
        )
        .with_desc("random i64 → RDRAND")
        .only64(),
    );
    rules.push(
        rule!(
            "rdseed_i32",
            Opcode::Call,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::RDSEED32,
            2,
            1368,
            Some(X86Feature::RDSEED)
        )
        .with_desc("seed i32 → RDSEED"),
    );
    rules.push(
        rule!(
            "rdseed_i64",
            Opcode::Call,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::RDSEED64,
            2,
            1367,
            Some(X86Feature::RDSEED)
        )
        .with_desc("seed i64 → RDSEED")
        .only64(),
    );

    // ========================================================================
    // Non-temporal stores
    // ========================================================================

    rules.push(
        rule!(
            "movnti_i32",
            Opcode::Store,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::MOVNTI,
            2,
            1360,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("non-temporal store → MOVNTI (SSE2)"),
    );
    rules.push(
        rule!(
            "movntps",
            Opcode::Store,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::MOVNTPS,
            2,
            1359,
            Some(X86Feature::SSE)
        )
        .with_operands(vec![
            TypeConstraint::Pointer,
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("non-temporal f32 store → MOVNTPS (SSE)"),
    );

    // ========================================================================
    // Cache management
    // ========================================================================

    rules.push(
        rule!(
            "clflush",
            Opcode::Store,
            TypeConstraint::Pointer,
            X86Opcode::CLFLUSH,
            2,
            1350,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("cache flush → CLFLUSH (SSE2)"),
    );
    rules.push(
        rule!(
            "clflushopt",
            Opcode::Store,
            TypeConstraint::Pointer,
            X86Opcode::CLFLUSHOPT,
            2,
            1349,
            Some(X86Feature::CLFLUSHOPT)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("opt cache flush → CLFLUSHOPT"),
    );
    rules.push(
        rule!(
            "clwb",
            Opcode::Store,
            TypeConstraint::Pointer,
            X86Opcode::CLWB,
            2,
            1348,
            Some(X86Feature::CLWB)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("cache write-back → CLWB"),
    );

    // ========================================================================
    // Prefetch
    // ========================================================================

    rules.push(
        rule!(
            "prefetch_t0",
            Opcode::Load,
            TypeConstraint::Pointer,
            X86Opcode::PREFETCHT0,
            2,
            1340,
            Some(X86Feature::SSE)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("prefetch T0 → PREFETCHT0 (SSE)"),
    );
    rules.push(
        rule!(
            "prefetch_t1",
            Opcode::Load,
            TypeConstraint::Pointer,
            X86Opcode::PREFETCHT1,
            2,
            1339,
            Some(X86Feature::SSE)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("prefetch T1 → PREFETCHT1 (SSE)"),
    );
    rules.push(
        rule!(
            "prefetch_nta",
            Opcode::Load,
            TypeConstraint::Pointer,
            X86Opcode::PREFETCHNTA,
            2,
            1338,
            Some(X86Feature::SSE)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("prefetch NTA → PREFETCHNTA (SSE)"),
    );
    rules.push(
        rule!(
            "prefetchw",
            Opcode::Load,
            TypeConstraint::Pointer,
            X86Opcode::PREFETCHW,
            2,
            1337,
            Some(X86Feature::PREFETCHW)
        )
        .with_operands(vec![TypeConstraint::Pointer])
        .with_desc("prefetch for write → PREFETCHW"),
    );

    // ========================================================================
    // Unreachable
    // ========================================================================

    rules.push(
        rule!(
            "unreachable",
            Opcode::Unreachable,
            TypeConstraint::Any,
            X86Opcode::UD2,
            0,
            1330,
            Some(X86Feature::X86)
        )
        .with_desc("unreachable → UD2"),
    );

    // ========================================================================
    // Freeze
    // ========================================================================

    rules.push(
        rule!(
            "freeze",
            Opcode::Freeze,
            TypeConstraint::Any,
            X86Opcode::MOV,
            1,
            1320,
            Some(X86Feature::X86)
        )
        .with_desc("freeze → MOV (passthrough)"),
    );

    // ========================================================================
    // VAArg (lowered by frontend, minimal placeholder)
    // ========================================================================

    rules.push(
        rule!(
            "va_arg",
            Opcode::VAArg,
            TypeConstraint::Any,
            X86Opcode::MOV,
            3,
            1310,
            Some(X86Feature::X86)
        )
        .with_desc("va_arg → MOV (placeholder)"),
    );

    // ========================================================================
    // ExtractValue / InsertValue
    // ========================================================================

    rules.push(
        rule!(
            "extractvalue",
            Opcode::ExtractValue,
            TypeConstraint::AnyInt,
            X86Opcode::MOV,
            2,
            1300,
            Some(X86Feature::X86)
        )
        .with_desc("extractvalue → MOV"),
    );
    rules.push(
        rule!(
            "insertvalue",
            Opcode::InsertValue,
            TypeConstraint::AnyInt,
            X86Opcode::MOV,
            2,
            1299,
            Some(X86Feature::X86)
        )
        .with_desc("insertvalue → MOV"),
    );

    // ========================================================================
    // Vector extract/insert element
    // ========================================================================

    rules.push(
        rule!(
            "extract_elem_i32",
            Opcode::ExtractElement,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PEXTRD,
            2,
            1290,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("extract i32 lane → PEXTRD (SSE4.1)"),
    );
    rules.push(
        rule!(
            "insert_elem_i32",
            Opcode::InsertElement,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::PINSRD,
            2,
            1289,
            Some(X86Feature::SSE41)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("insert i32 lane → PINSRD (SSE4.1)"),
    );

    // ========================================================================
    // Extract element (SSE2 integer variants)
    // ========================================================================

    rules.push(
        rule!(
            "pextrw",
            Opcode::ExtractElement,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::PEXTRW,
            2,
            1285,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I16),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("extract i16 lane → PEXTRW (SSE2)"),
    );
    rules.push(
        rule!(
            "pinsrw",
            Opcode::InsertElement,
            TypeConstraint::Int(IntWidth::I16),
            X86Opcode::PINSRW,
            2,
            1284,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I16),
            TypeConstraint::Int(IntWidth::I16),
            TypeConstraint::Int(IntWidth::I8),
        ])
        .with_desc("insert i16 lane → PINSRW (SSE2)"),
    );

    // ========================================================================
    // Packed convert (SSE2)
    // ========================================================================

    rules.push(
        rule!(
            "cvtdq2ps",
            Opcode::SIToFP,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::CVTDQ2PS,
            1,
            1280,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Int(IntWidth::I32)])
        .with_desc("i32→f32 packed → CVTDQ2PS (SSE2)"),
    );
    rules.push(
        rule!(
            "cvtps2dq",
            Opcode::FPToSI,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::CVTPS2DQ,
            1,
            1279,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![TypeConstraint::Float(FloatWidth::F32)])
        .with_desc("f32→i32 packed → CVTPS2DQ (SSE2)"),
    );

    // ========================================================================
    // FREM — not directly supported; goes through x87 or library call
    // ========================================================================

    rules.push(
        rule!(
            "frem_f32",
            Opcode::FRem,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::DIVSS,
            2,
            1270,
            Some(X86Feature::SSE)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F32),
            TypeConstraint::Float(FloatWidth::F32),
        ])
        .with_desc("frem f32 → DIVSS (partial, needs remainder extraction)"),
    );
    rules.push(
        rule!(
            "frem_f64",
            Opcode::FRem,
            TypeConstraint::Float(FloatWidth::F64),
            X86Opcode::DIVSD,
            2,
            1269,
            Some(X86Feature::SSE2)
        )
        .with_operands(vec![
            TypeConstraint::Float(FloatWidth::F64),
            TypeConstraint::Float(FloatWidth::F64),
        ])
        .with_desc("frem f64 → DIVSD (partial, needs remainder extraction)"),
    );

    // ========================================================================
    // URem / SRem — integer remainder via DIV/IDIV
    // ========================================================================

    rules.push(
        rule!(
            "urem_i32",
            Opcode::URem,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::DIV,
            20,
            1260,
            Some(X86Feature::X86)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 urem → DIV (result in EDX)"),
    );
    rules.push(
        rule!(
            "srem_i32",
            Opcode::SRem,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::IDIV,
            20,
            1259,
            Some(X86Feature::X86)
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .with_desc("i32 srem → IDIV (result in EDX)"),
    );

    // Sort by descending priority
    rules.sort_by_key(|r| -(r.priority as i32));

    rules
}

// ============================================================================
// Build complex patterns
// ============================================================================

/// Build all complex (multi-instruction) ISel patterns.
pub fn build_complex_patterns() -> Vec<ISelComplexPattern> {
    vec![
        // LEA: (add (mul base, scale), offset) → LEA
        ISelComplexPattern {
            name: "complex_lea".to_string(),
            ir_opcode: Opcode::Add,
            sub_instructions: vec![Opcode::Mul, Opcode::Add],
            x86_sequence: vec![X86Opcode::LEA],
            cost: 1,
            priority: 8000,
            required_feature: Some(X86Feature::X86),
            only_64bit: false,
            only_32bit: false,
            description: "(add (mul base, scale), offset) → LEA".to_string(),
        },
        // memcpy → REP MOVSB sequence
        ISelComplexPattern {
            name: "complex_memcpy".to_string(),
            ir_opcode: Opcode::Call,
            sub_instructions: vec![],
            x86_sequence: vec![X86Opcode::MOVSB, X86Opcode::REP],
            cost: 10,
            priority: 7500,
            required_feature: Some(X86Feature::X86),
            only_64bit: false,
            only_32bit: false,
            description: "memcpy → REP MOVSB (string copy)".to_string(),
        },
        // memset → REP STOSB sequence
        ISelComplexPattern {
            name: "complex_memset".to_string(),
            ir_opcode: Opcode::Call,
            sub_instructions: vec![],
            x86_sequence: vec![X86Opcode::STOSB, X86Opcode::REP],
            cost: 10,
            priority: 7499,
            required_feature: Some(X86Feature::X86),
            only_64bit: false,
            only_32bit: false,
            description: "memset → REP STOSB (string fill)".to_string(),
        },
        // memmove → DF clear + REP MOVSB sequence
        ISelComplexPattern {
            name: "complex_memmove".to_string(),
            ir_opcode: Opcode::Call,
            sub_instructions: vec![],
            x86_sequence: vec![X86Opcode::MOVSB, X86Opcode::REP],
            cost: 12,
            priority: 7498,
            required_feature: Some(X86Feature::X86),
            only_64bit: false,
            only_32bit: false,
            description: "memmove → CLD + REP MOVSB".to_string(),
        },
        // i64 mul on 32-bit: MUL (EDX:EAX = EAX * src)
        ISelComplexPattern {
            name: "complex_i64_mul_32bit".to_string(),
            ir_opcode: Opcode::Mul,
            sub_instructions: vec![],
            x86_sequence: vec![X86Opcode::MUL, X86Opcode::MOV, X86Opcode::MOV],
            cost: 4,
            priority: 7000,
            required_feature: Some(X86Feature::X86),
            only_64bit: false,
            only_32bit: true,
            description: "i64 mul on 32‑bit → MUL + MOV sequence".to_string(),
        },
        // i64 div on 32-bit: DIV (expands EDX:EAX / src → EAX rem EDX)
        ISelComplexPattern {
            name: "complex_i64_div_32bit".to_string(),
            ir_opcode: Opcode::SDiv,
            sub_instructions: vec![],
            x86_sequence: vec![X86Opcode::IDIV, X86Opcode::MOV, X86Opcode::MOV],
            cost: 30,
            priority: 6999,
            required_feature: Some(X86Feature::X86),
            only_64bit: false,
            only_32bit: true,
            description: "i64 div on 32‑bit → IDIV + MOV sequence".to_string(),
        },
        // Atomic Compare-and-Swap full sequence
        ISelComplexPattern {
            name: "complex_atomic_cmpxchg".to_string(),
            ir_opcode: Opcode::CmpXchg,
            sub_instructions: vec![],
            x86_sequence: vec![X86Opcode::CMP, X86Opcode::JE, X86Opcode::MOV],
            cost: 5,
            priority: 6900,
            required_feature: Some(X86Feature::X86),
            only_64bit: false,
            only_32bit: false,
            description: "cmpxchg → CMP + JE + MOV + XCHG sequence".to_string(),
        },
        // Math intrinsics: sin/cos via x87
        ISelComplexPattern {
            name: "complex_fsin".to_string(),
            ir_opcode: Opcode::Call,
            sub_instructions: vec![],
            x86_sequence: vec![X86Opcode::MOVSS, X86Opcode::CALL],
            cost: 50,
            priority: 6500,
            required_feature: Some(X86Feature::X86),
            only_64bit: false,
            only_32bit: false,
            description: "fsin → x87 FSIN / libcall".to_string(),
        },
        ISelComplexPattern {
            name: "complex_fcos".to_string(),
            ir_opcode: Opcode::Call,
            sub_instructions: vec![],
            x86_sequence: vec![X86Opcode::MOVSS, X86Opcode::CALL],
            cost: 50,
            priority: 6499,
            required_feature: Some(X86Feature::X86),
            only_64bit: false,
            only_32bit: false,
            description: "fcos → x87 FCOS / libcall".to_string(),
        },
        // INVOKE (lower to CALL + landing pad setup)
        ISelComplexPattern {
            name: "complex_invoke".to_string(),
            ir_opcode: Opcode::Invoke,
            sub_instructions: vec![],
            x86_sequence: vec![X86Opcode::CALL, X86Opcode::MOV],
            cost: 3,
            priority: 6400,
            required_feature: Some(X86Feature::X86),
            only_64bit: false,
            only_32bit: false,
            description: "invoke → CALL + landing pad".to_string(),
        },
        // RESUME (exception handling resume)
        ISelComplexPattern {
            name: "complex_resume".to_string(),
            ir_opcode: Opcode::Resume,
            sub_instructions: vec![],
            x86_sequence: vec![X86Opcode::JMP],
            cost: 2,
            priority: 6399,
            required_feature: Some(X86Feature::X86),
            only_64bit: false,
            only_32bit: false,
            description: "resume → JMP (unwind)".to_string(),
        },
    ]
}

// ============================================================================
// X86ISelGoldenTable — the complete pattern database
// ============================================================================

/// The complete X86 instruction selection golden table.
///
/// This struct owns the entire pattern database and provides APIs for
/// querying, filtering, and matching patterns against IR instructions.
pub struct X86ISelGoldenTable {
    /// All simple pattern rules, sorted by descending priority.
    pub rules: Vec<ISelPatternRule>,

    /// All complex (multi-instruction) patterns.
    pub complex_patterns: Vec<ISelComplexPattern>,

    /// Feature gate controlling pattern availability.
    pub features: FeatureGate,

    /// Coverage tracker.
    pub coverage: ISelCoverageTracker,

    /// Per-opcode index for fast lookup.
    opcode_index: HashMap<Opcode, Vec<usize>>,

    /// Complex pattern per-opcode index.
    complex_index: HashMap<Opcode, Vec<usize>>,
}

impl X86ISelGoldenTable {
    /// Create a new golden table with all built-in patterns and the given features.
    pub fn new(features: FeatureGate) -> Self {
        let rules = build_golden_rules();
        let complex = build_complex_patterns();

        let mut coverage = ISelCoverageTracker::new();
        let mut opcode_index: HashMap<Opcode, Vec<usize>> = HashMap::new();
        let mut complex_index: HashMap<Opcode, Vec<usize>> = HashMap::new();

        for (i, rule) in rules.iter().enumerate() {
            let op_name = rule.ir_opcode.name().to_string();
            coverage.register_pattern(&op_name, rule.required_feature);
            opcode_index.entry(rule.ir_opcode).or_default().push(i);
        }

        for (i, cp) in complex.iter().enumerate() {
            let op_name = cp.ir_opcode.name().to_string();
            coverage.register_complex(&op_name);
            complex_index.entry(cp.ir_opcode).or_default().push(i);
        }

        Self {
            rules,
            complex_patterns: complex,
            features,
            coverage,
            opcode_index,
            complex_index,
        }
    }

    /// Look up all matching rules for an IR opcode, filtered by available features.
    pub fn lookup(&self, ir_opcode: Opcode) -> Vec<&ISelPatternRule> {
        self.opcode_index
            .get(&ir_opcode)
            .map(|indices| {
                indices
                    .iter()
                    .filter_map(|&i| {
                        let r = &self.rules[i];
                        if r.is_available(&self.features) {
                            Some(r)
                        } else {
                            None
                        }
                    })
                    .collect()
            })
            .unwrap_or_default()
    }

    /// Look up complex patterns for an IR opcode.
    pub fn lookup_complex(&self, ir_opcode: Opcode) -> Vec<&ISelComplexPattern> {
        self.complex_index
            .get(&ir_opcode)
            .map(|indices| {
                indices
                    .iter()
                    .filter_map(|&i| {
                        let cp = &self.complex_patterns[i];
                        if cp.is_available(&self.features) {
                            Some(cp)
                        } else {
                            None
                        }
                    })
                    .collect()
            })
            .unwrap_or_default()
    }

    /// Get the best (highest priority) rule for a given IR opcode.
    pub fn best_for(&self, ir_opcode: Opcode) -> Option<&ISelPatternRule> {
        self.lookup(ir_opcode).into_iter().next()
    }

    /// Total number of available patterns.
    pub fn total_available(&self) -> usize {
        self.rules
            .iter()
            .filter(|r| r.is_available(&self.features))
            .count()
    }

    /// Total number of patterns in the table (including unavailable ones).
    pub fn total_patterns(&self) -> usize {
        self.rules.len()
    }

    /// Total number of complex patterns.
    pub fn total_complex(&self) -> usize {
        self.complex_patterns.len()
    }

    /// Get all distinct IR opcodes that have coverage.
    pub fn covered_opcodes(&self) -> Vec<String> {
        let mut codes: Vec<_> = self
            .opcode_index
            .keys()
            .map(|op| op.name().to_string())
            .collect();
        codes.sort();
        codes
    }

    /// Get patterns for a specific feature.
    pub fn patterns_by_feature(&self, feature: X86Feature) -> Vec<&ISelPatternRule> {
        self.rules
            .iter()
            .filter(|r| r.required_feature == Some(feature))
            .collect()
    }

    /// Get patterns that are available (matching both feature gate and mode).
    pub fn available_patterns(&self) -> Vec<&ISelPatternRule> {
        self.rules
            .iter()
            .filter(|r| r.is_available(&self.features))
            .collect()
    }

    /// Dump a summary of the pattern table.
    pub fn summary(&self) -> String {
        format!(
            "X86 ISel Golden Table: {} total patterns, {} complex patterns, {} covered IR ops ({})",
            self.rules.len(),
            self.complex_patterns.len(),
            self.coverage.covered_opcodes().len(),
            self.features.enabled_features().len(),
        )
    }
}

impl Default for X86ISelGoldenTable {
    fn default() -> Self {
        Self::new(FeatureGate::default())
    }
}

// ============================================================================
// Re-export key types
// ============================================================================

/// Create a FeatureGate with all features enabled for 64-bit mode.
pub fn all_features_64bit() -> FeatureGate {
    let mut gate = FeatureGate {
        enabled: HashSet::new(),
        is_64bit: true,
    };
    for f in FeatureGate::all_features() {
        gate.enable_with_implications(f);
    }
    gate
}

/// Create a FeatureGate with baseline features for 64-bit mode.
pub fn baseline_64bit() -> FeatureGate {
    let mut gate = FeatureGate {
        enabled: HashSet::new(),
        is_64bit: true,
    };
    gate.enable_with_implications(X86Feature::X86_64);
    gate.enable(X86Feature::CMOV);
    gate
}

/// Create a FeatureGate for modern x86-64-v4 (AVX-512F with BW/DQ/VL).
pub fn x86_64_v4() -> FeatureGate {
    let mut gate = FeatureGate {
        enabled: HashSet::new(),
        is_64bit: true,
    };
    for f in &[
        X86Feature::X86_64,
        X86Feature::CMOV,
        X86Feature::CX8,
        X86Feature::CX16,
        X86Feature::SSE,
        X86Feature::SSE2,
        X86Feature::SSE3,
        X86Feature::SSSE3,
        X86Feature::SSE41,
        X86Feature::SSE42,
        X86Feature::POPCNT,
        X86Feature::AVX,
        X86Feature::AVX2,
        X86Feature::FMA,
        X86Feature::BMI,
        X86Feature::BMI2,
        X86Feature::LZCNT,
        X86Feature::MOVBE,
        X86Feature::AVX512F,
        X86Feature::AVX512BW,
        X86Feature::AVX512DQ,
        X86Feature::AVX512VL,
        X86Feature::AVX512CD,
    ] {
        gate.enable_with_implications(*f);
    }
    gate
}

// ============================================================================
// Tests
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    // ========================================================================
    // Helper utilities
    // ========================================================================

    fn default_gate() -> FeatureGate {
        FeatureGate::default()
    }

    fn all_features_gate() -> FeatureGate {
        all_features_64bit()
    }

    fn baseline_gate() -> FeatureGate {
        baseline_64bit()
    }

    // ========================================================================
    // IntWidth / FloatWidth / VectorWidth
    // ========================================================================

    #[test]
    fn test_int_width_bits() {
        assert_eq!(IntWidth::I8.bits(), 8);
        assert_eq!(IntWidth::I16.bits(), 16);
        assert_eq!(IntWidth::I32.bits(), 32);
        assert_eq!(IntWidth::I64.bits(), 64);
    }

    #[test]
    fn test_int_width_from_bits() {
        assert_eq!(IntWidth::from_bits(8), Some(IntWidth::I8));
        assert_eq!(IntWidth::from_bits(16), Some(IntWidth::I16));
        assert_eq!(IntWidth::from_bits(32), Some(IntWidth::I32));
        assert_eq!(IntWidth::from_bits(64), Some(IntWidth::I64));
        assert_eq!(IntWidth::from_bits(128), None);
    }

    #[test]
    fn test_float_width() {
        assert_eq!(FloatWidth::F32.bits(), 32);
        assert_eq!(FloatWidth::F64.bits(), 64);
    }

    #[test]
    fn test_vector_width() {
        assert_eq!(VectorWidth::from_lanes(4), Some(VectorWidth::V4));
        assert_eq!(VectorWidth::from_lanes(8), Some(VectorWidth::V8));
        assert_eq!(VectorWidth::from_lanes(1), None);
    }

    // ========================================================================
    // FeatureGate
    // ========================================================================

    #[test]
    fn test_feature_gate_basic() {
        let gate = FeatureGate::new(&[X86Feature::X86, X86Feature::SSE2], true);
        assert!(gate.has(X86Feature::X86));
        assert!(gate.has(X86Feature::SSE2));
        assert!(!gate.has(X86Feature::AVX));
        assert!(!gate.has(X86Feature::AVX2));
    }

    #[test]
    fn test_feature_gate_any_all() {
        let gate = FeatureGate::new(&[X86Feature::SSE, X86Feature::SSE2], true);
        assert!(gate.has_any(&[X86Feature::SSE, X86Feature::AVX]));
        assert!(!gate.has_any(&[X86Feature::AVX, X86Feature::AVX2]));
        assert!(gate.has_all(&[X86Feature::SSE, X86Feature::SSE2]));
        assert!(!gate.has_all(&[X86Feature::SSE, X86Feature::AVX]));
    }

    #[test]
    fn test_feature_implication_avx2() {
        let mut gate = FeatureGate::new(&[], true);
        gate.enable_with_implications(X86Feature::AVX2);
        assert!(gate.has(X86Feature::AVX2));
        assert!(gate.has(X86Feature::AVX));
        assert!(gate.has(X86Feature::SSE));
        assert!(gate.has(X86Feature::SSE2));
        assert!(gate.has(X86Feature::SSE3));
        assert!(gate.has(X86Feature::SSSE3));
        assert!(gate.has(X86Feature::SSE41));
        assert!(gate.has(X86Feature::SSE42));
        assert!(gate.has(X86Feature::FMA));
        assert!(gate.has(X86Feature::BMI));
        assert!(gate.has(X86Feature::BMI2));
    }

    #[test]
    fn test_feature_implication_avx512f() {
        let mut gate = FeatureGate::new(&[], true);
        gate.enable_with_implications(X86Feature::AVX512F);
        assert!(gate.has(X86Feature::AVX512F));
        assert!(gate.has(X86Feature::AVX2));
        assert!(gate.has(X86Feature::AVX));
        assert!(gate.has(X86Feature::SSE2));
        assert!(gate.has(X86Feature::FMA));
        assert!(gate.has(X86Feature::F16C));
        assert!(gate.has(X86Feature::BMI));
        assert!(gate.has(X86Feature::BMI2));
    }

    #[test]
    fn test_feature_gate_default() {
        let gate = FeatureGate::default();
        assert!(gate.is_64bit);
        assert!(gate.has(X86Feature::X86));
        assert!(gate.has(X86Feature::X86_64));
        assert!(gate.has(X86Feature::SSE2));
    }

    #[test]
    fn test_feature_gate_enabled_list() {
        let gate = FeatureGate::new(&[X86Feature::X86, X86Feature::SSE, X86Feature::AVX], true);
        let enabled = gate.enabled_features();
        assert!(enabled.contains(&X86Feature::X86));
        assert!(enabled.contains(&X86Feature::SSE));
        assert!(enabled.contains(&X86Feature::AVX));
        assert!(!enabled.contains(&X86Feature::SSE2));
    }

    // ========================================================================
    // ISelPatternRule
    // ========================================================================

    #[test]
    fn test_pattern_rule_creation() {
        let rule = ISelPatternRule::new(
            "test_add",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::ADD,
            1,
            1000,
            Some(X86Feature::X86),
        );
        assert_eq!(rule.name, "test_add");
        assert_eq!(rule.ir_opcode, Opcode::Add);
        assert_eq!(rule.cost, 1);
        assert_eq!(rule.priority, 1000);
    }

    #[test]
    fn test_pattern_rule_availability() {
        let rule = ISelPatternRule::new(
            "avx_only",
            Opcode::FAdd,
            TypeConstraint::Float(FloatWidth::F32),
            X86Opcode::VADDPS,
            1,
            1000,
            Some(X86Feature::AVX),
        );

        let baseline = baseline_gate();
        assert!(!rule.is_available(&baseline));

        let avx_gate = FeatureGate::new(&[X86Feature::AVX], true);
        // Enable implications so SSE base is present
        let mut full_avx = FeatureGate::new(&[], true);
        full_avx.enable_with_implications(X86Feature::AVX);
        assert!(rule.is_available(&full_avx));
    }

    #[test]
    fn test_pattern_rule_64bit_only() {
        let rule = ISelPatternRule::new(
            "only64",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I64),
            X86Opcode::ADD,
            1,
            1000,
            Some(X86Feature::X86_64),
        )
        .only64();

        let gate64 = FeatureGate::new(&[X86Feature::X86, X86Feature::X86_64], true);
        assert!(rule.is_available(&gate64));

        let gate32 = FeatureGate::new(&[X86Feature::X86], false);
        assert!(!rule.is_available(&gate32));
    }

    #[test]
    fn test_pattern_rule_32bit_only() {
        let rule = ISelPatternRule::new(
            "only32",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::ADD,
            1,
            1000,
            Some(X86Feature::X86),
        )
        .only32();

        let gate64 = FeatureGate::new(&[X86Feature::X86], true);
        assert!(!rule.is_available(&gate64));

        let gate32 = FeatureGate::new(&[X86Feature::X86], false);
        assert!(rule.is_available(&gate32));
    }

    #[test]
    fn test_pattern_builders() {
        let rule = ISelPatternRule::new(
            "builder_test",
            Opcode::Add,
            TypeConstraint::Int(IntWidth::I32),
            X86Opcode::ADD,
            1,
            5000,
            Some(X86Feature::X86),
        )
        .with_operands(vec![
            TypeConstraint::Int(IntWidth::I32),
            TypeConstraint::Int(IntWidth::I32),
        ])
        .commutative()
        .with_flags()
        .with_desc("test builder pattern");

        assert!(rule.is_commutative);
        assert!(rule.sets_flags);
        assert_eq!(rule.operand_types.len(), 2);
        assert_eq!(rule.description, "test builder pattern");
    }

    // ========================================================================
    // Golden table: basic existence
    // ========================================================================

    #[test]
    fn test_golden_table_has_patterns() {
        let table = X86ISelGoldenTable::default();
        assert!(table.total_patterns() > 0);
        assert!(table.total_complex() > 0);
    }

    #[test]
    fn test_golden_table_default_has_many_patterns() {
        let table = X86ISelGoldenTable::default();
        let count = table.total_patterns();
        assert!(count > 200, "expected > 200 patterns, got {}", count);
    }

    // ========================================================================
    // Coverage: all IR ops have some pattern
    // ========================================================================

    #[test]
    fn test_coverage_add() {
        let table = X86ISelGoldenTable::default();
        let patterns = table.lookup(Opcode::Add);
        assert!(!patterns.is_empty(), "Add should have patterns");
    }

    #[test]
    fn test_coverage_sub() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Sub).is_empty());
    }

    #[test]
    fn test_coverage_mul() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Mul).is_empty());
    }

    #[test]
    fn test_coverage_udiv() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::UDiv).is_empty());
    }

    #[test]
    fn test_coverage_sdiv() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::SDiv).is_empty());
    }

    #[test]
    fn test_coverage_urem() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::URem).is_empty());
    }

    #[test]
    fn test_coverage_srem() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::SRem).is_empty());
    }

    #[test]
    fn test_coverage_fadd() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::FAdd).is_empty());
    }

    #[test]
    fn test_coverage_fsub() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::FSub).is_empty());
    }

    #[test]
    fn test_coverage_fmul() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::FMul).is_empty());
    }

    #[test]
    fn test_coverage_fdiv() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::FDiv).is_empty());
    }

    #[test]
    fn test_coverage_frem() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::FRem).is_empty());
    }

    #[test]
    fn test_coverage_shl() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Shl).is_empty());
    }

    #[test]
    fn test_coverage_lshr() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::LShr).is_empty());
    }

    #[test]
    fn test_coverage_ashr() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::AShr).is_empty());
    }

    #[test]
    fn test_coverage_and() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::And).is_empty());
    }

    #[test]
    fn test_coverage_or() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Or).is_empty());
    }

    #[test]
    fn test_coverage_xor() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Xor).is_empty());
    }

    #[test]
    fn test_coverage_icmp() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::ICmp).is_empty());
    }

    #[test]
    fn test_coverage_fcmp() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::FCmp).is_empty());
    }

    #[test]
    fn test_coverage_load() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Load).is_empty());
    }

    #[test]
    fn test_coverage_store() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Store).is_empty());
    }

    #[test]
    fn test_coverage_alloca() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Alloca).is_empty());
    }

    #[test]
    fn test_coverage_gep() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::GetElementPtr).is_empty());
    }

    #[test]
    fn test_coverage_trunc() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Trunc).is_empty());
    }

    #[test]
    fn test_coverage_zext() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::ZExt).is_empty());
    }

    #[test]
    fn test_coverage_sext() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::SExt).is_empty());
    }

    #[test]
    fn test_coverage_fptrunc() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::FPTrunc).is_empty());
    }

    #[test]
    fn test_coverage_fpext() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::FPExt).is_empty());
    }

    #[test]
    fn test_coverage_fptosi() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::FPToSI).is_empty());
    }

    #[test]
    fn test_coverage_fptoui() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::FPToUI).is_empty());
    }

    #[test]
    fn test_coverage_sitofp() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::SIToFP).is_empty());
    }

    #[test]
    fn test_coverage_uitofp() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::UIToFP).is_empty());
    }

    #[test]
    fn test_coverage_ptrtoint() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::PtrToInt).is_empty());
    }

    #[test]
    fn test_coverage_inttoptr() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::IntToPtr).is_empty());
    }

    #[test]
    fn test_coverage_bitcast() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::BitCast).is_empty());
    }

    #[test]
    fn test_coverage_br() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Br).is_empty());
    }

    #[test]
    fn test_coverage_switch() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Switch).is_empty());
    }

    #[test]
    fn test_coverage_call() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Call).is_empty());
    }

    #[test]
    fn test_coverage_ret() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Ret).is_empty());
    }

    #[test]
    fn test_coverage_phi() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Phi).is_empty());
    }

    #[test]
    fn test_coverage_select() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Select).is_empty());
    }

    #[test]
    fn test_coverage_freeze() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Freeze).is_empty());
    }

    #[test]
    fn test_coverage_unreachable() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Unreachable).is_empty());
    }

    #[test]
    fn test_coverage_extractelement() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::ExtractElement).is_empty());
    }

    #[test]
    fn test_coverage_insertelement() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::InsertElement).is_empty());
    }

    #[test]
    fn test_coverage_shufflevector() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::ShuffleVector).is_empty());
    }

    #[test]
    fn test_coverage_extractvalue() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::ExtractValue).is_empty());
    }

    #[test]
    fn test_coverage_insertvalue() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::InsertValue).is_empty());
    }

    #[test]
    fn test_coverage_fence() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::Fence).is_empty());
    }

    #[test]
    fn test_coverage_cmpxchg() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::CmpXchg).is_empty());
    }

    #[test]
    fn test_coverage_atomicrmw() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::AtomicRMW).is_empty());
    }

    #[test]
    fn test_coverage_va_arg() {
        let table = X86ISelGoldenTable::default();
        assert!(!table.lookup(Opcode::VAArg).is_empty());
    }

    // ========================================================================
    // Coverage: opcodes that may not have direct patterns but have complex ones
    // ========================================================================

    #[test]
    fn test_coverage_invoke_complex() {
        let table = X86ISelGoldenTable::default();
        let complex = table.lookup_complex(Opcode::Invoke);
        assert!(!complex.is_empty(), "Invoke should have complex patterns");
    }

    #[test]
    fn test_coverage_resume_complex() {
        let table = X86ISelGoldenTable::default();
        let complex = table.lookup_complex(Opcode::Resume);
        assert!(!complex.is_empty(), "Resume should have complex patterns");
    }

    // ========================================================================
    // Priority ordering
    // ========================================================================

    #[test]
    fn test_rules_sorted_by_priority_descending() {
        let table = X86ISelGoldenTable::default();
        for window in table.rules.windows(2) {
            assert!(
                window[0].priority >= window[1].priority,
                "Rules not sorted by priority: {} (pri={}) before {} (pri={})",
                window[0].name,
                window[0].priority,
                window[1].name,
                window[1].priority,
            );
        }
    }

    #[test]
    fn test_best_for_returns_highest_priority() {
        let table = X86ISelGoldenTable::default();
        let best = table.best_for(Opcode::Add);
        assert!(best.is_some());
        let all = table.lookup(Opcode::Add);
        let max_priority = all.iter().map(|r| r.priority).max().unwrap();
        assert_eq!(best.unwrap().priority, max_priority);
    }

    // ========================================================================
    // Feature gating
    // ========================================================================

    #[test]
    fn test_avx512_patterns_gated_without_feature() {
        let table = X86ISelGoldenTable::new(baseline_gate());
        let all_add = table.lookup(Opcode::Add);
        // None should have AVX-512 feature requirement met
        let avx512_matches: Vec<_> = all_add
            .iter()
            .filter(|r| match r.required_feature {
                Some(f) => matches!(
                    f,
                    X86Feature::AVX512F
                        | X86Feature::AVX512BW
                        | X86Feature::AVX512DQ
                        | X86Feature::AVX512VL
                ),
                None => false,
            })
            .collect();
        assert!(
            avx512_matches.is_empty(),
            "AVX-512 patterns should not be available without AVX-512F feature"
        );
    }

    #[test]
    fn test_avx512_patterns_available_with_feature() {
        let table = X86ISelGoldenTable::new(all_features_gate());
        let all_add = table.lookup(Opcode::Add);
        let has_avx512 = all_add
            .iter()
            .any(|r| matches!(r.required_feature, Some(X86Feature::AVX512F)));
        if has_avx512 {
            assert!(true);
        } else {
            // With all features, we should see AVX-512 patterns
            let avx512_in_rules: Vec<_> = table
                .rules
                .iter()
                .filter(|r| {
                    r.ir_opcode == Opcode::Add
                        && matches!(r.required_feature, Some(X86Feature::AVX512F))
                })
                .collect();
            assert!(
                !avx512_in_rules.is_empty(),
                "There should be AVX-512F rules for Add"
            );
        }
    }

    #[test]
    fn test_sse_patterns_available_in_baseline() {
        let table = X86ISelGoldenTable::new(baseline_gate());
        let sse_patterns: Vec<_> = table
            .lookup(Opcode::FAdd)
            .iter()
            .filter(|r| {
                matches!(
                    r.required_feature,
                    Some(X86Feature::SSE) | Some(X86Feature::SSE2)
                )
            })
            .collect();
        assert!(
            !sse_patterns.is_empty(),
            "SSE FAdd patterns should be available in baseline"
        );
    }

    #[test]
    fn test_feature_disable_removes_patterns() {
        let mut gate = all_features_gate();
        let table_all = X86ISelGoldenTable::new(gate.clone());
        let all_count = table_all.lookup(Opcode::FAdd).len();

        gate.disable(X86Feature::AVX);
        gate.disable(X86Feature::AVX2);
        gate.disable(X86Feature::AVX512F);
        gate.disable(X86Feature::FMA);
        let table_reduced = X86ISelGoldenTable::new(gate);
        let reduced_count = table_reduced.lookup(Opcode::FAdd).len();

        assert!(
            reduced_count <= all_count,
            "Disabling features should not increase pattern count"
        );
    }

    // ========================================================================
    // Complex patterns
    // ========================================================================

    #[test]
    fn test_complex_patterns_exist() {
        let table = X86ISelGoldenTable::default();
        assert!(
            table.total_complex() > 5,
            "Should have several complex patterns"
        );
    }

    #[test]
    fn test_complex_lea_exists() {
        let table = X86ISelGoldenTable::default();
        let complex = table.lookup_complex(Opcode::Add);
        let lea_pattern = complex.iter().find(|cp| cp.name == "complex_lea");
        assert!(lea_pattern.is_some(), "LEA complex pattern should exist");
    }

    #[test]
    fn test_complex_memcpy_exists() {
        let table = X86ISelGoldenTable::default();
        let complex = table.lookup_complex(Opcode::Call);
        let memcpy = complex.iter().find(|cp| cp.name == "complex_memcpy");
        assert!(memcpy.is_some(), "memcpy complex pattern should exist");
    }

    #[test]
    fn test_complex_memset_exists() {
        let table = X86ISelGoldenTable::default();
        let complex = table.lookup_complex(Opcode::Call);
        let memset = complex.iter().find(|cp| cp.name == "complex_memset");
        assert!(memset.is_some(), "memset complex pattern should exist");
    }

    #[test]
    fn test_complex_memmove_exists() {
        let table = X86ISelGoldenTable::default();
        let complex = table.lookup_complex(Opcode::Call);
        let memmove = complex.iter().find(|cp| cp.name == "complex_memmove");
        assert!(memmove.is_some(), "memmove complex pattern should exist");
    }

    #[test]
    fn test_complex_32bit_specific() {
        let mut gate = baseline_gate();
        gate.is_64bit = false;
        let table = X86ISelGoldenTable::new(gate);
        let complex = table.lookup_complex(Opcode::Mul);
        let mul_32 = complex.iter().find(|cp| cp.name == "complex_i64_mul_32bit");
        assert!(
            mul_32.is_some(),
            "32-bit i64 mul complex pattern should be available in 32-bit mode"
        );
    }

    #[test]
    fn test_complex_32bit_not_in_64bit() {
        let table = X86ISelGoldenTable::new(baseline_gate()); // 64-bit
        let complex = table.lookup_complex(Opcode::Mul);
        let mul_32 = complex.iter().find(|cp| cp.name == "complex_i64_mul_32bit");
        assert!(
            mul_32.is_none(),
            "32-bit i64 mul complex pattern should NOT be available in 64-bit mode"
        );
    }

    // ========================================================================
    // Coverage tracker
    // ========================================================================

    #[test]
    fn test_coverage_tracker_basic() {
        let mut tracker = ISelCoverageTracker::new();
        tracker.register_pattern("add", Some(X86Feature::X86));
        tracker.register_pattern("add", Some(X86Feature::AVX));
        tracker.register_pattern("sub", Some(X86Feature::X86));

        assert!(tracker.is_covered("add"));
        assert!(tracker.is_covered("sub"));
        assert!(!tracker.is_covered("mul"));
        assert_eq!(tracker.count_for("add"), 2);
        assert_eq!(tracker.count_for("sub"), 1);
    }

    #[test]
    fn test_coverage_tracker_gaps() {
        let mut tracker = ISelCoverageTracker::new();
        tracker.register_pattern("add", Some(X86Feature::X86));
        tracker.register_pattern("sub", Some(X86Feature::X86));
        tracker.register_pattern("ret", Some(X86Feature::X86));
        tracker.register_pattern("br", Some(X86Feature::X86));
        tracker.register_pattern("call", Some(X86Feature::X86));
        tracker.register_pattern("load", Some(X86Feature::X86));
        tracker.register_pattern("store", Some(X86Feature::X86));

        let gaps = tracker.gaps();
        // Should have gaps for uncovered ops
        assert!(!gaps.is_empty(), "Should have some gaps");
    }

    #[test]
    fn test_coverage_tracker_summary() {
        let mut tracker = ISelCoverageTracker::new();
        for op in &["add", "sub", "mul", "ret", "br", "call"] {
            tracker.register_pattern(op, Some(X86Feature::X86));
        }
        let summary = tracker.summary();
        assert!(
            summary.contains("6/48"),
            "Summary should show covered count"
        );
    }

    // ========================================================================
    // ISelPatternMatcher
    // ========================================================================

    #[test]
    fn test_matcher_creation() {
        let matcher = ISelPatternMatcher::new(FeatureGate::default());
        assert!(matcher.total_patterns() > 0);
    }

    #[test]
    fn test_matcher_find_best() {
        let mut matcher = ISelPatternMatcher::new(FeatureGate::default());
        let result = matcher.find_best(Opcode::Add, &TypeConstraint::Int(IntWidth::I32));
        assert!(result.is_some());
        let r = result.unwrap();
        assert_eq!(r.rule.ir_opcode, Opcode::Add);
        assert!(!r.is_complex);
    }

    #[test]
    fn test_matcher_find_complex() {
        let mut matcher = ISelPatternMatcher::new(FeatureGate::default());
        let result = matcher.find_complex(Opcode::Add);
        // ADD has the LEA complex pattern
        assert!(result.is_some());
    }

    #[test]
    fn test_matcher_returns_highest_priority() {
        let mut matcher = ISelPatternMatcher::new(FeatureGate::default());
        let result = matcher.find_best(Opcode::Add, &TypeConstraint::Int(IntWidth::I32));
        let best = result.unwrap();
        let all = matcher.patterns_for(Opcode::Add);
        let max_pri = all.iter().map(|r| r.priority).max().unwrap();
        assert_eq!(best.rule.priority, max_pri);
    }

    // ========================================================================
    // Helper functions
    // ========================================================================

    #[test]
    fn test_all_features_64bit() {
        let gate = all_features_64bit();
        assert!(gate.is_64bit);
        assert!(gate.has(X86Feature::X86_64));
        assert!(gate.has(X86Feature::AVX512F));
        assert!(gate.has(X86Feature::AVX512BW));
        assert!(gate.has(X86Feature::AVX512DQ));
        assert!(gate.has(X86Feature::SSE2));
    }

    #[test]
    fn test_baseline_64bit() {
        let gate = baseline_64bit();
        assert!(gate.is_64bit);
        assert!(gate.has(X86Feature::X86_64));
        assert!(gate.has(X86Feature::X86));
        assert!(gate.has(X86Feature::SSE));
        assert!(gate.has(X86Feature::SSE2));
        assert!(gate.has(X86Feature::CMOV));
        assert!(!gate.has(X86Feature::AVX));
    }

    #[test]
    fn test_x86_64_v4() {
        let gate = x86_64_v4();
        assert!(gate.is_64bit);
        assert!(gate.has(X86Feature::AVX512F));
        assert!(gate.has(X86Feature::AVX512BW));
        assert!(gate.has(X86Feature::AVX512DQ));
        assert!(gate.has(X86Feature::AVX512VL));
        assert!(gate.has(X86Feature::AVX2));
        assert!(gate.has(X86Feature::FMA));
        assert!(!gate.has(X86Feature::AVX512BF16));
    }

    // ========================================================================
    // X86ISelGoldenTable summary
    // ========================================================================

    #[test]
    fn test_golden_table_summary() {
        let table = X86ISelGoldenTable::default();
        let summary = table.summary();
        assert!(summary.contains("patterns"));
        assert!(summary.contains("complex"));
    }

    #[test]
    fn test_golden_table_covered_opcodes() {
        let table = X86ISelGoldenTable::default();
        let covered = table.covered_opcodes();
        assert!(covered.contains(&"add".to_string()));
        assert!(covered.contains(&"sub".to_string()));
        assert!(covered.contains(&"mul".to_string()));
        assert!(covered.contains(&"ret".to_string()));
        assert!(covered.contains(&"br".to_string()));
        assert!(covered.contains(&"call".to_string()));
    }

    // ========================================================================
    // Feature-specific pattern lookup
    // ========================================================================

    #[test]
    fn test_patterns_by_feature() {
        let table = X86ISelGoldenTable::default();
        let sse_patterns = table.patterns_by_feature(X86Feature::SSE2);
        assert!(!sse_patterns.is_empty(), "Should have SSE2 patterns");

        // All SSE2 patterns should have SSE2 as required feature
        for p in &sse_patterns {
            assert_eq!(p.required_feature, Some(X86Feature::SSE2));
        }
    }

    // ========================================================================
    // Pattern completeness: verify every important IR opcode has coverage
    // ========================================================================

    #[test]
    fn test_all_major_ir_ops_covered() {
        let table = X86ISelGoldenTable::default();
        let expected_covered = [
            "add",
            "sub",
            "mul",
            "udiv",
            "sdiv",
            "urem",
            "srem",
            "fadd",
            "fsub",
            "fmul",
            "fdiv",
            "frem",
            "shl",
            "lshr",
            "ashr",
            "and",
            "or",
            "xor",
            "icmp",
            "fcmp",
            "load",
            "store",
            "alloca",
            "getelementptr",
            "trunc",
            "zext",
            "sext",
            "fptrunc",
            "fpext",
            "fptoui",
            "fptosi",
            "uitofp",
            "sitofp",
            "ptrtoint",
            "inttoptr",
            "bitcast",
            "br",
            "switch",
            "call",
            "ret",
            "phi",
            "select",
            "freeze",
            "unreachable",
            "extractelement",
            "insertelement",
            "shufflevector",
            "extractvalue",
            "insertvalue",
            "fence",
            "cmpxchg",
            "atomicrmw",
        ];

        let covered = table.covered_opcodes();
        for op in &expected_covered {
            assert!(
                covered.contains(&op.to_string()),
                "IR opcode '{}' should be covered",
                op
            );
        }
    }

    // ========================================================================
    // Cost sanity
    // ========================================================================

    #[test]
    fn test_all_patterns_have_valid_cost() {
        let table = X86ISelGoldenTable::default();
        for rule in &table.rules {
            assert!(
                rule.cost <= 1000,
                "Pattern {} has unreasonably high cost {}",
                rule.name,
                rule.cost
            );
        }
    }

    // ========================================================================
    // Description completeness
    // ========================================================================

    #[test]
    fn test_all_patterns_have_description() {
        let table = X86ISelGoldenTable::default();
        for rule in &table.rules {
            assert!(
                !rule.description.is_empty(),
                "Pattern {} has no description",
                rule.name
            );
        }
    }

    // ========================================================================
    // X86Opcode uniqueness: not all rules should map to MOV
    // ========================================================================

    #[test]
    fn test_not_all_same_opcode() {
        let table = X86ISelGoldenTable::default();
        let mov_count = table
            .rules
            .iter()
            .filter(|r| r.x86_opcode == X86Opcode::MOV)
            .count();
        assert!(
            mov_count < table.rules.len(),
            "Not all patterns should be MOV"
        );
    }

    // ========================================================================
    // TypeConstraint
    // ========================================================================

    #[test]
    fn test_type_constraint_any_matches() {
        assert!(TypeConstraint::Any.matches(&TypeConstraint::Int(IntWidth::I32)));
        assert!(TypeConstraint::Int(IntWidth::I32).matches(&TypeConstraint::Any));
        assert!(TypeConstraint::AnyInt.matches(&TypeConstraint::Int(IntWidth::I8)));
        assert!(TypeConstraint::AnyFloat.matches(&TypeConstraint::Float(FloatWidth::F32)));
        assert!(!TypeConstraint::AnyInt.matches(&TypeConstraint::Float(FloatWidth::F32)));
    }

    #[test]
    fn test_type_constraint_display() {
        assert_eq!(TypeConstraint::Int(IntWidth::I32).to_string(), "i32");
        assert_eq!(TypeConstraint::Float(FloatWidth::F64).to_string(), "f64");
        assert_eq!(TypeConstraint::Any.to_string(), "any");
        assert_eq!(TypeConstraint::Pointer.to_string(), "ptr");
    }

    // ========================================================================
    // X86Feature name
    // ========================================================================

    #[test]
    fn test_x86_feature_names() {
        assert_eq!(X86Feature::X86.name(), "x86");
        assert_eq!(X86Feature::SSE2.name(), "sse2");
        assert_eq!(X86Feature::AVX.name(), "avx");
        assert_eq!(X86Feature::AVX512F.name(), "avx512f");
    }

    // ========================================================================
    // ISelComplexPattern availability
    // ========================================================================

    #[test]
    fn test_complex_pattern_availability() {
        let cp = ISelComplexPattern {
            name: "test".to_string(),
            ir_opcode: Opcode::Add,
            sub_instructions: vec![],
            x86_sequence: vec![X86Opcode::ADD],
            cost: 1,
            priority: 1000,
            required_feature: Some(X86Feature::AVX),
            only_64bit: false,
            only_32bit: false,
            description: "test".to_string(),
        };

        let baseline = baseline_gate();
        assert!(!cp.is_available(&baseline));

        let avx_gate = FeatureGate::new(&[X86Feature::AVX], true);
        assert!(cp.is_available(&avx_gate));
    }

    // ========================================================================
    // MatchStats
    // ========================================================================

    #[test]
    fn test_match_stats_recording() {
        let mut stats = MatchStats::new();
        assert_eq!(stats.total_queries, 0);
        assert_eq!(stats.matches, 0);

        stats.record_hit("add");
        assert_eq!(stats.total_queries, 1);
        assert_eq!(stats.matches, 1);
        assert_eq!(stats.misses, 0);

        stats.record_miss();
        assert_eq!(stats.total_queries, 2);
        assert_eq!(stats.misses, 1);

        stats.record_complex("add");
        assert_eq!(stats.complex_matches, 1);
    }

    #[test]
    fn test_match_stats_hit_rate() {
        let mut stats = MatchStats::new();
        stats.record_hit("add");
        stats.record_hit("sub");
        stats.record_miss();
        assert!((stats.hit_rate() - 66.666).abs() < 1.0);
    }

    // ========================================================================
    // Available patterns count
    // ========================================================================

    #[test]
    fn test_available_patterns() {
        let table = X86ISelGoldenTable::new(baseline_gate());
        let available = table.available_patterns();
        let total = table.total_patterns();
        assert!(available.len() <= total);
    }

    // ========================================================================
    // Golden table default
    // ========================================================================

    #[test]
    fn test_golden_table_default() {
        let table = X86ISelGoldenTable::default();
        assert!(table.features.is_64bit);
        assert!(table.features.has(X86Feature::X86));
        assert!(table.features.has(X86Feature::X86_64));
        assert!(table.features.has(X86Feature::SSE2));
        assert!(table.total_patterns() > 100);
    }
}

#[cfg(test)]
mod golden_isel_integration_tests {
    use super::*;

    /// Full integration test: build the golden table with all features,
    /// verify all major IR ops have available patterns.
    #[test]
    fn test_integration_full_table() {
        let table = X86ISelGoldenTable::new(all_features_64bit());

        // Verify counts
        assert!(table.total_patterns() > 300);
        assert!(table.total_available() > 200);
        assert!(table.total_complex() > 8);

        // Verify every major opcode resolves
        for opcode in &[
            Opcode::Add,
            Opcode::Sub,
            Opcode::Mul,
            Opcode::UDiv,
            Opcode::SDiv,
            Opcode::FAdd,
            Opcode::FSub,
            Opcode::FMul,
            Opcode::FDiv,
            Opcode::And,
            Opcode::Or,
            Opcode::Xor,
            Opcode::Shl,
            Opcode::LShr,
            Opcode::AShr,
            Opcode::ICmp,
            Opcode::FCmp,
            Opcode::Load,
            Opcode::Store,
            Opcode::Alloca,
            Opcode::Br,
            Opcode::Ret,
            Opcode::Call,
            Opcode::Phi,
            Opcode::Select,
        ] {
            let patterns = table.lookup(*opcode);
            assert!(
                !patterns.is_empty(),
                "Opcode {:?} should have available patterns",
                opcode
            );
        }
    }

    /// Test that the table works with a mutable matcher (stats tracking).
    #[test]
    fn test_integration_matcher_stats() {
        let mut matcher = ISelPatternMatcher::new(baseline_64bit());

        // Query several opcodes
        let ops = [
            Opcode::Add,
            Opcode::Sub,
            Opcode::Mul,
            Opcode::Load,
            Opcode::Store,
        ];
        for op in &ops {
            let result = matcher.find_best(*op, &TypeConstraint::AnyInt);
            assert!(result.is_some(), "Should find pattern for {:?}", op);
        }

        // Query an opcode we know is missing direct patterns (Invoke)
        let result = matcher.find_complex(Opcode::Invoke);
        assert!(result.is_some(), "Should find complex pattern for Invoke");

        // Stats should show queries
        assert!(matcher.stats.total_queries >= 5);
        assert!(matcher.stats.complex_matches >= 1);
    }

    /// Regression: the table must be deterministically ordered.
    #[test]
    fn test_integration_deterministic_order() {
        let table1 = X86ISelGoldenTable::default();
        let table2 = X86ISelGoldenTable::default();

        // Patterns should be identical in order and content
        assert_eq!(table1.rules.len(), table2.rules.len());
        for (a, b) in table1.rules.iter().zip(table2.rules.iter()) {
            assert_eq!(a.name, b.name);
            assert_eq!(a.priority, b.priority);
            assert_eq!(a.x86_opcode, b.x86_opcode);
        }
    }

    /// Regression: all complex patterns must have valid opcode sequences.
    #[test]
    fn test_integration_complex_valid_sequences() {
        let table = X86ISelGoldenTable::default();
        for cp in &table.complex_patterns {
            assert!(
                !cp.x86_sequence.is_empty(),
                "Complex pattern '{}' must have a non-empty instruction sequence",
                cp.name
            );
            assert!(
                cp.cost > 0,
                "Complex pattern '{}' must have positive cost",
                cp.name
            );
        }
    }
}