llvm-native-core 0.1.10

//! X86 Final Remaining ISel — Complete coverage for the newest ISA extensions.
//!
//! This module provides ISel patterns for the final set of x86 instruction
//! extensions that were not covered by the main ISel table:
//!
//! - **MPX** (Memory Protection Extensions): BNDMK/BNDCL/BNDCU/BNDCN/
//!   BNDMOV/BNDLDX/BNDSTX
//! - **CET** (Control-flow Enforcement Technology): ENDBR32/ENDBR64, RDSSPD/
//!   RDSSPQ, INCSSPD/INCSSPQ, SAVEPREVSSP/RSTORSSP, SETSSBSY/CLRSSBSY,
//!   WRSSD/WRSSQ/WRUSSD/WRUSSQ
//! - **Key Locker**: LOADIWKEY, AESENC128KL/AESDEC128KL, AESENC256KL/
//!   AESDEC256KL, AESENCWIDE128KL/AESDECWIDE128KL, ENCODEKEY128/ENCODEKEY256
//! - **UINTR** (User Interrupts): SENDUIP/UIRET/TESTUI/CLUI/STUI
//! - **FRED** (Flexible Return and Event Delivery): ERETS/ERETU/LKGS
//! - **CMPCCXADD full**: all 16 condition code variants
//! - **RAO-INT** (Remote Atomic Operations): AADD/AAND/AOR/AXOR all sizes
//! - **PREFETCHITI**: PREFETCHIT0/PREFETCHIT1
//! - **MSRLIST**: RDMSRLIST/WRMSRLIST
//! - **HRESET**: HRESET
//! - **SERIALIZE**: SERIALIZE
//! - **CLDEMOTE**: CLDEMOTE
//! - **MOVDIRI/MOVDIR64B**: MOVDIRI/MOVDIR64B
//! - **ENQCMD/ENQCMDS**: ENQCMD/ENQCMDS
//! - **TSXLDTRK**: XSUSLDTRK/XRESLDTRK
//! - **Additional**: CMPccXADD, WRMSRNS, RDMSR, RDPMC, RDTSC/RDTSCP, etc.
//!
//! Clean-room behavioural reconstruction from:
//! - Intel® 64 and IA-32 Architectures Software Developer's Manual
//! - Intel Architecture Instruction Set Extensions and Future Features
//! - Zero LLVM source code consultation

use crate::x86::x86_instr_info::X86Opcode;
use std::collections::{HashMap, HashSet};

use super::x86_isel_table::{
    ISelStats, ISelTableEntry, PatCondition, PatNode, PatResult, PatternCoverage,
};

// ============================================================================
// Final ISel Table Builder
// ============================================================================

/// Build the final remaining ISel pattern table covering MPX, CET, Key Locker,
/// UINTR, FRED, CMPCCXADD, RAO-INT, and all remaining x86 extensions.
pub fn build_final_isel_table() -> Vec<ISelTableEntry> {
    let mut table = Vec::new();

    // ================================================================
    // SECTION 1: MPX — Memory Protection Extensions
    // ================================================================

    // BNDMK — Create LowerBound and UpperBound in a register
    table.push(ISelTableEntry {
        name: "mpx_bndmk32",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("mpx".into()),
        result: PatResult {
            opcode: X86Opcode::BNDMK,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 3,
            requires_feature: Some("mpx"),
        },
        priority: 1000,
        desc: "BNDMK — create bounds register (32-bit)",
    });
    table.push(ISelTableEntry {
        name: "mpx_bndmk64",
        pattern: PatNode::Any,
        condition: PatCondition::All(vec![
            PatCondition::HasFeature("mpx".into()),
            PatCondition::Is64Bit,
        ]),
        result: PatResult {
            opcode: X86Opcode::BNDMK,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 3,
            requires_feature: Some("mpx"),
        },
        priority: 1001,
        desc: "BNDMK — create bounds register (64-bit)",
    });

    // BNDCL — Check Lower Bound
    table.push(ISelTableEntry {
        name: "mpx_bndcl32",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("mpx".into()),
        result: PatResult {
            opcode: X86Opcode::BNDCL,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 2,
            requires_feature: Some("mpx"),
        },
        priority: 1002,
        desc: "BNDCL — check lower bound (32-bit)",
    });

    // BNDCU — Check Upper Bound
    table.push(ISelTableEntry {
        name: "mpx_bndcu32",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("mpx".into()),
        result: PatResult {
            opcode: X86Opcode::BNDCU,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 2,
            requires_feature: Some("mpx"),
        },
        priority: 1003,
        desc: "BNDCU — check upper bound (32-bit)",
    });

    // BNDCN — Check Upper Bound with 1's complement
    table.push(ISelTableEntry {
        name: "mpx_bndcn32",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("mpx".into()),
        result: PatResult {
            opcode: X86Opcode::BNDCN,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 2,
            requires_feature: Some("mpx"),
        },
        priority: 1004,
        desc: "BNDCN — check upper bound complement (32-bit)",
    });

    // BNDMOV — Move bounds
    table.push(ISelTableEntry {
        name: "mpx_bndmov",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("mpx".into()),
        result: PatResult {
            opcode: X86Opcode::BNDMOV,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("mpx"),
        },
        priority: 1005,
        desc: "BNDMOV — move bounds between registers or memory",
    });

    // BNDLDX — Load bounds using address translation
    table.push(ISelTableEntry {
        name: "mpx_bndldx",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("mpx".into()),
        result: PatResult {
            opcode: X86Opcode::BNDLDX,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 5,
            requires_feature: Some("mpx"),
        },
        priority: 1006,
        desc: "BNDLDX — load extended bounds",
    });

    // BNDSTX — Store bounds using address translation
    table.push(ISelTableEntry {
        name: "mpx_bndstx",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("mpx".into()),
        result: PatResult {
            opcode: X86Opcode::BNDSTX,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 5,
            requires_feature: Some("mpx"),
        },
        priority: 1007,
        desc: "BNDSTX — store extended bounds",
    });

    // ================================================================
    // SECTION 2: CET — Control-flow Enforcement Technology
    // ================================================================

    // ENDBR32 — Terminate Indirect Branch in 32-bit mode
    table.push(ISelTableEntry {
        name: "cet_endbr32",
        pattern: PatNode::Any,
        condition: PatCondition::All(vec![
            PatCondition::HasFeature("cet".into()),
            PatCondition::Is32Bit,
        ]),
        result: PatResult {
            opcode: X86Opcode::ENDBR32,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 0,
            requires_feature: Some("cet"),
        },
        priority: 1010,
        desc: "ENDBR32 — end branch 32-bit (CET)",
    });

    // ENDBR64 — Terminate Indirect Branch in 64-bit mode
    table.push(ISelTableEntry {
        name: "cet_endbr64",
        pattern: PatNode::Any,
        condition: PatCondition::All(vec![
            PatCondition::HasFeature("cet".into()),
            PatCondition::Is64Bit,
        ]),
        result: PatResult {
            opcode: X86Opcode::ENDBR64,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 0,
            requires_feature: Some("cet"),
        },
        priority: 1011,
        desc: "ENDBR64 — end branch 64-bit (CET)",
    });

    // RDSSPD — Read Shadow Stack Pointer (32-bit)
    table.push(ISelTableEntry {
        name: "cet_rdsspd",
        pattern: PatNode::Any,
        condition: PatCondition::All(vec![
            PatCondition::HasFeature("shstk".into()),
            PatCondition::Is32Bit,
        ]),
        result: PatResult {
            opcode: X86Opcode::RDSSPD,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("shstk"),
        },
        priority: 1012,
        desc: "RDSSPD — read shadow stack pointer 32-bit",
    });

    // RDSSPQ — Read Shadow Stack Pointer (64-bit)
    table.push(ISelTableEntry {
        name: "cet_rdsspq",
        pattern: PatNode::Any,
        condition: PatCondition::All(vec![
            PatCondition::HasFeature("shstk".into()),
            PatCondition::Is64Bit,
        ]),
        result: PatResult {
            opcode: X86Opcode::RDSSPQ,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("shstk"),
        },
        priority: 1013,
        desc: "RDSSPQ — read shadow stack pointer 64-bit",
    });

    // INCSSPD — Increment Shadow Stack Pointer (32-bit)
    table.push(ISelTableEntry {
        name: "cet_incsspd",
        pattern: PatNode::Any,
        condition: PatCondition::All(vec![
            PatCondition::HasFeature("shstk".into()),
            PatCondition::Is32Bit,
        ]),
        result: PatResult {
            opcode: X86Opcode::INCSSPD,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("shstk"),
        },
        priority: 1014,
        desc: "INCSSPD — increment shadow stack pointer 32",
    });

    // INCSSPQ — Increment Shadow Stack Pointer (64-bit)
    table.push(ISelTableEntry {
        name: "cet_incsspq",
        pattern: PatNode::Any,
        condition: PatCondition::All(vec![
            PatCondition::HasFeature("shstk".into()),
            PatCondition::Is64Bit,
        ]),
        result: PatResult {
            opcode: X86Opcode::INCSSPQ,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("shstk"),
        },
        priority: 1015,
        desc: "INCSSPQ — increment shadow stack pointer 64",
    });

    // SAVEPREVSSP — Save Previous Shadow Stack Pointer
    table.push(ISelTableEntry {
        name: "cet_saveprevssp",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("shstk".into()),
        result: PatResult {
            opcode: X86Opcode::SAVEPREVSSP,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("shstk"),
        },
        priority: 1016,
        desc: "SAVEPREVSSP — save previous shadow stack pointer",
    });

    // RSTORSSP — Restore Shadow Stack Pointer
    table.push(ISelTableEntry {
        name: "cet_rstorssp",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("shstk".into()),
        result: PatResult {
            opcode: X86Opcode::RSTORSSP,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 2,
            requires_feature: Some("shstk"),
        },
        priority: 1017,
        desc: "RSTORSSP — restore shadow stack pointer",
    });

    // SETSSBSY — Set Shadow Stack Busy
    table.push(ISelTableEntry {
        name: "cet_setssbsy",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("shstk".into()),
        result: PatResult {
            opcode: X86Opcode::SETSSBSY,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("shstk"),
        },
        priority: 1018,
        desc: "SETSSBSY — set shadow stack busy flag",
    });

    // CLRSSBSY — Clear Shadow Stack Busy
    table.push(ISelTableEntry {
        name: "cet_clrssbsy",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("shstk".into()),
        result: PatResult {
            opcode: X86Opcode::CLRSSBSY,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("shstk"),
        },
        priority: 1019,
        desc: "CLRSSBSY — clear shadow stack busy flag",
    });

    // WRSSD — Write to Shadow Stack (32-bit)
    table.push(ISelTableEntry {
        name: "cet_wrssd",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("shstk".into()),
        result: PatResult {
            opcode: X86Opcode::WRSSD,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 2,
            requires_feature: Some("shstk"),
        },
        priority: 1020,
        desc: "WRSSD — write to shadow stack (32-bit)",
    });

    // WRSSQ — Write to Shadow Stack (64-bit)
    table.push(ISelTableEntry {
        name: "cet_wrssq",
        pattern: PatNode::MemRef,
        condition: PatCondition::All(vec![
            PatCondition::HasFeature("shstk".into()),
            PatCondition::Is64Bit,
        ]),
        result: PatResult {
            opcode: X86Opcode::WRSSQ,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 2,
            requires_feature: Some("shstk"),
        },
        priority: 1021,
        desc: "WRSSQ — write to shadow stack (64-bit)",
    });

    // WRUSSD — Write to User Shadow Stack (32-bit)
    table.push(ISelTableEntry {
        name: "cet_wrussd",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("shstk".into()),
        result: PatResult {
            opcode: X86Opcode::WRUSSD,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 2,
            requires_feature: Some("shstk"),
        },
        priority: 1022,
        desc: "WRUSSD — write to user shadow stack (32-bit)",
    });

    // WRUSSQ — Write to User Shadow Stack (64-bit)
    table.push(ISelTableEntry {
        name: "cet_wrussq",
        pattern: PatNode::MemRef,
        condition: PatCondition::All(vec![
            PatCondition::HasFeature("shstk".into()),
            PatCondition::Is64Bit,
        ]),
        result: PatResult {
            opcode: X86Opcode::WRUSSQ,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 2,
            requires_feature: Some("shstk"),
        },
        priority: 1023,
        desc: "WRUSSQ — write to user shadow stack (64-bit)",
    });

    // ================================================================
    // SECTION 3: Key Locker — AES Key Locker Instructions
    // ================================================================

    // LOADIWKEY — Load Internal Wrapping Key
    table.push(ISelTableEntry {
        name: "kl_loadiwkey",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("kl".into()),
        result: PatResult {
            opcode: X86Opcode::LOADIWKEY,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 20,
            requires_feature: Some("kl"),
        },
        priority: 1030,
        desc: "LOADIWKEY — load internal wrapping key for AES KL",
    });

    // AESENC128KL — AES Encrypt with 128-bit Key Locker
    table.push(ISelTableEntry {
        name: "kl_aesenc128kl",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("kl".into()),
        result: PatResult {
            opcode: X86Opcode::AESENC128KL,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 10,
            requires_feature: Some("kl"),
        },
        priority: 1031,
        desc: "AESENC128KL — AES encrypt one round, 128-bit key",
    });

    // AESDEC128KL — AES Decrypt with 128-bit Key Locker
    table.push(ISelTableEntry {
        name: "kl_aesdec128kl",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("kl".into()),
        result: PatResult {
            opcode: X86Opcode::AESDEC128KL,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 10,
            requires_feature: Some("kl"),
        },
        priority: 1032,
        desc: "AESDEC128KL — AES decrypt one round, 128-bit key",
    });

    // AESENC256KL — AES Encrypt with 256-bit Key Locker
    table.push(ISelTableEntry {
        name: "kl_aesenc256kl",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("kl".into()),
        result: PatResult {
            opcode: X86Opcode::AESENC256KL,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 12,
            requires_feature: Some("kl"),
        },
        priority: 1033,
        desc: "AESENC256KL — AES encrypt one round, 256-bit key",
    });

    // AESDEC256KL — AES Decrypt with 256-bit Key Locker
    table.push(ISelTableEntry {
        name: "kl_aesdec256kl",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("kl".into()),
        result: PatResult {
            opcode: X86Opcode::AESDEC256KL,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 12,
            requires_feature: Some("kl"),
        },
        priority: 1034,
        desc: "AESDEC256KL — AES decrypt one round, 256-bit key",
    });

    // AESENCWIDE128KL — Wide AES Encrypt 128-bit
    table.push(ISelTableEntry {
        name: "kl_aesencwide128kl",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("kl".into()),
        result: PatResult {
            opcode: X86Opcode::AESENCWIDE128KL,
            num_operands: 8,
            sets_flags: false,
            is_conditional: false,
            cost: 40,
            requires_feature: Some("kl-wide"),
        },
        priority: 1035,
        desc: "AESENCWIDE128KL — wide AES encrypt 8 blocks, 128-bit",
    });

    // AESDECWIDE128KL — Wide AES Decrypt 128-bit
    table.push(ISelTableEntry {
        name: "kl_aesdecwide128kl",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("kl".into()),
        result: PatResult {
            opcode: X86Opcode::AESDECWIDE128KL,
            num_operands: 8,
            sets_flags: false,
            is_conditional: false,
            cost: 40,
            requires_feature: Some("kl-wide"),
        },
        priority: 1036,
        desc: "AESDECWIDE128KL — wide AES decrypt 8 blocks, 128-bit",
    });

    // ENCODEKEY128 — Encode 128-bit Key
    table.push(ISelTableEntry {
        name: "kl_encodekey128",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("kl".into()),
        result: PatResult {
            opcode: X86Opcode::ENCODEKEY128,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 15,
            requires_feature: Some("kl"),
        },
        priority: 1037,
        desc: "ENCODEKEY128 — encode 128-bit AES key handle",
    });

    // ENCODEKEY256 — Encode 256-bit Key
    table.push(ISelTableEntry {
        name: "kl_encodekey256",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("kl".into()),
        result: PatResult {
            opcode: X86Opcode::ENCODEKEY256,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 18,
            requires_feature: Some("kl"),
        },
        priority: 1038,
        desc: "ENCODEKEY256 — encode 256-bit AES key handle",
    });

    // ================================================================
    // SECTION 4: UINTR — User Interrupts
    // ================================================================

    // SENDUIP — Send User Interprocessor Interrupt
    table.push(ISelTableEntry {
        name: "uintr_senduip",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("uintr".into()),
        result: PatResult {
            opcode: X86Opcode::SENDUIP,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 4,
            requires_feature: Some("uintr"),
        },
        priority: 1050,
        desc: "SENDUIP — send user IPI",
    });

    // UIRET — User Interrupt Return
    table.push(ISelTableEntry {
        name: "uintr_uiret",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("uintr".into()),
        result: PatResult {
            opcode: X86Opcode::UIRET,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 8,
            requires_feature: Some("uintr"),
        },
        priority: 1051,
        desc: "UIRET — user interrupt return",
    });

    // TESTUI — Test User Interrupt Flag
    table.push(ISelTableEntry {
        name: "uintr_testui",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("uintr".into()),
        result: PatResult {
            opcode: X86Opcode::TESTUI,
            num_operands: 0,
            sets_flags: true,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("uintr"),
        },
        priority: 1052,
        desc: "TESTUI — test user interrupt flag",
    });

    // CLUI — Clear User Interrupt Flag
    table.push(ISelTableEntry {
        name: "uintr_clui",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("uintr".into()),
        result: PatResult {
            opcode: X86Opcode::CLUI,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("uintr"),
        },
        priority: 1053,
        desc: "CLUI — clear user interrupt flag",
    });

    // STUI — Set User Interrupt Flag
    table.push(ISelTableEntry {
        name: "uintr_stui",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("uintr".into()),
        result: PatResult {
            opcode: X86Opcode::STUI,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("uintr"),
        },
        priority: 1054,
        desc: "STUI — set user interrupt flag",
    });

    // ================================================================
    // SECTION 5: FRED — Flexible Return and Event Delivery
    // ================================================================

    // ERETS — Enhanced Return from System call to same privilege level
    table.push(ISelTableEntry {
        name: "fred_erets",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("fred".into()),
        result: PatResult {
            opcode: X86Opcode::ERETS,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 15,
            requires_feature: Some("fred"),
        },
        priority: 1060,
        desc: "ERETS — enhanced return to same privilege (FRED)",
    });

    // ERETU — Enhanced Return from System call to User
    table.push(ISelTableEntry {
        name: "fred_eretu",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("fred".into()),
        result: PatResult {
            opcode: X86Opcode::ERETU,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 15,
            requires_feature: Some("fred"),
        },
        priority: 1061,
        desc: "ERETU — enhanced return to user (FRED)",
    });

    // LKGS — Load Kernel GS Base
    table.push(ISelTableEntry {
        name: "fred_lkgs",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("fred".into()),
        result: PatResult {
            opcode: X86Opcode::LKGS,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 3,
            requires_feature: Some("fred"),
        },
        priority: 1062,
        desc: "LKGS — load kernel GS base (FRED)",
    });

    // ================================================================
    // SECTION 6: CMPCCXADD — Compare and Conditionally Exchange and Add
    // ================================================================

    // All 16 condition code variants for CMPccXADD
    let cc_pairs: Vec<(&str, &str, u16)> = vec![
        ("cmpoxadd", "o", 1070),
        ("cmpnoxadd", "no", 1071),
        ("cmpbxadd", "b", 1072),
        ("cmpnbxadd", "nb", 1073),
        ("cmpzxadd", "z", 1074),
        ("cmpnzxadd", "nz", 1075),
        ("cmpbexadd", "be", 1076),
        ("cmpaxadd", "a", 1077),
        ("cmpsxadd", "s", 1078),
        ("cmpnsxadd", "ns", 1079),
        ("cmppxadd", "p", 1080),
        ("cmpnpxadd", "np", 1081),
        ("cmplxadd", "l", 1082),
        ("cmpgexadd", "ge", 1083),
        ("cmplexadd", "le", 1084),
        ("cmpgxadd", "g", 1085),
    ];

    for (name, cc, prio) in &cc_pairs {
        table.push(ISelTableEntry {
            name,
            pattern: PatNode::Any,
            condition: PatCondition::HasFeature("cmpccxadd".into()),
            result: PatResult {
                opcode: X86Opcode::CMPCCXADD,
                num_operands: 3,
                sets_flags: true,
                is_conditional: true,
                cost: 5,
                requires_feature: Some("cmpccxadd"),
            },
            priority: *prio,
            desc: Box::leak(
                format!(
                    "CMP{}XADD — compare and exchange+add on condition {}",
                    cc, cc
                )
                .into_boxed_str(),
            ),
        });
    }

    // ================================================================
    // SECTION 7: RAO-INT — Remote Atomic Operations
    // ================================================================

    // AADD — Atomic ADD (byte, word, dword, qword)
    let rao_variants = [
        ("rao_aadd_b", "AADD", "byte", 1090),
        ("rao_aadd_w", "AADD", "word", 1091),
        ("rao_aadd_d", "AADD", "dword", 1092),
        ("rao_aadd_q", "AADD", "qword", 1093),
        ("rao_aand_b", "AAND", "byte", 1094),
        ("rao_aand_w", "AAND", "word", 1095),
        ("rao_aand_d", "AAND", "dword", 1096),
        ("rao_aand_q", "AAND", "qword", 1097),
        ("rao_aor_b", "AOR", "byte", 1098),
        ("rao_aor_w", "AOR", "word", 1099),
        ("rao_aor_d", "AOR", "dword", 1100),
        ("rao_aor_q", "AOR", "qword", 1101),
        ("rao_axor_b", "AXOR", "byte", 1102),
        ("rao_axor_w", "AXOR", "word", 1103),
        ("rao_axor_d", "AXOR", "dword", 1104),
        ("rao_axor_q", "AXOR", "qword", 1105),
    ];

    for (name, op_name, size, prio) in &rao_variants {
        let opcode = match *op_name {
            "AADD" => X86Opcode::AADD,
            "AAND" => X86Opcode::AAND,
            "AOR" => X86Opcode::AOR,
            "AXOR" => X86Opcode::AXOR,
            _ => X86Opcode::AADD,
        };
        table.push(ISelTableEntry {
            name,
            pattern: PatNode::Any,
            condition: PatCondition::HasFeature("rao-int".into()),
            result: PatResult {
                opcode,
                num_operands: 2,
                sets_flags: false,
                is_conditional: false,
                cost: 8,
                requires_feature: Some("rao-int"),
            },
            priority: *prio,
            desc: Box::leak(
                format!("{} - atomic {} ({})", op_name, op_name, size).into_boxed_str(),
            ),
        });
    }

    // ================================================================
    // SECTION 8: PREFETCHITI — Prefetch Instructions to ITLB
    // ================================================================

    // PREFETCHIT0 — Prefetch code to all cache levels
    table.push(ISelTableEntry {
        name: "prefetchiti_t0",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("prefetchiti".into()),
        result: PatResult {
            opcode: X86Opcode::PREFETCHIT0,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("prefetchiti"),
        },
        priority: 1110,
        desc: "PREFETCHIT0 — prefetch code to ITLB temporal",
    });

    // PREFETCHIT1 — Prefetch code to L2 and higher
    table.push(ISelTableEntry {
        name: "prefetchiti_t1",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("prefetchiti".into()),
        result: PatResult {
            opcode: X86Opcode::PREFETCHIT1,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("prefetchiti"),
        },
        priority: 1111,
        desc: "PREFETCHIT1 — prefetch code to ITLB non-temporal",
    });

    // ================================================================
    // SECTION 9: MSRLIST — MSR List Read/Write
    // ================================================================

    // RDMSRLIST — Read list of MSRs
    table.push(ISelTableEntry {
        name: "msrlist_rdmsrlist",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("msrlist".into()),
        result: PatResult {
            opcode: X86Opcode::RDMSRLIST,
            num_operands: 3,
            sets_flags: false,
            is_conditional: false,
            cost: 10,
            requires_feature: Some("msrlist"),
        },
        priority: 1120,
        desc: "RDMSRLIST — read list of model-specific registers",
    });

    // WRMSRLIST — Write list of MSRs
    table.push(ISelTableEntry {
        name: "msrlist_wrmsrlist",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("msrlist".into()),
        result: PatResult {
            opcode: X86Opcode::WRMSRLIST,
            num_operands: 3,
            sets_flags: false,
            is_conditional: false,
            cost: 10,
            requires_feature: Some("msrlist"),
        },
        priority: 1121,
        desc: "WRMSRLIST — write list of model-specific registers",
    });

    // ================================================================
    // SECTION 10: HRESET — History Reset
    // ================================================================

    table.push(ISelTableEntry {
        name: "hreset",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("hreset".into()),
        result: PatResult {
            opcode: X86Opcode::HRESET,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 50,
            requires_feature: Some("hreset"),
        },
        priority: 1130,
        desc: "HRESET — reset prediction history",
    });

    // ================================================================
    // SECTION 11: SERIALIZE — Serialize Instruction Execution
    // ================================================================

    table.push(ISelTableEntry {
        name: "serialize",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("serialize".into()),
        result: PatResult {
            opcode: X86Opcode::SERIALIZE,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 10,
            requires_feature: Some("serialize"),
        },
        priority: 1140,
        desc: "SERIALIZE — serialize instruction execution",
    });

    // ================================================================
    // SECTION 12: CLDEMOTE — Cache Line Demote
    // ================================================================

    table.push(ISelTableEntry {
        name: "cldemote",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("cldemote".into()),
        result: PatResult {
            opcode: X86Opcode::CLDEMOTE,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 5,
            requires_feature: Some("cldemote"),
        },
        priority: 1150,
        desc: "CLDEMOTE — demote cache line to more remote cache",
    });

    // ================================================================
    // SECTION 13: MOVDIRI / MOVDIR64B — Direct Store Instructions
    // ================================================================

    // MOVDIRI — Move Doubleword as Direct Store (32-bit)
    table.push(ISelTableEntry {
        name: "movdiri32",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("movdiri".into()),
        result: PatResult {
            opcode: X86Opcode::MOVDIRI,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 4,
            requires_feature: Some("movdiri"),
        },
        priority: 1160,
        desc: "MOVDIRI — move dword as direct store",
    });

    // MOVDIR64B — Move 64 Bytes as Direct Store
    table.push(ISelTableEntry {
        name: "movdir64b",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("movdir64b".into()),
        result: PatResult {
            opcode: X86Opcode::MOVDIR64B,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 20,
            requires_feature: Some("movdir64b"),
        },
        priority: 1161,
        desc: "MOVDIR64B — move 64 bytes as direct store",
    });

    // ================================================================
    // SECTION 14: ENQCMD / ENQCMDS — Enqueue Command
    // ================================================================

    // ENQCMD — Enqueue Command (user)
    table.push(ISelTableEntry {
        name: "enqcmd",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("enqcmd".into()),
        result: PatResult {
            opcode: X86Opcode::ENQCMD,
            num_operands: 2,
            sets_flags: true,
            is_conditional: false,
            cost: 15,
            requires_feature: Some("enqcmd"),
        },
        priority: 1170,
        desc: "ENQCMD — enqueue command (user mode)",
    });

    // ENQCMDS — Enqueue Command (supervisor)
    table.push(ISelTableEntry {
        name: "enqcmds",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("enqcmd".into()),
        result: PatResult {
            opcode: X86Opcode::ENQCMDS,
            num_operands: 2,
            sets_flags: true,
            is_conditional: false,
            cost: 15,
            requires_feature: Some("enqcmd"),
        },
        priority: 1171,
        desc: "ENQCMDS — enqueue command (supervisor mode)",
    });

    // ================================================================
    // SECTION 15: TSXLDTRK — TSX Load Address Tracking
    // ================================================================

    // XSUSLDTRK — Suspend Load Address Tracking
    table.push(ISelTableEntry {
        name: "tsxldtrk_xsusldtrk",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("tsxldtrk".into()),
        result: PatResult {
            opcode: X86Opcode::XSUSLDTRK,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("tsxldtrk"),
        },
        priority: 1180,
        desc: "XSUSLDTRK — suspend TSX load address tracking",
    });

    // XRESLDTRK — Resume Load Address Tracking
    table.push(ISelTableEntry {
        name: "tsxldtrk_xresldtrk",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("tsxldtrk".into()),
        result: PatResult {
            opcode: X86Opcode::XRESLDTRK,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("tsxldtrk"),
        },
        priority: 1181,
        desc: "XRESLDTRK — resume TSX load address tracking",
    });

    // ================================================================
    // SECTION 16: Additional System Instructions
    // ================================================================

    // WRMSRNS — Write MSR Non-Serializing
    table.push(ISelTableEntry {
        name: "wrmsrns",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("wrmsrns".into()),
        result: PatResult {
            opcode: X86Opcode::WRMSRNS,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 5,
            requires_feature: Some("wrmsrns"),
        },
        priority: 1190,
        desc: "WRMSRNS — write MSR non-serializing",
    });

    // RDMSR — Read MSR
    table.push(ISelTableEntry {
        name: "rdmsr",
        pattern: PatNode::Any,
        condition: PatCondition::Always,
        result: PatResult {
            opcode: X86Opcode::RDMSR,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 15,
            requires_feature: None,
        },
        priority: 1191,
        desc: "RDMSR — read model-specific register",
    });

    // WRMSR — Write MSR
    table.push(ISelTableEntry {
        name: "wrmsr",
        pattern: PatNode::Any,
        condition: PatCondition::Always,
        result: PatResult {
            opcode: X86Opcode::WRMSR,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 15,
            requires_feature: None,
        },
        priority: 1192,
        desc: "WRMSR — write model-specific register",
    });

    // RDPMC — Read Performance Monitoring Counter
    table.push(ISelTableEntry {
        name: "rdpmc",
        pattern: PatNode::Any,
        condition: PatCondition::Always,
        result: PatResult {
            opcode: X86Opcode::RDPMC,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 20,
            requires_feature: None,
        },
        priority: 1193,
        desc: "RDPMC — read performance monitoring counter",
    });

    // RDTSC — Read Time-Stamp Counter
    table.push(ISelTableEntry {
        name: "rdtsc",
        pattern: PatNode::Any,
        condition: PatCondition::Always,
        result: PatResult {
            opcode: X86Opcode::RDTSC,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 10,
            requires_feature: None,
        },
        priority: 1194,
        desc: "RDTSC — read time-stamp counter",
    });

    // RDTSCP — Read Time-Stamp Counter and Processor ID
    table.push(ISelTableEntry {
        name: "rdtscp",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("rdtscp".into()),
        result: PatResult {
            opcode: X86Opcode::RDTSCP,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 15,
            requires_feature: Some("rdtscp"),
        },
        priority: 1195,
        desc: "RDTSCP — read time-stamp counter and processor ID",
    });

    // RDPID — Read Processor ID
    table.push(ISelTableEntry {
        name: "rdpid",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("rdpid".into()),
        result: PatResult {
            opcode: X86Opcode::RDPID,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("rdpid"),
        },
        priority: 1196,
        desc: "RDPID — read processor ID",
    });

    // INVPCID — Invalidate Process-Context Identifier
    table.push(ISelTableEntry {
        name: "invpcid",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("invpcid".into()),
        result: PatResult {
            opcode: X86Opcode::INVPCID,
            num_operands: 2,
            sets_flags: false,
            is_conditional: false,
            cost: 50,
            requires_feature: Some("invpcid"),
        },
        priority: 1197,
        desc: "INVPCID — invalidate process-context identifier",
    });

    // ================================================================
    // SECTION 17: CLWB / CLFLUSHOPT / CLFLUSH — Cache Line Management
    // ================================================================

    table.push(ISelTableEntry {
        name: "clwb",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("clwb".into()),
        result: PatResult {
            opcode: X86Opcode::CLWB,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 3,
            requires_feature: Some("clwb"),
        },
        priority: 1200,
        desc: "CLWB — cache line write back",
    });

    table.push(ISelTableEntry {
        name: "clflushopt",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("clflushopt".into()),
        result: PatResult {
            opcode: X86Opcode::CLFLUSHOPT,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 3,
            requires_feature: Some("clflushopt"),
        },
        priority: 1201,
        desc: "CLFLUSHOPT — optimized cache line flush",
    });

    table.push(ISelTableEntry {
        name: "clflush",
        pattern: PatNode::MemRef,
        condition: PatCondition::Always,
        result: PatResult {
            opcode: X86Opcode::CLFLUSH,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 5,
            requires_feature: None,
        },
        priority: 1202,
        desc: "CLFLUSH — flush cache line",
    });

    // ================================================================
    // SECTION 18: PREFETCHW / PREFETCHWT1 — Prefetch Variants
    // ================================================================

    table.push(ISelTableEntry {
        name: "prefetchw",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("3dnowprefetch".into()),
        result: PatResult {
            opcode: X86Opcode::PREFETCHW,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("3dnowprefetch"),
        },
        priority: 1210,
        desc: "PREFETCHW — prefetch data into cache with intent to write",
    });

    table.push(ISelTableEntry {
        name: "prefetchwt1",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("prefetchwt1".into()),
        result: PatResult {
            opcode: X86Opcode::PREFETCHWT1,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("prefetchwt1"),
        },
        priority: 1211,
        desc: "PREFETCHWT1 — prefetch hint T1 with intent to write",
    });

    // ================================================================
    // SECTION 19: MONITORX / MWAITX — AMD Lightweight Profiling
    // ================================================================

    table.push(ISelTableEntry {
        name: "monitorx",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("monitorx".into()),
        result: PatResult {
            opcode: X86Opcode::MONITORX,
            num_operands: 3,
            sets_flags: false,
            is_conditional: false,
            cost: 10,
            requires_feature: Some("monitorx"),
        },
        priority: 1220,
        desc: "MONITORX — set up monitor address range (AMD)",
    });

    table.push(ISelTableEntry {
        name: "mwaitx",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("mwaitx".into()),
        result: PatResult {
            opcode: X86Opcode::MWAITX,
            num_operands: 3,
            sets_flags: false,
            is_conditional: false,
            cost: 10,
            requires_feature: Some("mwaitx"),
        },
        priority: 1221,
        desc: "MWAITX — monitor wait with timeout (AMD)",
    });

    // ================================================================
    // SECTION 20: CLZERO / RDPRU — AMD Extensions
    // ================================================================

    table.push(ISelTableEntry {
        name: "clzero",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("clzero".into()),
        result: PatResult {
            opcode: X86Opcode::CLZERO,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 5,
            requires_feature: Some("clzero"),
        },
        priority: 1230,
        desc: "CLZERO — zero out cache line (AMD)",
    });

    table.push(ISelTableEntry {
        name: "rdpru",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("rdpru".into()),
        result: PatResult {
            opcode: X86Opcode::RDPRU,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 5,
            requires_feature: Some("rdpru"),
        },
        priority: 1231,
        desc: "RDPRU — read processor register (AMD)",
    });

    // ================================================================
    // SECTION 21: PTWRITE — Processor Trace Write
    // ================================================================

    table.push(ISelTableEntry {
        name: "ptwrite32",
        pattern: PatNode::Any,
        condition: PatCondition::All(vec![
            PatCondition::HasFeature("ptwrite".into()),
            PatCondition::Is32Bit,
        ]),
        result: PatResult {
            opcode: X86Opcode::PTWRITE,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 5,
            requires_feature: Some("ptwrite"),
        },
        priority: 1240,
        desc: "PTWRITE — write data to processor trace (32-bit)",
    });

    table.push(ISelTableEntry {
        name: "ptwrite64",
        pattern: PatNode::Any,
        condition: PatCondition::All(vec![
            PatCondition::HasFeature("ptwrite".into()),
            PatCondition::Is64Bit,
        ]),
        result: PatResult {
            opcode: X86Opcode::PTWRITE,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 5,
            requires_feature: Some("ptwrite"),
        },
        priority: 1241,
        desc: "PTWRITE — write data to processor trace (64-bit)",
    });

    // ================================================================
    // SECTION 22: WBNOINVD — Write Back and Do Not Invalidate Cache
    // ================================================================

    table.push(ISelTableEntry {
        name: "wbnoinvd",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("wbnoinvd".into()),
        result: PatResult {
            opcode: X86Opcode::WBNOINVD,
            num_operands: 0,
            sets_flags: false,
            is_conditional: false,
            cost: 100,
            requires_feature: Some("wbnoinvd"),
        },
        priority: 1250,
        desc: "WBNOINVD — write back and do not invalidate cache",
    });

    // ================================================================
    // SECTION 23: TPAUSE / UMONITOR / UMWAIT — User Wait
    // ================================================================

    table.push(ISelTableEntry {
        name: "tpause",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("waitpkg".into()),
        result: PatResult {
            opcode: X86Opcode::TPAUSE,
            num_operands: 2,
            sets_flags: true,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("waitpkg"),
        },
        priority: 1260,
        desc: "TPAUSE — timed pause (waitpkg)",
    });

    table.push(ISelTableEntry {
        name: "umonitor",
        pattern: PatNode::MemRef,
        condition: PatCondition::HasFeature("waitpkg".into()),
        result: PatResult {
            opcode: X86Opcode::UMONITOR,
            num_operands: 1,
            sets_flags: false,
            is_conditional: false,
            cost: 5,
            requires_feature: Some("waitpkg"),
        },
        priority: 1261,
        desc: "UMONITOR — user monitor address (waitpkg)",
    });

    table.push(ISelTableEntry {
        name: "umwait",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("waitpkg".into()),
        result: PatResult {
            opcode: X86Opcode::UMWAIT,
            num_operands: 2,
            sets_flags: true,
            is_conditional: false,
            cost: 1,
            requires_feature: Some("waitpkg"),
        },
        priority: 1262,
        desc: "UMWAIT — user monitor wait (waitpkg)",
    });

    // ================================================================
    // SECTION 24: GFNI — Galois Field New Instructions
    // ================================================================

    table.push(ISelTableEntry {
        name: "gf2p8affineinvqb",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("gfni".into()),
        result: PatResult {
            opcode: X86Opcode::GF2P8AFFINEINVQB,
            num_operands: 3,
            sets_flags: false,
            is_conditional: false,
            cost: 5,
            requires_feature: Some("gfni"),
        },
        priority: 1270,
        desc: "GF2P8AFFINEINVQB — Galois field affine inverse",
    });

    table.push(ISelTableEntry {
        name: "gf2p8affineqb",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("gfni".into()),
        result: PatResult {
            opcode: X86Opcode::GF2P8AFFINEQB,
            num_operands: 3,
            sets_flags: false,
            is_conditional: false,
            cost: 5,
            requires_feature: Some("gfni"),
        },
        priority: 1271,
        desc: "GF2P8AFFINEQB — Galois field affine transform",
    });

    table.push(ISelTableEntry {
        name: "gf2p8mulb",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("gfni".into()),
        result: PatResult {
            opcode: X86Opcode::GF2P8MULB,
            num_operands: 3,
            sets_flags: false,
            is_conditional: false,
            cost: 3,
            requires_feature: Some("gfni"),
        },
        priority: 1272,
        desc: "GF2P8MULB — Galois field multiply byte",
    });

    // ================================================================
    // SECTION 25: Additional WAITPKG / UMONITOR / UMWAIT variants
    // ================================================================

    // Additional UMWAIT with state controls for C0.1 and C0.2
    table.push(ISelTableEntry {
        name: "umwait_c01",
        pattern: PatNode::Any,
        condition: PatCondition::HasFeature("waitpkg".into()),
        result: PatResult {
            opcode: X86Opcode::UMWAIT,
            num_operands: 3,
            sets_flags: true,
            is_conditional: false,
            cost: 2,
            requires_feature: Some("waitpkg"),
        },
        priority: 1280,
        desc: "UMWAIT C0.1 — user monitor wait with power state control",
    });

    // ================================================================
    // SECTION 26: CMPCCXADD — Repeat for indexed forms (memory+register)
    // ================================================================

    // CMPCCXADD with memory destination
    table.push(ISelTableEntry {
        name: "cmpccxadd_mem",
        pattern: PatNode::Tree(
            Box::new(PatNode::BinOp("cmpxchg".into())),
            vec![Box::new(PatNode::MemRef)],
        ),
        condition: PatCondition::HasFeature("cmpccxadd".into()),
        result: PatResult {
            opcode: X86Opcode::CMPCCXADD,
            num_operands: 3,
            sets_flags: true,
            is_conditional: true,
            cost: 8,
            requires_feature: Some("cmpccxadd"),
        },
        priority: 1290,
        desc: "CMPccXADD [mem], reg — compare and conditionally exchange+add to memory",
    });

    table
}

// ============================================================================
// Final ISel Engine
// ============================================================================

/// Final remaining ISel engine for MPX, CET, Key Locker, and other new extensions.
pub struct FinalIselEngine {
    pub patterns: Vec<ISelTableEntry>,
    pub coverage: PatternCoverage,
    pub stats: ISelStats,
    /// Available features on the target CPU.
    pub features: HashSet<String>,
}

impl FinalIselEngine {
    /// Create with the given feature set.
    pub fn new(features: HashSet<String>) -> Self {
        let patterns = build_final_isel_table();
        let mut coverage = PatternCoverage::new();
        for p in &patterns {
            if let Some(feat) = p.result.requires_feature {
                coverage.add_coverage(feat);
            }
            coverage.add_coverage(p.name);
        }
        FinalIselEngine {
            patterns,
            coverage,
            stats: ISelStats::new(),
            features,
        }
    }

    /// Look up patterns for a given instruction.
    pub fn lookup_by_opcode(&self, opcode: X86Opcode) -> Vec<&ISelTableEntry> {
        self.patterns
            .iter()
            .filter(|p| p.result.opcode == opcode)
            .collect()
    }

    /// Look up patterns requiring a specific feature.
    pub fn patterns_by_feature(&self, feature: &str) -> Vec<&ISelTableEntry> {
        self.patterns
            .iter()
            .filter(|p| p.result.requires_feature == Some(feature))
            .collect()
    }

    /// Check if the target has a given feature.
    pub fn has_feature(&self, feature: &str) -> bool {
        self.features.contains(feature)
    }

    /// Total number of patterns.
    pub fn pattern_count(&self) -> usize {
        self.patterns.len()
    }

    /// Generate a coverage report.
    pub fn coverage_report(&self) -> String {
        let mut report = String::new();
        report.push_str(&format!("Final ISel — {} patterns\n", self.patterns.len()));
        let features = [
            "mpx",
            "cet",
            "shstk",
            "kl",
            "kl-wide",
            "uintr",
            "fred",
            "cmpccxadd",
            "rao-int",
            "prefetchiti",
            "msrlist",
            "hreset",
            "serialize",
            "cldemote",
            "movdiri",
            "movdir64b",
            "enqcmd",
            "tsxldtrk",
            "gfni",
            "waitpkg",
            "ptwrite",
            "clwb",
            "clflushopt",
            "wbnoinvd",
            "rdtscp",
            "rdpid",
        ];
        for feat in &features {
            if self.has_feature(feat) {
                let count = self.patterns_by_feature(feat).len();
                report.push_str(&format!("  {}: {} patterns\n", feat, count));
            }
        }
        report
    }

    /// Get features that have pattern coverage.
    pub fn covered_features(&self) -> Vec<String> {
        let mut feats = HashSet::new();
        for p in &self.patterns {
            if let Some(f) = p.result.requires_feature {
                feats.insert(f.to_string());
            }
        }
        let mut v: Vec<String> = feats.into_iter().collect();
        v.sort();
        v
    }
}

impl Default for FinalIselEngine {
    fn default() -> Self {
        let mut features = HashSet::new();
        for feat in &[
            "mpx",
            "cet",
            "shstk",
            "kl",
            "kl-wide",
            "uintr",
            "fred",
            "cmpccxadd",
            "rao-int",
            "prefetchiti",
            "msrlist",
            "hreset",
            "serialize",
            "cldemote",
            "movdiri",
            "movdir64b",
            "enqcmd",
            "tsxldtrk",
            "gfni",
            "waitpkg",
            "ptwrite",
            "clwb",
            "clflushopt",
            "wbnoinvd",
            "rdtscp",
            "rdpid",
            "invpcid",
            "3dnowprefetch",
            "prefetchwt1",
            "monitorx",
            "mwaitx",
            "clzero",
            "rdpru",
            "wrmsrns",
        ] {
            features.insert(feat.to_string());
        }
        Self::new(features)
    }
}

// ============================================================================
// Tests
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_final_table_not_empty() {
        let table = build_final_isel_table();
        assert!(!table.is_empty(), "Final ISel table should not be empty");
        assert!(
            table.len() > 50,
            "Should have 50+ patterns, got {}",
            table.len()
        );
    }

    #[test]
    fn test_mpx_patterns() {
        let table = build_final_isel_table();
        let mpx: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("mpx"))
            .collect();
        assert!(
            mpx.len() >= 8,
            "Should have 8+ MPX patterns, got {}",
            mpx.len()
        );
    }

    #[test]
    fn test_cet_patterns() {
        let table = build_final_isel_table();
        let cet: Vec<_> = table
            .iter()
            .filter(|p| {
                p.result.requires_feature == Some("cet")
                    || p.result.requires_feature == Some("shstk")
            })
            .collect();
        assert!(
            cet.len() >= 12,
            "Should have 12+ CET patterns, got {}",
            cet.len()
        );
    }

    #[test]
    fn test_key_locker_patterns() {
        let table = build_final_isel_table();
        let kl: Vec<_> = table
            .iter()
            .filter(|p| {
                p.result.requires_feature == Some("kl")
                    || p.result.requires_feature == Some("kl-wide")
            })
            .collect();
        assert!(
            kl.len() >= 8,
            "Should have 8+ Key Locker patterns, got {}",
            kl.len()
        );
    }

    #[test]
    fn test_uintr_patterns() {
        let table = build_final_isel_table();
        let uintr: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("uintr"))
            .collect();
        assert!(
            uintr.len() >= 5,
            "Should have 5+ UINTR patterns, got {}",
            uintr.len()
        );
    }

    #[test]
    fn test_fred_patterns() {
        let table = build_final_isel_table();
        let fred: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("fred"))
            .collect();
        assert!(
            fred.len() >= 3,
            "Should have 3+ FRED patterns, got {}",
            fred.len()
        );
    }

    #[test]
    fn test_cmpccxadd_all_16_conditions() {
        let table = build_final_isel_table();
        let cca: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("cmpccxadd"))
            .collect();
        assert!(
            cca.len() >= 17,
            "Should have 17+ CMPccXADD patterns (16 conds + mem), got {}",
            cca.len()
        );
    }

    #[test]
    fn test_rao_int_all_16_variants() {
        let table = build_final_isel_table();
        let rao: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("rao-int"))
            .collect();
        assert!(
            rao.len() >= 16,
            "Should have 16+ RAO-INT patterns, got {}",
            rao.len()
        );
    }

    #[test]
    fn test_prefetchiti_patterns() {
        let table = build_final_isel_table();
        let pfi: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("prefetchiti"))
            .collect();
        assert!(
            pfi.len() >= 2,
            "Should have 2 PREFETCHITI patterns, got {}",
            pfi.len()
        );
    }

    #[test]
    fn test_msrlist_patterns() {
        let table = build_final_isel_table();
        let ml: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("msrlist"))
            .collect();
        assert!(
            ml.len() >= 2,
            "Should have 2 MSRLIST patterns, got {}",
            ml.len()
        );
    }

    #[test]
    fn test_other_patterns() {
        let table = build_final_isel_table();
        let other: Vec<_> = table
            .iter()
            .filter(|p| {
                p.result.requires_feature == Some("hreset")
                    || p.result.requires_feature == Some("serialize")
                    || p.result.requires_feature == Some("cldemote")
            })
            .collect();
        assert!(
            other.len() >= 3,
            "Should have HRESET/SERIALIZE/CLDEMOTE patterns"
        );
    }

    #[test]
    fn test_movdiri_patterns() {
        let table = build_final_isel_table();
        let md: Vec<_> = table
            .iter()
            .filter(|p| {
                p.result.requires_feature == Some("movdiri")
                    || p.result.requires_feature == Some("movdir64b")
            })
            .collect();
        assert!(
            md.len() >= 2,
            "Should have 2 MOVDIRI patterns, got {}",
            md.len()
        );
    }

    #[test]
    fn test_enqcmd_patterns() {
        let table = build_final_isel_table();
        let eq: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("enqcmd"))
            .collect();
        assert!(
            eq.len() >= 2,
            "Should have 2 ENQCMD patterns, got {}",
            eq.len()
        );
    }

    #[test]
    fn test_tsxldtrk_patterns() {
        let table = build_final_isel_table();
        let tsx: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("tsxldtrk"))
            .collect();
        assert!(
            tsx.len() >= 2,
            "Should have 2 TSXLDTRK patterns, got {}",
            tsx.len()
        );
    }

    #[test]
    fn test_gfni_patterns() {
        let table = build_final_isel_table();
        let gf: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("gfni"))
            .collect();
        assert!(
            gf.len() >= 3,
            "Should have 3 GFNI patterns, got {}",
            gf.len()
        );
    }

    #[test]
    fn test_waitpkg_patterns() {
        let table = build_final_isel_table();
        let wp: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("waitpkg"))
            .collect();
        assert!(
            wp.len() >= 4,
            "Should have 4+ WAITPKG patterns, got {}",
            wp.len()
        );
    }

    #[test]
    fn test_clwb_clflushopt_patterns() {
        let table = build_final_isel_table();
        let cl: Vec<_> = table
            .iter()
            .filter(|p| {
                p.result.requires_feature == Some("clwb")
                    || p.result.requires_feature == Some("clflushopt")
            })
            .collect();
        assert!(cl.len() >= 2, "Should have CLWB and CLFLUSHOPT patterns");
    }

    #[test]
    fn test_rdmsr_wrmsr_patterns() {
        let table = build_final_isel_table();
        let msr: Vec<_> = table
            .iter()
            .filter(|p| p.result.opcode == X86Opcode::RDMSR || p.result.opcode == X86Opcode::WRMSR)
            .collect();
        assert!(!msr.is_empty(), "Should have RDMSR/WRMSR patterns");
    }

    #[test]
    fn test_rdtsc_rdtscp_patterns() {
        let table = build_final_isel_table();
        let tsc: Vec<_> = table
            .iter()
            .filter(|p| p.result.opcode == X86Opcode::RDTSC || p.result.opcode == X86Opcode::RDTSCP)
            .collect();
        assert!(tsc.len() >= 2, "Should have RDTSC/RDTSCP patterns");
    }

    #[test]
    fn test_engine_lookup_by_opcode() {
        let engine = FinalIselEngine::default();
        let mpx = engine.lookup_by_opcode(X86Opcode::BNDMK);
        assert!(!mpx.is_empty());
    }

    #[test]
    fn test_engine_patterns_by_feature() {
        let engine = FinalIselEngine::default();
        let mpx = engine.patterns_by_feature("mpx");
        assert!(mpx.len() >= 8);
    }

    #[test]
    fn test_engine_has_feature() {
        let engine = FinalIselEngine::default();
        assert!(engine.has_feature("mpx"));
        assert!(engine.has_feature("cet"));
        assert!(engine.has_feature("kl"));
        assert!(!engine.has_feature("nonexistent_feature"));
    }

    #[test]
    fn test_engine_coverage_report() {
        let engine = FinalIselEngine::default();
        let report = engine.coverage_report();
        assert!(report.contains("Final ISel"));
        assert!(report.contains("mpx"));
        assert!(report.contains("cet"));
        assert!(report.contains("kl"));
    }

    #[test]
    fn test_engine_covered_features() {
        let engine = FinalIselEngine::default();
        let feats = engine.covered_features();
        assert!(!feats.is_empty());
        assert!(feats.contains(&"mpx".to_string()));
        assert!(feats.contains(&"shstk".to_string()));
    }

    #[test]
    fn test_all_patterns_have_descriptions() {
        let table = build_final_isel_table();
        for p in &table {
            assert!(
                !p.desc.is_empty(),
                "Pattern should have description: {}",
                p.name
            );
        }
    }

    #[test]
    fn test_all_patterns_have_valid_cost() {
        let table = build_final_isel_table();
        for p in &table {
            assert!(
                p.result.cost > 0
                    || p.result.opcode == X86Opcode::ENDBR32
                    || p.result.opcode == X86Opcode::ENDBR64,
                "Pattern {} should have positive cost (or be ENDBR)",
                p.name
            );
        }
    }

    #[test]
    fn test_ptwrite_patterns() {
        let table = build_final_isel_table();
        let ptw: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("ptwrite"))
            .collect();
        assert!(
            ptw.len() >= 2,
            "Should have PTWRITE 32/64 patterns, got {}",
            ptw.len()
        );
    }

    #[test]
    fn test_wbnoinvd_pattern() {
        let table = build_final_isel_table();
        let wb: Vec<_> = table
            .iter()
            .filter(|p| p.result.requires_feature == Some("wbnoinvd"))
            .collect();
        assert!(wb.len() >= 1, "Should have WBNOINVD pattern");
    }

    #[test]
    fn test_cmpccxadd_condition_list() {
        // Verify all 16 condition codes are present via their unique priorities
        let table = build_final_isel_table();
        let cc_priorities: Vec<u16> = table
            .iter()
            .filter(|p| p.name.starts_with("cmp") && p.name.ends_with("xadd"))
            .map(|p| p.priority)
            .collect();
        assert_eq!(cc_priorities.len(), 17); // 16 condition codes + 1 memory form
    }
}