llvm-native-core 0.1.15

//! AArch64 SME2.1 Instruction Selection Table
//!
//! SME2.1 extends SME2 with outer product operations using FP8 accumulation,
//! multi-vector BF16 to FP32 dot product, streaming SVE with SME2 overlays,
//! 512-bit ZA storage access patterns, tile transpose operations, and
//! structured sparsity with 2:4 pattern support.
//!
//! Clean-room behavioural reconstruction from:
//! - Arm Architecture Reference Manual Supplement — SME2.1
//! - Arm A64 ISA XML descriptions
//! - Black-box oracle interrogation
//! - Zero LLVM source code consultation

use crate::arm::arm_instr_info::ArmOpcode;
use crate::opcode::Opcode;

use super::arm_isel_table::IselPattern;

// ============================================================================
// SME2.1 ISel Table
// ============================================================================

/// SME2.1 ISel statistics.
#[derive(Debug, Default, Clone)]
pub struct SME21IselStats {
    pub total_patterns: usize,
    pub fp8_outer_product: usize,
    pub bf16_fp32_dot: usize,
    pub streaming_sve_overlay: usize,
    pub za_512bit_access: usize,
    pub tile_transpose: usize,
    pub structured_sparsity: usize,
}

/// Complete SME2.1 ISel table.
pub struct SME21IselTable {
    /// All SME2.1 ISel patterns.
    pub patterns: Vec<IselPattern>,
    /// ISel statistics.
    pub stats: SME21IselStats,
}

/// Create the SME2.1 ISel pattern table.
pub fn sme21_isel_table() -> Vec<IselPattern> {
    let mut table = Vec::new();

    // ================================================================
    // SME2.1 — Outer Product with FP8 Accumulation
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description:
            "fop8a za0.s, p0/m, p0/m, z0.b, z1.b — SME2.1 FP8→FP32 outer product accumulate",
        result_opcode: ArmOpcode::FOP8A_SME21,
        priority: 100,
        num_operands: 5,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_fp8"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description:
            "fop8a za0.h, p0/m, p0/m, z0.b, z1.b — SME2.1 FP8→FP16 outer product accumulate",
        result_opcode: ArmOpcode::FOP8A_SME21,
        priority: 101,
        num_operands: 5,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_fp8"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FMul,
        description:
            "fop8a za0.s(zeroed), p0/m, p0/m, z0.b, z1.b — SME2.1 FP8 fresh outer product FP32",
        result_opcode: ArmOpcode::FOP8A_SME21,
        priority: 110,
        num_operands: 5,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_fp8"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FMul,
        description:
            "fop8a za0.h(zeroed), p0/m, p0/m, z0.b, z1.b — SME2.1 FP8 fresh outer product FP16",
        result_opcode: ArmOpcode::FOP8A_SME21,
        priority: 111,
        num_operands: 5,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_fp8"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — Outer Product with FP8 Subtraction
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::FSub,
        description: "fop8s za0.s, p0/m, p0/m, z0.b, z1.b — SME2.1 FP8 outer product subtract FP32",
        result_opcode: ArmOpcode::FOP8S_SME21,
        priority: 100,
        num_operands: 5,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_fp8"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FSub,
        description: "fop8s za0.h, p0/m, p0/m, z0.b, z1.b — SME2.1 FP8 outer product subtract FP16",
        result_opcode: ArmOpcode::FOP8S_SME21,
        priority: 101,
        num_operands: 5,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_fp8"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — Multi-Vector BF16 to FP32 Dot Product
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description: "bfdot za0.s, {z0.h-z3.h}, {z4.h-z7.h} — SME2.1 multi-vec BF16→FP32 dot product accumulate",
        result_opcode: ArmOpcode::BFDOT_SME21,
        priority: 100,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_bfdot"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FMul,
        description: "bfdot za0.s(zerod), {z0.h-z3.h}, {z4.h-z7.h} — SME2.1 multi-vec BF16 fresh dot product",
        result_opcode: ArmOpcode::BFDOT_SME21,
        priority: 110,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_bfdot"),
        cond: None,
    });
    // 2-vector variant
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description: "bfdot za0.s, {z0.h-z1.h}, {z2.h-z3.h} — SME2.1 multi-vec BF16→FP32 dot product (2-reg)",
        result_opcode: ArmOpcode::BFDOT_SME21_2,
        priority: 120,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_bfdot"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FMul,
        description: "bfdot za0.s(zerod), {z0.h-z1.h}, {z2.h-z3.h} — SME2.1 multi-vec BF16 fresh dot prod (2-reg)",
        result_opcode: ArmOpcode::BFDOT_SME21_2,
        priority: 130,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_bfdot"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — Multi-Vector FP16 to FP32 Dot Product
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description: "fdot za0.s, {z0.h-z3.h}, {z4.h-z7.h} — SME2.1 multi-vec FP16→FP32 dot product accumulate",
        result_opcode: ArmOpcode::FDOT_SME21_MULTI,
        priority: 100,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_fp16dot"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FMul,
        description:
            "fdot za0.s(zerod), {z0.h-z3.h}, {z4.h-z7.h} — SME2.1 multi-vec FP16 fresh dot product",
        result_opcode: ArmOpcode::FDOT_SME21_MULTI,
        priority: 110,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_fp16dot"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — Streaming SVE with SME2 Overlays
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description: "fadd za0.s[w, #0], za0.s[w, #0], z0.s — SME2.1 streaming SVE add to ZA slice",
        result_opcode: ArmOpcode::FADD_SME21_STREAM,
        priority: 100,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_stream"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FSub,
        description:
            "fsub za0.s[w, #0], za0.s[w, #0], z0.s — SME2.1 streaming SVE sub from ZA slice",
        result_opcode: ArmOpcode::FSUB_SME21_STREAM,
        priority: 100,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_stream"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FMul,
        description: "fmul za0.s[w, #0], za0.s[w, #0], z0.s — SME2.1 streaming SVE mul ZA slice",
        result_opcode: ArmOpcode::FMUL_SME21_STREAM,
        priority: 100,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_stream"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description: "fmla za0.s[w, #0], p0/m, z0.s, z1.s — SME2.1 streaming SVE ZA FMLA",
        result_opcode: ArmOpcode::FMLA_SME21_STREAM,
        priority: 100,
        num_operands: 4,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_stream"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FSub,
        description: "fmls za0.s[w, #0], p0/m, z0.s, z1.s — SME2.1 streaming SVE ZA FMLS",
        result_opcode: ArmOpcode::FMLS_SME21_STREAM,
        priority: 100,
        num_operands: 4,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_stream"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — Streaming SVE Load/Store with ZA
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::Load,
        description: "ld1w {za0.s[w]}, p0/z, [x0] — SME2.1 streaming SVE load ZA slice from memory",
        result_opcode: ArmOpcode::LD1W_SME21_STREAM,
        priority: 100,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_stream"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::Store,
        description: "st1w {za0.s[w]}, p0, [x0] — SME2.1 streaming SVE store ZA slice to memory",
        result_opcode: ArmOpcode::ST1W_SME21_STREAM,
        priority: 100,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_stream"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — 512-bit ZA Storage Access Patterns
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::Load,
        description: "ldr za0, [x0] — SME2.1 512-bit ZA register load",
        result_opcode: ArmOpcode::LDR_ZA512_SME21,
        priority: 100,
        num_operands: 2,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_za512"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::Store,
        description: "str za0, [x0] — SME2.1 512-bit ZA register store",
        result_opcode: ArmOpcode::STR_ZA512_SME21,
        priority: 100,
        num_operands: 2,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_za512"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::Load,
        description: "ld1w {za0h.s[w]}, p0/z, [x0] — SME2.1 streaming 512-bit ZA horizontal load",
        result_opcode: ArmOpcode::LD1H_ZA512_SME21,
        priority: 100,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_za512"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::Store,
        description: "st1w {za0h.s[w]}, p0, [x0] — SME2.1 streaming 512-bit ZA horizontal store",
        result_opcode: ArmOpcode::ST1H_ZA512_SME21,
        priority: 100,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_za512"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::Load,
        description: "ld1w {za0v.s[w]}, p0/z, [x0] — SME2.1 streaming 512-bit ZA vertical load",
        result_opcode: ArmOpcode::LD1V_ZA512_SME21,
        priority: 101,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_za512"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::Store,
        description: "st1w {za0v.s[w]}, p0, [x0] — SME2.1 streaming 512-bit ZA vertical store",
        result_opcode: ArmOpcode::ST1V_ZA512_SME21,
        priority: 101,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_za512"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — Tile Transpose Operations
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::ShuffleVector,
        description:
            "trn1 za0.s, za1.s, za2.s — SME2.1 tile transpose even elements across ZA slices",
        result_opcode: ArmOpcode::TRN1_ZA_SME21,
        priority: 100,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_transpose"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::ShuffleVector,
        description:
            "trn2 za0.s, za1.s, za2.s — SME2.1 tile transpose odd elements across ZA slices",
        result_opcode: ArmOpcode::TRN2_ZA_SME21,
        priority: 101,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_transpose"),
        cond: None,
    });
    // Tile transpose with FP8 granularity
    table.push(IselPattern {
        ir_opcode: Opcode::ShuffleVector,
        description: "trn1 za0.b, za1.b, za2.b — SME2.1 tile transpose byte elements across ZA",
        result_opcode: ArmOpcode::TRN1_ZA_SME21,
        priority: 110,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_transpose_fp8"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::ShuffleVector,
        description: "trn2 za0.b, za1.b, za2.b — SME2.1 tile transpose odd byte elements across ZA",
        result_opcode: ArmOpcode::TRN2_ZA_SME21,
        priority: 111,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_transpose_fp8"),
        cond: None,
    });
    // Tile transpose with half-precision
    table.push(IselPattern {
        ir_opcode: Opcode::ShuffleVector,
        description:
            "trn1 za0.h, za1.h, za2.h — SME2.1 tile transpose half-word elements across ZA",
        result_opcode: ArmOpcode::TRN1_ZA_SME21,
        priority: 120,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_transpose"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::ShuffleVector,
        description: "trn2 za0.h, za1.h, za2.h — SME2.1 tile transpose odd half-word across ZA",
        result_opcode: ArmOpcode::TRN2_ZA_SME21,
        priority: 121,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_transpose"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — Tile Transpose with ZA Accumulation
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description: "trn1a za0.s, za1.s — SME2.1 tile transpose-accumulate ZA slices",
        result_opcode: ArmOpcode::TRN1A_ZA_SME21,
        priority: 200,
        num_operands: 2,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_transpose_acc"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description: "trn2a za0.s, za1.s — SME2.1 tile transpose-accumulate odd ZA slices",
        result_opcode: ArmOpcode::TRN2A_ZA_SME21,
        priority: 201,
        num_operands: 2,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_transpose_acc"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — Structured Sparsity with 2:4 Pattern Support
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description:
            "fmla_sp za0.s, {z0.h-z3.h}, {z4.h-z7.h}, #imm — SME2.1 2:4 sparse FMLA with metadata",
        result_opcode: ArmOpcode::FMLA_SP_SME21,
        priority: 100,
        num_operands: 4,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_sparsity"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description:
            "fmla_sp za0.s, {z0.h-z1.h}, {z2.h-z3.h}, #imm — SME2.1 2:4 sparse FMLA (2-reg)",
        result_opcode: ArmOpcode::FMLA_SP_SME21_2,
        priority: 110,
        num_operands: 4,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_sparsity"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FMul,
        description:
            "fmul_sp za0.s(zeroed), {z0.h-z3.h}, {z4.h-z7.h}, #imm — SME2.1 2:4 sparse FMUL fresh",
        result_opcode: ArmOpcode::FMUL_SP_SME21,
        priority: 120,
        num_operands: 4,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_sparsity"),
        cond: None,
    });
    // 2:4 sparse dot product variants
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description: "fdot_sp za0.s, {z0.h-z3.h}, {z4.h-z7.h}, #imm — SME2.1 2:4 sparse dot product with metadata",
        result_opcode: ArmOpcode::FDOT_SP_SME21,
        priority: 100,
        num_operands: 4,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_sparsity"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FMul,
        description:
            "fdot_sp za0.s(zeroed), {z0.h-z3.h}, {z4.h-z7.h}, #imm — SME2.1 2:4 sparse dot fresh",
        result_opcode: ArmOpcode::FDOT_SP_SME21,
        priority: 110,
        num_operands: 4,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_sparsity"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — 2:4 Structured Sparsity Metadata Load
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::Load,
        description: "ldr_sp z0, [x0] — SME2.1 load 2:4 sparsity metadata vector",
        result_opcode: ArmOpcode::LDR_SP_META_SME21,
        priority: 100,
        num_operands: 2,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_sparsity"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::Load,
        description:
            "ld1rqw_sp {z0.q-z3.q}, p0/z, [x0] — SME2.1 load and replicate sparse quad-words",
        result_opcode: ArmOpcode::LD1RQW_SP_SME21,
        priority: 101,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_sparsity"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — Pack/Unpack for Structured Sparsity
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::ShuffleVector,
        description:
            "pk_sp z0.h, z1.h, #imm — SME2.1 pack dense vector to 2:4 sparse with metadata",
        result_opcode: ArmOpcode::PK_SP_SME21,
        priority: 100,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_sparsity"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::ShuffleVector,
        description:
            "upk_sp z0.h, z1.h, #imm — SME2.1 unpack 2:4 sparse vector to dense with metadata",
        result_opcode: ArmOpcode::UPK_SP_SME21,
        priority: 100,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_sparsity"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — ZA Slice Broadcast and Reduce
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::InsertElement,
        description: "movprfx za0.s[w], za1.s[w] — SME2.1 ZA slice broadcast with mov prefix",
        result_opcode: ArmOpcode::TRN1_ZA_SME21,
        priority: 300,
        num_operands: 2,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_za512"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::ExtractElement,
        description: "mov za0.s[w], z0.s — SME2.1 move vector element to ZA slice",
        result_opcode: ArmOpcode::LDR_ZA512_SME21,
        priority: 310,
        num_operands: 2,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_za512"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — FP8 Outer Product Zero-Accumulate
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::FMul,
        description:
            "fmop8a za0.s(zeroed), p0/m, p0/m, z0.b, z1.b — SME2.1 FP8 zero-acc outer product",
        result_opcode: ArmOpcode::FOP8A_SME21,
        priority: 200,
        num_operands: 5,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_fp8"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FMul,
        description: "fmop8s za0.s(zeroed), p0/m, p0/m, z0.b, z1.b — SME2.1 FP8 zero-acc outer sub",
        result_opcode: ArmOpcode::FOP8S_SME21,
        priority: 201,
        num_operands: 5,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_fp8"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — SVE Predicate to ZA Mask Transfer
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::Select,
        description: "sel za0.s[w], za1.s[w], p0/m — SME2.1 select ZA slice by predicate mask",
        result_opcode: ArmOpcode::FMLA_SP_SME21,
        priority: 400,
        num_operands: 3,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_stream"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — Additional FP8 Outer Product Variants
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description:
            "fop8a za0.s, p0/m, p0/m, z0.b, z1.b — SME2.1 FP8 mixed-format outer product add FP32",
        result_opcode: ArmOpcode::FOP8A_SME21,
        priority: 130,
        num_operands: 5,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_fp8"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FSub,
        description:
            "fop8s za0.s, p0/m, p0/m, z0.b, z1.b — SME2.1 FP8 mixed-format outer product sub FP32",
        result_opcode: ArmOpcode::FOP8S_SME21,
        priority: 130,
        num_operands: 5,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_fp8"),
        cond: None,
    });

    // ================================================================
    // SME2.1 — Additional Structured Sparsity Variants
    // ================================================================
    table.push(IselPattern {
        ir_opcode: Opcode::FAdd,
        description: "fmla_sp za0.s, {z0.h-z3.h}, {z4.h-z7.h}, #imm — SME2.1 2:4 sparse FMLA with metadata (alt)",
        result_opcode: ArmOpcode::FMLA_SP_SME21,
        priority: 140,
        num_operands: 4,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_sparsity"),
        cond: None,
    });
    table.push(IselPattern {
        ir_opcode: Opcode::FMul,
        description: "fmul_sp za0.s, {z0.h-z3.h}, {z4.h-z7.h}, #imm — SME2.1 2:4 sparse FMUL with metadata (alt)",
        result_opcode: ArmOpcode::FMUL_SP_SME21,
        priority: 140,
        num_operands: 4,
        imm_constraint: None,
        is_two_address: false,
        required_feature: Some("sme2p1_sparsity"),
        cond: None,
    });

    table
}

/// Build the SME2.1 ISel table with statistics.
pub fn build_sme21_isel_table() -> SME21IselTable {
    let patterns = sme21_isel_table();
    let total = patterns.len();
    let fp8_op = patterns
        .iter()
        .filter(|p| {
            matches!(
                p.result_opcode,
                ArmOpcode::FOP8A_SME21 | ArmOpcode::FOP8S_SME21
            )
        })
        .count();
    let bf16_dot = patterns
        .iter()
        .filter(|p| {
            matches!(
                p.result_opcode,
                ArmOpcode::BFDOT_SME21 | ArmOpcode::BFDOT_SME21_2
            )
        })
        .count();
    let stream = patterns
        .iter()
        .filter(|p| {
            matches!(
                p.result_opcode,
                ArmOpcode::FADD_SME21_STREAM
                    | ArmOpcode::FSUB_SME21_STREAM
                    | ArmOpcode::FMUL_SME21_STREAM
                    | ArmOpcode::FMLA_SME21_STREAM
                    | ArmOpcode::FMLS_SME21_STREAM
                    | ArmOpcode::LD1W_SME21_STREAM
                    | ArmOpcode::ST1W_SME21_STREAM
                    | ArmOpcode::FDOT_SME21_MULTI
            )
        })
        .count();
    let za512 = patterns
        .iter()
        .filter(|p| {
            matches!(
                p.result_opcode,
                ArmOpcode::LDR_ZA512_SME21
                    | ArmOpcode::STR_ZA512_SME21
                    | ArmOpcode::LD1H_ZA512_SME21
                    | ArmOpcode::ST1H_ZA512_SME21
                    | ArmOpcode::LD1V_ZA512_SME21
                    | ArmOpcode::ST1V_ZA512_SME21
            )
        })
        .count();
    let transpose = patterns
        .iter()
        .filter(|p| {
            matches!(
                p.result_opcode,
                ArmOpcode::TRN1_ZA_SME21
                    | ArmOpcode::TRN2_ZA_SME21
                    | ArmOpcode::TRN1A_ZA_SME21
                    | ArmOpcode::TRN2A_ZA_SME21
            )
        })
        .count();
    let sparsity = patterns
        .iter()
        .filter(|p| {
            matches!(
                p.result_opcode,
                ArmOpcode::FMLA_SP_SME21
                    | ArmOpcode::FMLA_SP_SME21_2
                    | ArmOpcode::FMUL_SP_SME21
                    | ArmOpcode::FDOT_SP_SME21
                    | ArmOpcode::LDR_SP_META_SME21
                    | ArmOpcode::LD1RQW_SP_SME21
                    | ArmOpcode::PK_SP_SME21
                    | ArmOpcode::UPK_SP_SME21
            )
        })
        .count();

    SME21IselTable {
        patterns,
        stats: SME21IselStats {
            total_patterns: total,
            fp8_outer_product: fp8_op,
            bf16_fp32_dot: bf16_dot,
            streaming_sve_overlay: stream,
            za_512bit_access: za512,
            tile_transpose: transpose,
            structured_sparsity: sparsity,
        },
    }
}

/// SME2.1 ISel engine.
pub struct SME21IselEngine {
    pub table: SME21IselTable,
    pub features: SME21Features,
}

/// SME2.1 feature flags.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SME21Features {
    pub has_sme2p1: bool,
    pub has_sme2p1_fp8: bool,
    pub has_sme2p1_bfdot: bool,
    pub has_sme2p1_fp16dot: bool,
    pub has_sme2p1_stream: bool,
    pub has_sme2p1_za512: bool,
    pub has_sme2p1_transpose: bool,
    pub has_sme2p1_transpose_fp8: bool,
    pub has_sme2p1_transpose_acc: bool,
    pub has_sme2p1_sparsity: bool,
}

impl Default for SME21Features {
    fn default() -> Self {
        Self {
            has_sme2p1: false,
            has_sme2p1_fp8: false,
            has_sme2p1_bfdot: false,
            has_sme2p1_fp16dot: false,
            has_sme2p1_stream: false,
            has_sme2p1_za512: false,
            has_sme2p1_transpose: false,
            has_sme2p1_transpose_fp8: false,
            has_sme2p1_transpose_acc: false,
            has_sme2p1_sparsity: false,
        }
    }
}

impl SME21Features {
    /// Full SME2.1 with all sub-features enabled.
    pub fn sme21_full() -> Self {
        Self {
            has_sme2p1: true,
            has_sme2p1_fp8: true,
            has_sme2p1_bfdot: true,
            has_sme2p1_fp16dot: true,
            has_sme2p1_stream: true,
            has_sme2p1_za512: true,
            has_sme2p1_transpose: true,
            has_sme2p1_transpose_fp8: true,
            has_sme2p1_transpose_acc: true,
            has_sme2p1_sparsity: true,
        }
    }

    /// Check if a feature string is available.
    pub fn has_feature(&self, feat: &str) -> bool {
        match feat {
            "sme2p1" => self.has_sme2p1,
            "sme2p1_fp8" => self.has_sme2p1_fp8,
            "sme2p1_bfdot" => self.has_sme2p1_bfdot,
            "sme2p1_fp16dot" => self.has_sme2p1_fp16dot,
            "sme2p1_stream" => self.has_sme2p1_stream,
            "sme2p1_za512" => self.has_sme2p1_za512,
            "sme2p1_transpose" => self.has_sme2p1_transpose,
            "sme2p1_transpose_fp8" => self.has_sme2p1_transpose_fp8,
            "sme2p1_transpose_acc" => self.has_sme2p1_transpose_acc,
            "sme2p1_sparsity" => self.has_sme2p1_sparsity,
            _ => false,
        }
    }
}

impl SME21IselEngine {
    /// Create a new SME2.1 ISel engine.
    pub fn new(features: SME21Features) -> Self {
        let table = build_sme21_isel_table();
        Self { table, features }
    }

    /// Check whether a pattern is applicable.
    pub fn is_pattern_applicable(&self, pattern: &IselPattern) -> bool {
        match pattern.required_feature {
            Some(feat) => self.features.has_feature(feat),
            None => true,
        }
    }

    /// Select the best pattern for a given IR opcode.
    pub fn select_pattern(&self, ir_opcodes: &[Opcode]) -> Option<&IselPattern> {
        let mut best: Option<&IselPattern> = None;
        for p in &self.table.patterns {
            if ir_opcodes.contains(&p.ir_opcode) && self.is_pattern_applicable(p) {
                match best {
                    None => best = Some(p),
                    Some(b) if p.priority < b.priority => best = Some(p),
                    _ => {}
                }
            }
        }
        best
    }

    /// Get statistics about the SME2.1 ISel table.
    pub fn stats(&self) -> String {
        format!(
            "SME2.1 ISel: {} patterns (FP8 outer product: {}, BF16→FP32 dot: {}, streaming SVE: {}, ZA512 access: {}, transpose: {}, sparsity: {})",
            self.table.stats.total_patterns,
            self.table.stats.fp8_outer_product,
            self.table.stats.bf16_fp32_dot,
            self.table.stats.streaming_sve_overlay,
            self.table.stats.za_512bit_access,
            self.table.stats.tile_transpose,
            self.table.stats.structured_sparsity,
        )
    }

    /// Check if 2:4 structured sparsity is supported.
    pub fn has_sparsity(&self) -> bool {
        self.features.has_sme2p1_sparsity
    }

    /// Check if FP8 outer products are available.
    pub fn has_fp8_ops(&self) -> bool {
        self.features.has_sme2p1_fp8
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_sme21_isel_table_builds() {
        let table = build_sme21_isel_table();
        assert!(table.stats.total_patterns > 20);
        assert!(table.stats.fp8_outer_product > 0);
        assert!(table.stats.bf16_fp32_dot > 0);
        assert!(table.stats.streaming_sve_overlay > 0);
        assert!(table.stats.za_512bit_access > 0);
        assert!(table.stats.tile_transpose > 0);
        assert!(table.stats.structured_sparsity > 0);
    }

    #[test]
    fn test_sme21_feature_flags() {
        let f = SME21Features::default();
        assert!(!f.has_sme2p1);
        assert!(!f.has_feature("sme2p1_fp8"));

        let f = SME21Features::sme21_full();
        assert!(f.has_sme2p1);
        assert!(f.has_feature("sme2p1_fp8"));
        assert!(f.has_feature("sme2p1_bfdot"));
        assert!(f.has_feature("sme2p1_stream"));
        assert!(f.has_feature("sme2p1_za512"));
        assert!(f.has_feature("sme2p1_transpose"));
        assert!(f.has_feature("sme2p1_transpose_fp8"));
        assert!(f.has_feature("sme2p1_transpose_acc"));
        assert!(f.has_feature("sme2p1_sparsity"));
    }

    #[test]
    fn test_sme21_isel_engine_select() {
        let engine = SME21IselEngine::new(SME21Features::sme21_full());
        let pat = engine.select_pattern(&[Opcode::FAdd]);
        assert!(pat.is_some());
    }

    #[test]
    fn test_sme21_isel_pattern_applicable() {
        let engine = SME21IselEngine::new(SME21Features::sme21_full());
        let fp8_pat = IselPattern {
            ir_opcode: Opcode::FAdd,
            description: "test",
            result_opcode: ArmOpcode::FOP8A_SME21,
            priority: 100,
            num_operands: 5,
            imm_constraint: None,
            is_two_address: false,
            required_feature: Some("sme2p1_fp8"),
            cond: None,
        };
        assert!(engine.is_pattern_applicable(&fp8_pat));
    }

    #[test]
    fn test_sme21_pattern_disabled_features() {
        let engine = SME21IselEngine::new(SME21Features::default());
        let sp_pat = IselPattern {
            ir_opcode: Opcode::FAdd,
            description: "test",
            result_opcode: ArmOpcode::FMLA_SP_SME21,
            priority: 100,
            num_operands: 4,
            imm_constraint: None,
            is_two_address: false,
            required_feature: Some("sme2p1_sparsity"),
            cond: None,
        };
        assert!(!engine.is_pattern_applicable(&sp_pat));
    }

    #[test]
    fn test_sme21_engine_has_sparsity() {
        let engine = SME21IselEngine::new(SME21Features::sme21_full());
        assert!(engine.has_sparsity());
        assert!(engine.has_fp8_ops());

        let engine_min = SME21IselEngine::new(SME21Features::default());
        assert!(!engine_min.has_sparsity());
        assert!(!engine_min.has_fp8_ops());
    }

    #[test]
    fn test_sme21_all_patterns_valid() {
        let table = build_sme21_isel_table();
        for p in &table.patterns {
            assert!(p.num_operands > 0);
            assert!(p.priority > 0);
            assert!(!p.description.is_empty());
        }
    }
}