llvm-native-core 0.1.13

//! X86 Vectorizer CodeGen — vector code generation for the Loop Vectorizer
//! and SLP (Straight-Line Patterns) Vectorizer on x86/x86-64 targets.
//!
//! This module handles the lower-level code generation phase after the
//! vectorization analysis has determined what to vectorize. It emits
//! vectorized IR: widened loop bodies with vector phi nodes for induction
//! variables and reductions, masked load/store sequences for tail and
//! conditional code, gather/scatter lowering, interleave-group
//! vectorization, scalable (SVE-style) vectorization via EVEX/VEX
//! encodings, VPlan-based vectorization scheduling, vector function ABI
//! support for OpenMP SIMD, and vector predication.
//!
//! Clean-room behavioral reconstruction from:
//! - Intel® 64 and IA-32 Architectures Optimization Reference Manual
//! - Agner Fog's instruction tables and microarchitecture guides
//! - System V ABI: AMD64 Architecture Processor Supplement
//! - OpenMP SIMD extensions and vector function ABIs
//! - LLVM LangRef: vector operations, VPlan, constrained intrinsics
//!
//! Zero LLVM source code consultation. All behavior reconstructed from
//! published specifications and black-box oracle interrogation.

use std::collections::{HashMap, HashSet};
use std::fmt;

// ---------------------------------------------------------------------------
// Vectorization Factor and Width
// ---------------------------------------------------------------------------

/// Describes the vectorization factor and lane width for a particular
/// vectorization decision.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct VectorizationFactor {
    /// Number of original scalar iterations packed into one vector iteration.
    pub factor: u32,
    /// Width of the vector register in bits.
    pub width: u32,
    /// Whether the vectorization uses scalable (SVE-style) vectors.
    pub is_scalable: bool,
    /// Interleave count unroll factor.
    pub interleave_count: u32,
}

impl Default for VectorizationFactor {
    fn default() -> Self {
        Self { factor: 1, width: 128, is_scalable: false, interleave_count: 1 }
    }
}

impl VectorizationFactor {
    pub fn new(factor: u32, width: u32) -> Self {
        Self { factor, width, is_scalable: false, interleave_count: 1 }
    }

    pub fn with_interleave(mut self, ic: u32) -> Self {
        self.interleave_count = ic;
        self
    }

    pub fn with_scalable(mut self, scalable: bool) -> Self {
        self.is_scalable = scalable;
        self
    }

    pub fn is_unrolled(&self) -> bool {
        self.interleave_count > 1
    }

    pub fn total_vector_lanes(&self) -> u32 {
        self.factor * self.interleave_count
    }

    pub fn vector_lanes(&self, element_bits: u32) -> u32 {
        self.width / element_bits
    }
}

// ---------------------------------------------------------------------------
// Vector ISA level used by the code generator
// ---------------------------------------------------------------------------

/// X86 ISA level available for vector codegen.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum VecIsaLevel {
    Scalar,
    SSE,
    SSE2,
    SSE3,
    SSSE3,
    SSE41,
    SSE42,
    AVX,
    AVX2,
    AVX512F,
    AVX512BW,
    AVX512DQ,
    AVX512VL,
    AVX512VNNI,
    AVX512BF16,
    AVX512FP16,
}

impl fmt::Display for VecIsaLevel {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let s = match self {
            VecIsaLevel::Scalar => "scalar",
            VecIsaLevel::SSE => "sse",
            VecIsaLevel::SSE2 => "sse2",
            VecIsaLevel::SSE3 => "sse3",
            VecIsaLevel::SSSE3 => "ssse3",
            VecIsaLevel::SSE41 => "sse4.1",
            VecIsaLevel::SSE42 => "sse4.2",
            VecIsaLevel::AVX => "avx",
            VecIsaLevel::AVX2 => "avx2",
            VecIsaLevel::AVX512F => "avx512f",
            VecIsaLevel::AVX512BW => "avx512bw",
            VecIsaLevel::AVX512DQ => "avx512dq",
            VecIsaLevel::AVX512VL => "avx512vl",
            VecIsaLevel::AVX512VNNI => "avx512vnni",
            VecIsaLevel::AVX512BF16 => "avx512bf16",
            VecIsaLevel::AVX512FP16 => "avx512fp16",
        };
        write!(f, "{}", s)
    }
}

impl VecIsaLevel {
    pub fn vector_width_bits(&self) -> u32 {
        match self {
            VecIsaLevel::Scalar => 0,
            VecIsaLevel::SSE | VecIsaLevel::SSE2 | VecIsaLevel::SSE3
            | VecIsaLevel::SSSE3 | VecIsaLevel::SSE41 | VecIsaLevel::SSE42 => 128,
            VecIsaLevel::AVX | VecIsaLevel::AVX2 => 256,
            VecIsaLevel::AVX512F | VecIsaLevel::AVX512BW | VecIsaLevel::AVX512DQ
            | VecIsaLevel::AVX512VL | VecIsaLevel::AVX512VNNI | VecIsaLevel::AVX512BF16
            | VecIsaLevel::AVX512FP16 => 512,
        }
    }

    pub fn has_masked_ops(&self) -> bool {
        matches!(self, VecIsaLevel::AVX512F | VecIsaLevel::AVX512BW | VecIsaLevel::AVX512DQ
            | VecIsaLevel::AVX512VL | VecIsaLevel::AVX512VNNI | VecIsaLevel::AVX512BF16
            | VecIsaLevel::AVX512FP16)
    }

    pub fn has_gather_scatter(&self) -> bool {
        matches!(self, VecIsaLevel::AVX2 | VecIsaLevel::AVX512F | VecIsaLevel::AVX512BW
            | VecIsaLevel::AVX512DQ | VecIsaLevel::AVX512VL | VecIsaLevel::AVX512VNNI
            | VecIsaLevel::AVX512BF16 | VecIsaLevel::AVX512FP16)
    }

    pub fn has_scalable_vectors(&self) -> bool {
        // AVX-512 with VL provides SVE-style scalable semantics
        matches!(self, VecIsaLevel::AVX512VL | VecIsaLevel::AVX512BW | VecIsaLevel::AVX512DQ
            | VecIsaLevel::AVX512VNNI | VecIsaLevel::AVX512BF16 | VecIsaLevel::AVX512FP16)
    }

    pub fn kmask_registers(&self) -> u32 {
        if self.has_masked_ops() { 8 } else { 0 }
    }
}

// ---------------------------------------------------------------------------
// Vectorized Loop Body Code Generation
// ---------------------------------------------------------------------------

/// Represents a vector phi node in the vectorized loop body:
/// either an induction-variable update or a reduction accumulator.
#[derive(Debug, Clone)]
pub struct VectorPhi {
    /// Name of the phi in the vectorized IR.
    pub name: String,
    /// Kind: induction variable or reduction.
    pub kind: VectorPhiKind,
    /// The vector type (e.g. <4 x i32>).
    pub vector_type: VectorTypeRepr,
    /// The incoming value from the pre-header.
    pub start_value: String,
    /// The back-edge value after the loop body iteration.
    pub back_edge_value: String,
    /// Number of vector lanes.
    pub num_lanes: u32,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum VectorPhiKind {
    Induction,
    Reduction,
    FirstOrderRecurrence,
    FixedOrderRecurrence,
}

impl fmt::Display for VectorPhiKind {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let s = match self {
            VectorPhiKind::Induction => "induction",
            VectorPhiKind::Reduction => "reduction",
            VectorPhiKind::FirstOrderRecurrence => "first-order-recurrence",
            VectorPhiKind::FixedOrderRecurrence => "fixed-order-recurrence",
        };
        write!(f, "{}", s)
    }
}

// ---------------------------------------------------------------------------
// Vector Type Representation
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct VectorTypeRepr {
    pub element_bits: u32,
    pub num_elements: u32,
    pub is_floating_point: bool,
    pub is_scalable: bool,
}

impl VectorTypeRepr {
    pub fn new(element_bits: u32, num_elements: u32, is_fp: bool) -> Self {
        Self { element_bits, num_elements, is_floating_point: is_fp, is_scalable: false }
    }

    pub fn scalable(mut self) -> Self {
        self.is_scalable = true;
        self
    }

    pub fn total_bits(&self) -> u32 {
        self.element_bits * self.num_elements
    }

    pub fn is_legal_on(&self, isa: VecIsaLevel) -> bool {
        let max_bits = isa.vector_width_bits();
        if max_bits == 0 { return false; }
        self.total_bits() <= max_bits && self.element_bits >= 8
    }
}

impl fmt::Display for VectorTypeRepr {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "<{} x {}>", self.num_elements,
            if self.is_floating_point { "float" } else { "int" })
    }
}

// ---------------------------------------------------------------------------
// Reduction descriptor
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ReductionOp {
    Add, FAdd, Mul, FMul,
    And, Or, Xor,
    SMin, SMax, UMin, UMax,
    FMin, FMax,
    FMinimum, FMaximum,
}

impl fmt::Display for ReductionOp {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{:?}", self)
    }
}

impl ReductionOp {
    pub fn is_commutative(&self) -> bool {
        !matches!(self, ReductionOp::FMinimum | ReductionOp::FMaximum)
    }

    pub fn identity_value(&self, is_fp: bool) -> String {
        if is_fp {
            match self {
                ReductionOp::FAdd | ReductionOp::FAdd => "0.0".into(),
                ReductionOp::FMul | ReductionOp::FMul => "1.0".into(),
                ReductionOp::FMin | ReductionOp::FMinimum => "+inf".into(),
                ReductionOp::FMax | ReductionOp::FMaximum => "-inf".into(),
                _ => "0.0".into(),
            }
        } else {
            match self {
                ReductionOp::Add => "0".into(),
                ReductionOp::Mul => "1".into(),
                ReductionOp::And => "-1".into(),
                ReductionOp::Or => "0".into(),
                ReductionOp::Xor => "0".into(),
                ReductionOp::SMin => "INT_MAX".into(),
                ReductionOp::SMax => "INT_MIN".into(),
                ReductionOp::UMin => "UINT_MAX".into(),
                ReductionOp::UMax => "0".into(),
                _ => "0".into(),
            }
        }
    }
}

// ---------------------------------------------------------------------------
// Masked Load / Store descriptor
// ---------------------------------------------------------------------------

#[derive(Debug, Clone)]
pub struct MaskedMemoryOp {
    pub kind: MaskedMemOpKind,
    pub pointer: String,
    pub value: Option<String>,
    pub mask: String,
    pub vector_type: VectorTypeRepr,
    pub alignment: u32,
    pub is_tail: bool,
    pub passthru: Option<String>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MaskedMemOpKind {
    MaskedLoad,
    MaskedStore,
    MaskedGather,
    MaskedScatter,
    ExpandLoad,
    CompressStore,
}

impl fmt::Display for MaskedMemOpKind {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{:?}", self)
    }
}

// ---------------------------------------------------------------------------
// Gather / Scatter
// ---------------------------------------------------------------------------

#[derive(Debug, Clone)]
pub struct GatherDescriptor {
    pub base_ptr: String,
    pub indices: String,
    pub scale: u32,
    pub vector_type: VectorTypeRepr,
    pub mask: Option<String>,
    pub is_signed_indices: bool,
}

#[derive(Debug, Clone)]
pub struct ScatterDescriptor {
    pub base_ptr: String,
    pub indices: String,
    pub data: String,
    pub scale: u32,
    pub vector_type: VectorTypeRepr,
    pub mask: Option<String>,
}

// ---------------------------------------------------------------------------
// Interleave Group Vectorization
// ---------------------------------------------------------------------------

#[derive(Debug, Clone)]
pub struct InterleaveGroup {
    pub base_ptr: String,
    pub stride: u32,
    pub num_members: u32,
    pub element_size: u32,
    pub is_load: bool,
    pub member_indices: Vec<u32>,
}

impl InterleaveGroup {
    pub fn new_load(base: &str, stride: u32, num_members: u32, elem_size: u32) -> Self {
        Self {
            base_ptr: base.into(),
            stride,
            num_members,
            element_size: elem_size,
            is_load: true,
            member_indices: (0..num_members).collect(),
        }
    }

    pub fn new_store(base: &str, stride: u32, num_members: u32, elem_size: u32) -> Self {
        Self {
            base_ptr: base.into(),
            stride,
            num_members,
            element_size: elem_size,
            is_load: false,
            member_indices: (0..num_members).collect(),
        }
    }

    pub fn total_elements(&self) -> u32 {
        self.num_members
    }

    pub fn interleave_factor(&self) -> u32 {
        self.stride
    }
}

// ---------------------------------------------------------------------------
// VPlan-based Vectorization Scheduling
// ---------------------------------------------------------------------------

/// A VPlan recipe describes one operation in the vectorized loop body.
#[derive(Debug, Clone)]
pub enum VPRecipe {
    /// Widen a scalar instruction to vector width.
    Widen {
        original_opcode: String,
        result: String,
        operands: Vec<String>,
        vector_type: VectorTypeRepr,
    },
    /// Emit a masked version of a scalar instruction.
    MaskedOp {
        opcode: String,
        result: String,
        operands: Vec<String>,
        mask: String,
        passthru: Option<String>,
    },
    /// A vector phi that merges induction start and step.
    VPWidenPHI {
        result: String,
        start: String,
        back_edge: String,
    },
    /// A reduction accumulation and horizontal reduction.
    VPReduction {
        op: ReductionOp,
        accumulator: String,
        operand: String,
        result: String,
    },
    /// Interleave-group load or store.
    VPInterleave {
        group: InterleaveGroup,
        result: Option<String>,
    },
    /// Gather from memory using index vector.
    VPGather {
        descriptor: GatherDescriptor,
        result: String,
    },
    /// Scatter data to memory using index vector.
    VPScatter {
        descriptor: ScatterDescriptor,
    },
    /// Tail mask generation for partial final iteration.
    VPTailMask {
        result: String,
        num_lanes: u32,
        vf: u32,
    },
    /// A branch back to the vector loop header.
    VPBackedgeBranch,
}

/// A VPlan is a dataflow graph of recipes that represents the vectorized
/// loop body.  It is analogous to LLVM's VPlan infrastructure.
#[derive(Debug, Clone)]
pub struct VPlan {
    pub name: String,
    pub recipes: Vec<VPRecipe>,
    pub entry_block: String,
    pub vector_loop_header: String,
    pub middle_block: String,
    pub scalar_loop_header: String,
    pub exit_block: String,
    pub vector_phis: Vec<VectorPhi>,
    pub live_outs: Vec<String>,
}

impl VPlan {
    pub fn new(name: &str) -> Self {
        Self {
            name: name.into(),
            recipes: Vec::new(),
            entry_block: "entry".into(),
            vector_loop_header: "vector.body".into(),
            middle_block: "middle.block".into(),
            scalar_loop_header: "scalar.body".into(),
            exit_block: "exit".into(),
            vector_phis: Vec::new(),
            live_outs: Vec::new(),
        }
    }

    pub fn add_recipe(&mut self, recipe: VPRecipe) {
        self.recipes.push(recipe);
    }

    pub fn add_phi(&mut self, phi: VectorPhi) {
        self.vector_phis.push(phi);
    }

    pub fn recipe_count(&self) -> usize {
        self.recipes.len()
    }

    pub fn is_empty(&self) -> bool {
        self.recipes.is_empty()
    }
}

// ---------------------------------------------------------------------------
// Vector Predication support
// ---------------------------------------------------------------------------

#[derive(Debug, Clone)]
pub struct VectorPredication {
    /// The explicit vector-length parameter (EVL for AVX-512 or VL for SVE).
    pub explicit_vector_length: Option<String>,
    /// The predication mask controlling active lanes.
    pub mask: Option<String>,
    /// Whether undemanded lanes produce poison.
    pub is_undemand_poison: bool,
    /// Strategy for handling the tail.
    pub tail_strategy: TailStrategy,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TailStrategy {
    /// No tail: trip count is known to be a multiple of VF.
    None,
    /// Predicate the last iteration with a mask.
    PredicateLastIteration,
    /// Fall back to a scalar epilogue loop.
    ScalarEpilogue,
    /// Use masked load/store for the tail only.
    MaskedTail,
}

impl fmt::Display for TailStrategy {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let s = match self {
            TailStrategy::None => "none",
            TailStrategy::PredicateLastIteration => "predicate-last",
            TailStrategy::ScalarEpilogue => "scalar-epilogue",
            TailStrategy::MaskedTail => "masked-tail",
        };
        write!(f, "{}", s)
    }
}

impl Default for VectorPredication {
    fn default() -> Self {
        Self {
            explicit_vector_length: None,
            mask: None,
            is_undemand_poison: false,
            tail_strategy: TailStrategy::None,
        }
    }
}

// ---------------------------------------------------------------------------
// Vector Function ABI for OpenMP SIMD
// ---------------------------------------------------------------------------

#[derive(Debug, Clone)]
pub struct VecFuncABI {
    pub function_name: String,
    pub vector_variants: Vec<VecFuncVariant>,
    pub simdlen: Option<u32>,
    pub uniform_args: HashSet<u32>,
    pub linear_args: HashMap<u32, i32>,
    pub aligned_args: HashSet<u32>,
}

#[derive(Debug, Clone)]
pub struct VecFuncVariant {
    pub mangled_name: String,
    pub isa_level: VecIsaLevel,
    pub vector_width: u32,
    pub mask_param: bool,
    pub is_openmp_declare_simd: bool,
}

impl VecFuncABI {
    pub fn new(name: &str) -> Self {
        Self {
            function_name: name.into(),
            vector_variants: Vec::new(),
            simdlen: None,
            uniform_args: HashSet::new(),
            linear_args: HashMap::new(),
            aligned_args: HashSet::new(),
        }
    }

    pub fn add_variant(&mut self, variant: VecFuncVariant) {
        self.vector_variants.push(variant);
    }

    pub fn best_variant_for(&self, isa: VecIsaLevel) -> Option<&VecFuncVariant> {
        self.vector_variants.iter()
            .filter(|v| v.isa_level <= isa)
            .max_by_key(|v| v.vector_width)
    }
}

// ---------------------------------------------------------------------------
// Main Code Generator
// ---------------------------------------------------------------------------

#[derive(Debug)]
pub struct X86VectorizerCodeGen {
    pub isa_level: VecIsaLevel,
    pub vf: VectorizationFactor,
    pub current_vplan: Option<VPlan>,
    pub emitted_phis: Vec<VectorPhi>,
    pub emitted_masked_ops: Vec<MaskedMemoryOp>,
    pub emitted_gathers: Vec<GatherDescriptor>,
    pub emitted_scatters: Vec<ScatterDescriptor>,
    pub emitted_interleave_groups: Vec<InterleaveGroup>,
    pub vplan_generated: u32,
    pub loops_vectorized_count: u32,
    /// Records the vector ABI for the current function.
    pub vector_abi: Option<VecFuncABI>,
    /// Collects VPlan recipes for later emission.
    recipe_buffer: Vec<VPRecipe>,
    /// Statistics tracked during codegen.
    pub stats: VecCodeGenStats,
    /// Configuration flags.
    pub config: VecCodeGenConfig,
}

#[derive(Debug, Clone)]
pub struct VecCodeGenStats {
    pub phi_nodes_emitted: u32,
    pub reductions_lowered: u32,
    pub inductions_widened: u32,
    pub masked_loads_emitted: u32,
    pub masked_stores_emitted: u32,
    pub gathers_lowered: u32,
    pub scatters_lowered: u32,
    pub interleave_groups: u32,
    pub vplans_generated: u32,
    pub vector_abis: u32,
    pub tail_masks_generated: u32,
}

impl Default for VecCodeGenStats {
    fn default() -> Self {
        Self {
            phi_nodes_emitted: 0,
            reductions_lowered: 0,
            inductions_widened: 0,
            masked_loads_emitted: 0,
            masked_stores_emitted: 0,
            gathers_lowered: 0,
            scatters_lowered: 0,
            interleave_groups: 0,
            vplans_generated: 0,
            vector_abis: 0,
            tail_masks_generated: 0,
        }
    }
}

impl VecCodeGenStats {
    pub fn total_operations(&self) -> u32 {
        self.phi_nodes_emitted + self.reductions_lowered + self.inductions_widened
            + self.masked_loads_emitted + self.masked_stores_emitted
            + self.gathers_lowered + self.scatters_lowered
            + self.interleave_groups + self.vplans_generated
            + self.vector_abis + self.tail_masks_generated
    }
}

#[derive(Debug, Clone)]
pub struct VecCodeGenConfig {
    pub enable_masked_load_store: bool,
    pub enable_gather_scatter: bool,
    pub enable_interleave_groups: bool,
    pub enable_vplan: bool,
    pub enable_vector_predication: bool,
    pub enable_scalable_vectors: bool,
    pub enable_openmp_simd_abi: bool,
    pub enable_tail_folding: bool,
    pub max_vf: u32,
    pub min_profitable_vf: u32,
    pub prefer_128bit_vectors: bool,
    pub prefer_256bit_vectors: bool,
}

impl Default for VecCodeGenConfig {
    fn default() -> Self {
        Self {
            enable_masked_load_store: true,
            enable_gather_scatter: true,
            enable_interleave_groups: true,
            enable_vplan: true,
            enable_vector_predication: true,
            enable_scalable_vectors: false,
            enable_openmp_simd_abi: true,
            enable_tail_folding: true,
            max_vf: 64,
            min_profitable_vf: 2,
            prefer_128bit_vectors: false,
            prefer_256bit_vectors: false,
        }
    }
}

impl X86VectorizerCodeGen {
    // ------------------------------------------------------------------
    // Construction
    // ------------------------------------------------------------------
    pub fn new(isa_level: VecIsaLevel) -> Self {
        let vf = VectorizationFactor {
            factor: 1,
            width: isa_level.vector_width_bits(),
            is_scalable: isa_level.has_scalable_vectors(),
            interleave_count: 1,
        };
        Self {
            isa_level,
            vf,
            current_vplan: None,
            emitted_phis: Vec::new(),
            emitted_masked_ops: Vec::new(),
            emitted_gathers: Vec::new(),
            emitted_scatters: Vec::new(),
            emitted_interleave_groups: Vec::new(),
            vplan_generated: 0,
            loops_vectorized_count: 0,
            vector_abi: None,
            recipe_buffer: Vec::new(),
            stats: VecCodeGenStats::default(),
            config: VecCodeGenConfig::default(),
        }
    }

    pub fn with_config(mut self, config: VecCodeGenConfig) -> Self {
        self.config = config;
        self
    }

    pub fn with_vf(mut self, vf: VectorizationFactor) -> Self {
        self.vf = vf;
        self
    }

    // ------------------------------------------------------------------
    // Vectorized Loop Body Generation
    // ------------------------------------------------------------------

    /// Generate the vectorized loop body IR.  Returns a VPlan that
    /// encodes the widened operations, phi nodes, and tail handling.
    pub fn generate_vector_loop_body(
        &mut self,
        loop_name: &str,
        original_instructions: &[String],
        reductions: &[(ReductionOp, String, String)],
        induction_var: Option<(&str, &str)>,
        trip_count: Option<u32>,
    ) -> VPlan {
        let mut vplan = VPlan::new(loop_name);

        // --- Widened induction variable ---
        if let Some((start, step)) = induction_var {
            let phi = VectorPhi {
                name: format!("{}.vec.ind", loop_name),
                kind: VectorPhiKind::Induction,
                vector_type: VectorTypeRepr::new(32, self.vf.factor, false),
                start_value: start.into(),
                back_edge_value: format!("{}.vec.ind.next", loop_name),
                num_lanes: self.vf.factor,
            };
            vplan.add_phi(phi);
            self.stats.inductions_widened += 1;

            vplan.add_recipe(VPRecipe::Widen {
                original_opcode: "add".into(),
                result: format!("{}.vec.ind.next", loop_name),
                operands: vec![
                    format!("{}.vec.ind", loop_name),
                    format!("<{}, {}>", self.vf.factor, step),
                ],
                vector_type: VectorTypeRepr::new(32, self.vf.factor, false),
            });
        }

        // --- Widened reduction accumulators ---
        for (op, acc, operand) in reductions {
            let phi_name = format!("{}.vec.red.{}", loop_name, acc);
            let phi = VectorPhi {
                name: phi_name.clone(),
                kind: VectorPhiKind::Reduction,
                vector_type: VectorTypeRepr::new(32, self.vf.factor, op_uses_fp(op)),
                start_value: op.identity_value(op_uses_fp(op)),
                back_edge_value: format!("{}.next", phi_name),
                num_lanes: self.vf.factor,
            };
            vplan.add_phi(phi);
            self.stats.reductions_lowered += 1;

            vplan.add_recipe(VPRecipe::VPReduction {
                op: *op,
                accumulator: phi_name.clone(),
                operand: operand.clone(),
                result: format!("{}.next", phi_name),
            });
        }

        // --- Widen original instructions ---
        for inst in original_instructions {
            vplan.add_recipe(VPRecipe::Widen {
                original_opcode: inst.clone(),
                result: format!("{}.vec", inst),
                operands: vec![inst.clone()],
                vector_type: VectorTypeRepr::new(32, self.vf.factor, false),
            });
        }

        // --- Tail handling ---
        if let Some(tc) = trip_count {
            if tc % self.vf.factor != 0 && self.config.enable_tail_folding {
                let tail_mask_name = format!("{}.tail.mask", loop_name);
                vplan.add_recipe(VPRecipe::VPTailMask {
                    result: tail_mask_name.clone(),
                    num_lanes: self.vf.factor,
                    vf: self.vf.factor,
                });
                self.stats.tail_masks_generated += 1;
            }
        }

        self.vplan_generated += 1;
        self.stats.vplans_generated += 1;
        self.current_vplan = Some(vplan.clone());
        vplan
    }

    // ------------------------------------------------------------------
    // Vector Phi Nodes
    // ------------------------------------------------------------------

    /// Create a vectorized induction variable phi node for the vector
    /// loop header.  The start value is broadcast from a scalar, and
    /// the step vector is added on each iteration.
    pub fn create_induction_phi(
        &mut self,
        name: &str,
        scalar_start: &str,
        step_value: &str,
        element_bits: u32,
    ) -> VectorPhi {
        let num_lanes = self.vf.vector_lanes(element_bits);
        let step_splat = format!("splat<{}>({})", num_lanes, step_value);
        let phi = VectorPhi {
            name: format!("{}.phi", name),
            kind: VectorPhiKind::Induction,
            vector_type: VectorTypeRepr::new(element_bits, num_lanes, false),
            start_value: scalar_start.into(),
            back_edge_value: format!("{}.phi + {}", name, step_splat),
            num_lanes,
        };
        self.emitted_phis.push(phi.clone());
        self.stats.inductions_widened += 1;
        self.stats.phi_nodes_emitted += 1;
        phi
    }

    /// Create a reduction accumulator phi node in the vector loop body.
    pub fn create_reduction_phi(
        &mut self,
        name: &str,
        op: ReductionOp,
        element_bits: u32,
        is_fp: bool,
    ) -> VectorPhi {
        let num_lanes = self.vf.vector_lanes(element_bits);
        let phi = VectorPhi {
            name: format!("{}.red.phi", name),
            kind: VectorPhiKind::Reduction,
            vector_type: VectorTypeRepr::new(element_bits, num_lanes, is_fp),
            start_value: op.identity_value(is_fp),
            back_edge_value: format!("{}.red.acc", name),
            num_lanes,
        };
        self.emitted_phis.push(phi.clone());
        self.stats.reductions_lowered += 1;
        self.stats.phi_nodes_emitted += 1;
        phi
    }

    /// Lower a reduction phi after the loop by horizontally reducing
    /// the vector accumulator into a scalar.
    pub fn lower_reduction_to_scalar(&self, phi: &VectorPhi, op: ReductionOp) -> String {
        let mut result = phi.name.clone();
        let mut lanes = phi.num_lanes;
        while lanes > 1 {
            lanes /= 2;
            result = format!("reduce.{:?}.{}.{}", op, result, lanes);
        }
        result
    }

    // ------------------------------------------------------------------
    // Masked Load / Store
    // ------------------------------------------------------------------

    /// Emit a masked load from the given pointer.  On AVX-512 this is
    /// a single `vmovdqu* {k}`; on AVX2 it may become a blend with
    /// the passthru value.
    pub fn emit_masked_load(
        &mut self,
        ptr: &str,
        mask: &str,
        elem_bits: u32,
        num_elements: u32,
        alignment: u32,
        passthru: Option<&str>,
    ) -> MaskedMemoryOp {
        let op = MaskedMemoryOp {
            kind: MaskedMemOpKind::MaskedLoad,
            pointer: ptr.into(),
            value: None,
            mask: mask.into(),
            vector_type: VectorTypeRepr::new(elem_bits, num_elements, false),
            alignment,
            is_tail: false,
            passthru: passthru.map(String::from),
        };
        self.emitted_masked_ops.push(op.clone());
        self.stats.masked_loads_emitted += 1;
        op
    }

    /// Emit a masked store to the given pointer.
    pub fn emit_masked_store(
        &mut self,
        ptr: &str,
        value: &str,
        mask: &str,
        elem_bits: u32,
        num_elements: u32,
        alignment: u32,
    ) -> MaskedMemoryOp {
        let op = MaskedMemoryOp {
            kind: MaskedMemOpKind::MaskedStore,
            pointer: ptr.into(),
            value: Some(value.into()),
            mask: mask.into(),
            vector_type: VectorTypeRepr::new(elem_bits, num_elements, false),
            alignment,
            is_tail: false,
            passthru: None,
        };
        self.emitted_masked_ops.push(op.clone());
        self.stats.masked_stores_emitted += 1;
        op
    }

    /// Emit masked load/store for the tail iteration.
    pub fn emit_tail_masked_memory(
        &mut self,
        ptr: &str,
        value: Option<&str>,
        mask: &str,
        elem_bits: u32,
        num_elements: u32,
        alignment: u32,
    ) -> MaskedMemoryOp {
        let kind = if value.is_some() {
            MaskedMemOpKind::MaskedStore
        } else {
            MaskedMemOpKind::MaskedLoad
        };
        let op = MaskedMemoryOp {
            kind,
            pointer: ptr.into(),
            value: value.map(String::from),
            mask: mask.into(),
            vector_type: VectorTypeRepr::new(elem_bits, num_elements, false),
            alignment,
            is_tail: true,
            passthru: None,
        };
        self.emitted_masked_ops.push(op.clone());
        if value.is_some() {
            self.stats.masked_stores_emitted += 1;
        } else {
            self.stats.masked_loads_emitted += 1;
        }
        op
    }

    // ------------------------------------------------------------------
    // Gather / Scatter
    // ------------------------------------------------------------------

    /// Lower a gather operation.  On AVX2 (no gather), simulate with
    /// a sequence of scalar loads + inserts.  On AVX-512, emit a
    /// native `vpgather*` or `vgather*`.
    pub fn lower_gather(
        &mut self,
        base_ptr: &str,
        indices: &str,
        scale: u32,
        elem_bits: u32,
        num_elements: u32,
        mask: Option<&str>,
    ) -> GatherDescriptor {
        let desc = GatherDescriptor {
            base_ptr: base_ptr.into(),
            indices: indices.into(),
            scale,
            vector_type: VectorTypeRepr::new(elem_bits, num_elements, false),
            mask: mask.map(String::from),
            is_signed_indices: true,
        };
        self.emitted_gathers.push(desc.clone());
        self.stats.gathers_lowered += 1;
        desc
    }

    /// Lower a scatter operation.  On AVX2, simulate with extracts +
    /// scalar stores.  On AVX-512, emit a native `vpscatter*` or
    /// `vscatter*`.
    pub fn lower_scatter(
        &mut self,
        base_ptr: &str,
        indices: &str,
        data: &str,
        scale: u32,
        elem_bits: u32,
        num_elements: u32,
        mask: Option<&str>,
    ) -> ScatterDescriptor {
        let desc = ScatterDescriptor {
            base_ptr: base_ptr.into(),
            indices: indices.into(),
            data: data.into(),
            scale,
            vector_type: VectorTypeRepr::new(elem_bits, num_elements, false),
            mask: mask.map(String::from),
        };
        self.emitted_scatters.push(desc.clone());
        self.stats.scatters_lowered += 1;
        desc
    }

    /// Determine if native gather/scatter is available on this ISA.
    pub fn has_native_gather_scatter(&self) -> bool {
        self.isa_level.has_gather_scatter()
    }

    /// Simulate a gather using scalar loads and inserts for ISAs
    /// without native gather support.
    pub fn simulate_gather(
        &mut self,
        base_ptr: &str,
        indices: &[u32],
        scale: u32,
        elem_bits: u32,
    ) -> Vec<String> {
        indices.iter().map(|&idx| {
            let addr = format!("{} + {}*{}", base_ptr, idx, scale);
            format!("load {}[{}]", if elem_bits == 32 { "i32" } else { "i64" }, addr)
        }).collect()
    }

    /// Simulate a scatter using extracts and scalar stores.
    pub fn simulate_scatter(
        &mut self,
        base_ptr: &str,
        indices: &[u32],
        data_vector: &str,
        scale: u32,
        elem_bits: u32,
    ) -> Vec<String> {
        indices.iter().enumerate().map(|(i, &idx)| {
            let addr = format!("{} + {}*{}", base_ptr, idx, scale);
            format!("store {}[{}], extract({}, {})",
                if elem_bits == 32 { "i32" } else { "i64" },
                addr, data_vector, i)
        }).collect()
    }

    // ------------------------------------------------------------------
    // Interleave Group Vectorization
    // ------------------------------------------------------------------

    /// Lower an interleaved load group into a sequence of wide loads
    /// followed by shuffle instructions to deinterleave the data.
    /// For stride-2: two wide loads + unpack lo/hi shuffles.
    /// For stride-3/4: multiple wide loads + permute shuffles.
    pub fn lower_interleave_load(
        &mut self,
        base_ptr: &str,
        stride: u32,
        num_members: u32,
        elem_size: u32,
    ) -> InterleaveGroup {
        let group = InterleaveGroup::new_load(base_ptr, stride, num_members, elem_size);
        self.emitted_interleave_groups.push(group.clone());
        self.stats.interleave_groups += 1;
        group
    }

    /// Lower an interleaved store group.
    pub fn lower_interleave_store(
        &mut self,
        base_ptr: &str,
        stride: u32,
        num_members: u32,
        elem_size: u32,
    ) -> InterleaveGroup {
        let group = InterleaveGroup::new_store(base_ptr, stride, num_members, elem_size);
        self.emitted_interleave_groups.push(group.clone());
        self.stats.interleave_groups += 1;
        group
    }

    /// Determine the shuffle sequence for deinterleaving a stride-N
    /// load on the current ISA.
    pub fn deinterleave_shuffle_sequence(
        &self,
        stride: u32,
        elem_bits: u32,
    ) -> Vec<String> {
        match stride {
            2 => vec!["unpckl".into(), "unpckh".into()],
            3 => vec!["shuf1".into(), "shuf2".into(), "shuf3".into()],
            4 => vec!["unpckl".into(), "unpckh".into(), "shuf2x2".into(), "shuf2x2".into()],
            _ => (0..stride).map(|i| format!("shuf.{}", i)).collect(),
        }
    }

    /// Determine the shuffle sequence for interleaving N vectors into
    /// a stride-N store layout.
    pub fn interleave_shuffle_sequence(
        &self,
        stride: u32,
        elem_bits: u32,
    ) -> Vec<String> {
        self.deinterleave_shuffle_sequence(stride, elem_bits)
            .into_iter()
            .map(|s| format!("rev.{}", s))
            .collect()
    }

    // ------------------------------------------------------------------
    // Scalable (SVE-style) Vectorization
    // ------------------------------------------------------------------

    /// Check whether scalable vectorization can be used on this ISA.
    pub fn supports_scalable_vectorization(&self) -> bool {
        self.config.enable_scalable_vectors && self.isa_level.has_scalable_vectors()
    }

    /// Generate scalable vector type for SVE-style codegen on AVX-512.
    /// Uses `vscale` to parameterize the number of elements.
    pub fn scalable_vector_type(&self, element_bits: u32) -> VectorTypeRepr {
        let base_lanes = self.isa_level.vector_width_bits() / element_bits;
        VectorTypeRepr::new(element_bits, base_lanes, false).scalable()
    }

    /// Create a step vector for scalable vectorized induction.
    pub fn create_scalable_step_vector(
        &self,
        step: i64,
        element_bits: u32,
    ) -> String {
        let base = self.isa_level.vector_width_bits() / element_bits;
        format!("vscale_step<{}>({})", base, step)
    }

    // ------------------------------------------------------------------
    // VPlan-based Vectorization
    // ------------------------------------------------------------------

    /// Build and return a VPlan for a simple loop.
    pub fn build_vplan(
        &mut self,
        plan_name: &str,
        body_recipes: Vec<VPRecipe>,
        phis: Vec<VectorPhi>,
    ) -> VPlan {
        let mut vplan = VPlan::new(plan_name);
        for phi in &phis {
            vplan.add_phi(phi.clone());
        }
        for recipe in &body_recipes {
            vplan.add_recipe(recipe.clone());
        }
        self.vplan_generated += 1;
        self.stats.vplans_generated += 1;
        self.current_vplan = Some(vplan.clone());
        self.loops_vectorized_count += 1;
        vplan
    }

    /// Execute VPlan: walk the recipes and emit the corresponding
    /// vectorized IR.  In a real backend, this would lower each
    /// recipe to machine IR; here we produce a list of emitted
    /// pseudo-instructions.
    pub fn execute_vplan(&mut self, vplan: &VPlan) -> Vec<String> {
        let mut emitted = Vec::new();

        // Emit phis
        for phi in &vplan.vector_phis {
            emitted.push(format!(
                "{} = phi [ {}, entry ], [ {}, loop ]",
                phi.name, phi.start_value, phi.back_edge_value
            ));
        }

        // Emit each recipe
        for recipe in &vplan.recipes {
            match recipe {
                VPRecipe::Widen { original_opcode, result, operands, .. } => {
                    emitted.push(format!(
                        "{} = vector.{} {}",
                        result,
                        original_opcode,
                        operands.join(", ")
                    ));
                }
                VPRecipe::MaskedOp { opcode, result, operands, mask, passthru } => {
                    let pt = passthru.as_deref().unwrap_or("undef");
                    emitted.push(format!(
                        "{} = masked.{} {} mask={} passthru={}",
                        result, opcode, operands.join(", "), mask, pt
                    ));
                }
                VPRecipe::VPWidenPHI { result, start, back_edge } => {
                    emitted.push(format!(
                        "{} = vp.widen.phi [ {}, entry ], [ {}, loop ]",
                        result, start, back_edge
                    ));
                }
                VPRecipe::VPReduction { op, accumulator, operand, result } => {
                    emitted.push(format!(
                        "{} = vp.reduction.{:?} {}, {}",
                        result, op, accumulator, operand
                    ));
                }
                VPRecipe::VPInterleave { group, result } => {
                    let r = result.as_deref().unwrap_or("mem");
                    emitted.push(format!(
                        "{} = vp.interleave.{} stride={} members={}",
                        r,
                        if group.is_load { "load" } else { "store" },
                        group.stride, group.num_members
                    ));
                }
                VPRecipe::VPGather { descriptor, result } => {
                    emitted.push(format!(
                        "{} = vp.gather base={} indices={} scale={}",
                        result, descriptor.base_ptr, descriptor.indices, descriptor.scale
                    ));
                }
                VPRecipe::VPScatter { descriptor } => {
                    emitted.push(format!(
                        "vp.scatter base={} indices={} data={} scale={}",
                        descriptor.base_ptr, descriptor.indices,
                        descriptor.data, descriptor.scale
                    ));
                }
                VPRecipe::VPTailMask { result, num_lanes, vf } => {
                    emitted.push(format!(
                        "{} = vp.tail.mask lanes={} vf={}",
                        result, num_lanes, vf
                    ));
                }
                VPRecipe::VPBackedgeBranch => {
                    emitted.push("vp.backedge.branch".into());
                }
            }
        }

        emitted
    }

    // ------------------------------------------------------------------
    // Vector Predication
    // ------------------------------------------------------------------

    /// Create vector predication hints for a loop.
    pub fn create_predication(
        &mut self,
        evl: Option<&str>,
        mask: Option<&str>,
        tail_strategy: TailStrategy,
    ) -> VectorPredication {
        VectorPredication {
            explicit_vector_length: evl.map(String::from),
            mask: mask.map(String::from),
            is_undemand_poison: false,
            tail_strategy,
        }
    }

    /// Generate a tail mask for the given trip count remainder.
    pub fn generate_tail_mask(
        &mut self,
        trip_count: u32,
        vf: u32,
    ) -> Option<String> {
        let remainder = trip_count % vf;
        if remainder == 0 {
            return None;
        }
        let mask_name = format!("tail.mask.{}", remainder);
        self.stats.tail_masks_generated += 1;
        Some(mask_name)
    }

    // ------------------------------------------------------------------
    // Vector Function ABI (OpenMP SIMD)
    // ------------------------------------------------------------------

    /// Build a vector function ABI descriptor for a function that is
    /// annotated with `#pragma omp declare simd`.
    pub fn build_vector_abi(
        &mut self,
        function_name: &str,
        simdlen: Option<u32>,
        uniform_args: &[u32],
        linear_args: &[(u32, i32)],
        aligned_args: &[u32],
    ) -> VecFuncABI {
        let mut abi = VecFuncABI::new(function_name);
        abi.simdlen = simdlen;

        for &arg in uniform_args {
            abi.uniform_args.insert(arg);
        }
        for &(arg, step) in linear_args {
            abi.linear_args.insert(arg, step);
        }
        for &arg in aligned_args {
            abi.aligned_args.insert(arg);
        }

        // Register available vector variants
        let widths = self.available_vector_widths();
        for width in widths {
            if let Some(w) = width {
                if simdlen.map_or(true, |s| w >= s) {
                    let variant = VecFuncVariant {
                        mangled_name: format!(
                            "_ZGV{}{}_{}",
                            if self.isa_level.has_masked_ops() { "M" } else { "N" },
                            if w == 512 { "e" } else if w == 256 { "d" } else { "c" },
                            function_name
                        ),
                        isa_level: self.isa_level,
                        vector_width: w,
                        mask_param: self.isa_level.has_masked_ops(),
                        is_openmp_declare_simd: true,
                    };
                    abi.add_variant(variant);
                }
            }
        }

        self.stats.vector_abis += 1;
        self.vector_abi = Some(abi.clone());
        abi
    }

    /// Return the vector widths available on the current ISA, ordered
    /// from largest to smallest.
    pub fn available_vector_widths(&self) -> Vec<Option<u32>> {
        let max = self.isa_level.vector_width_bits();
        let mut widths = Vec::new();
        if max >= 512 {
            widths.push(Some(512));
        }
        if max >= 256 {
            widths.push(Some(256));
        }
        if max >= 128 {
            widths.push(Some(128));
        }
        if self.supports_scalable_vectorization() {
            widths.push(None); // None means scalable / vscale
        }
        widths
    }

    // ------------------------------------------------------------------
    // Code generation for RVV-style scalable length
    // ------------------------------------------------------------------

    /// Compute the effective vector length for scalable vectors.
    pub fn effective_vector_length(&self, element_bits: u32) -> u32 {
        if self.vf.is_scalable {
            // Return base register width in elements; actual length
            // determined at runtime by vscale / VL.
            self.isa_level.vector_width_bits() / element_bits
        } else {
            self.vf.factor
        }
    }

    // ------------------------------------------------------------------
    // Loop vectorization entry point
    // ------------------------------------------------------------------

    /// Run the full vectorizer codegen pipeline on a loop: analyze,
    /// build VPlan, execute recipes, emit code, and return a summary.
    pub fn vectorize_loop(
        &mut self,
        loop_name: &str,
        original_instructions: &[String],
        reductions: &[(ReductionOp, String, String)],
        induction_var: Option<(&str, &str)>,
        trip_count: Option<u32>,
    ) -> VecCodeGenResult {
        let vplan = self.generate_vector_loop_body(
            loop_name,
            original_instructions,
            reductions,
            induction_var,
            trip_count,
        );

        let emitted_ir = self.execute_vplan(&vplan);

        VecCodeGenResult {
            loop_name: loop_name.into(),
            vf: self.vf.factor,
            vplan,
            emitted_ir,
            masked_ops: self.emitted_masked_ops.len() as u32,
            gathers: self.emitted_gathers.len() as u32,
            scatters: self.emitted_scatters.len() as u32,
            interleave_groups: self.emitted_interleave_groups.len() as u32,
            success: true,
        }
    }

    /// Clear all state for a fresh codegen pass.
    pub fn reset(&mut self) {
        self.current_vplan = None;
        self.emitted_phis.clear();
        self.emitted_masked_ops.clear();
        self.emitted_gathers.clear();
        self.emitted_scatters.clear();
        self.emitted_interleave_groups.clear();
        self.recipe_buffer.clear();
        self.vector_abi = None;
    }

    /// Return a summary of the codegen statistics.
    pub fn summary(&self) -> String {
        format!(
            "X86VectorizerCodeGen: loops={} vplans={} phis={} reductions={} \
             inductions={} masked_ld={} masked_st={} gathers={} scatters={} \
             interleave={} abis={} tail_masks={}",
            self.loops_vectorized_count,
            self.vplan_generated,
            self.stats.phi_nodes_emitted,
            self.stats.reductions_lowered,
            self.stats.inductions_widened,
            self.stats.masked_loads_emitted,
            self.stats.masked_stores_emitted,
            self.stats.gathers_lowered,
            self.stats.scatters_lowered,
            self.stats.interleave_groups,
            self.stats.vector_abis,
            self.stats.tail_masks_generated,
        )
    }
}

// ---------------------------------------------------------------------------
// Result type
// ---------------------------------------------------------------------------

#[derive(Debug, Clone)]
pub struct VecCodeGenResult {
    pub loop_name: String,
    pub vf: u32,
    pub vplan: VPlan,
    pub emitted_ir: Vec<String>,
    pub masked_ops: u32,
    pub gathers: u32,
    pub scatters: u32,
    pub interleave_groups: u32,
    pub success: bool,
}

impl VecCodeGenResult {
    pub fn instruction_count(&self) -> usize {
        self.emitted_ir.len()
    }
}

// ---------------------------------------------------------------------------
// Vector element descriptor
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VecElementType {
    I8, I16, I32, I64,
    U8, U16, U32, U64,
    F16, F32, F64,
    BF16,
}

impl VecElementType {
    pub fn bits(&self) -> u32 {
        match self {
            VecElementType::I8 | VecElementType::U8 => 8,
            VecElementType::I16 | VecElementType::U16 | VecElementType::F16 | VecElementType::BF16 => 16,
            VecElementType::I32 | VecElementType::U32 | VecElementType::F32 => 32,
            VecElementType::I64 | VecElementType::U64 | VecElementType::F64 => 64,
        }
    }

    pub fn is_float(&self) -> bool {
        matches!(self, VecElementType::F16 | VecElementType::F32 | VecElementType::F64 | VecElementType::BF16)
    }

    pub fn is_signed(&self) -> bool {
        matches!(self, VecElementType::I8 | VecElementType::I16 | VecElementType::I32 | VecElementType::I64)
    }
}

// ---------------------------------------------------------------------------
// Helper functions
// ---------------------------------------------------------------------------

fn op_uses_fp(op: &ReductionOp) -> bool {
    matches!(op, ReductionOp::FAdd | ReductionOp::FMul | ReductionOp::FMin
        | ReductionOp::FMax | ReductionOp::FMinimum | ReductionOp::FMaximum)
}

/// Convenience constructor.
pub fn make_x86_vectorizer_codegen(isa: VecIsaLevel) -> X86VectorizerCodeGen {
    X86VectorizerCodeGen::new(isa)
}

/// Construct with AVX2 defaults.
pub fn make_x86_vectorizer_codegen_avx2() -> X86VectorizerCodeGen {
    X86VectorizerCodeGen::new(VecIsaLevel::AVX2)
}

/// Construct with AVX-512 defaults.
pub fn make_x86_vectorizer_codegen_avx512() -> X86VectorizerCodeGen {
    X86VectorizerCodeGen::new(VecIsaLevel::AVX512F)
}

/// Construct with SSE2 defaults.
pub fn make_x86_vectorizer_codegen_sse2() -> X86VectorizerCodeGen {
    X86VectorizerCodeGen::new(VecIsaLevel::SSE2)
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests {
    use super::*;

    // --- Helpers ---

    fn make_sse2() -> X86VectorizerCodeGen {
        X86VectorizerCodeGen::new(VecIsaLevel::SSE2)
    }

    fn make_avx2() -> X86VectorizerCodeGen {
        X86VectorizerCodeGen::new(VecIsaLevel::AVX2)
    }

    fn make_avx512() -> X86VectorizerCodeGen {
        X86VectorizerCodeGen::new(VecIsaLevel::AVX512F)
    }

    fn make_avx512vl() -> X86VectorizerCodeGen {
        X86VectorizerCodeGen::new(VecIsaLevel::AVX512VL)
    }

    // --- VectorizationFactor ---

    #[test]
    fn test_vf_default() {
        let vf = VectorizationFactor::default();
        assert_eq!(vf.factor, 1);
        assert_eq!(vf.width, 128);
        assert!(!vf.is_scalable);
    }

    #[test]
    fn test_vf_new() {
        let vf = VectorizationFactor::new(4, 256);
        assert_eq!(vf.factor, 4);
        assert_eq!(vf.width, 256);
    }

    #[test]
    fn test_vf_with_interleave() {
        let vf = VectorizationFactor::new(4, 256).with_interleave(2);
        assert!(vf.is_unrolled());
        assert_eq!(vf.total_vector_lanes(), 8);
    }

    #[test]
    fn test_vf_with_scalable() {
        let vf = VectorizationFactor::new(8, 512).with_scalable(true);
        assert!(vf.is_scalable);
    }

    #[test]
    fn test_vf_vector_lanes() {
        let vf = VectorizationFactor::new(4, 256);
        assert_eq!(vf.vector_lanes(32), 8);
        assert_eq!(vf.vector_lanes(64), 4);
    }

    #[test]
    fn test_vf_total_vector_lanes() {
        let vf = VectorizationFactor::new(4, 256).with_interleave(2);
        assert_eq!(vf.total_vector_lanes(), 8);
    }

    // --- VecIsaLevel ---

    #[test]
    fn test_isa_level_widths() {
        assert_eq!(VecIsaLevel::Scalar.vector_width_bits(), 0);
        assert_eq!(VecIsaLevel::SSE.vector_width_bits(), 128);
        assert_eq!(VecIsaLevel::AVX.vector_width_bits(), 256);
        assert_eq!(VecIsaLevel::AVX512F.vector_width_bits(), 512);
    }

    #[test]
    fn test_isa_level_masked_ops() {
        assert!(!VecIsaLevel::SSE2.has_masked_ops());
        assert!(!VecIsaLevel::AVX2.has_masked_ops());
        assert!(VecIsaLevel::AVX512F.has_masked_ops());
        assert!(VecIsaLevel::AVX512VL.has_masked_ops());
    }

    #[test]
    fn test_isa_level_gather_scatter() {
        assert!(!VecIsaLevel::SSE2.has_gather_scatter());
        assert!(VecIsaLevel::AVX2.has_gather_scatter());
        assert!(VecIsaLevel::AVX512F.has_gather_scatter());
    }

    #[test]
    fn test_isa_level_scalable() {
        assert!(!VecIsaLevel::AVX2.has_scalable_vectors());
        assert!(VecIsaLevel::AVX512VL.has_scalable_vectors());
    }

    #[test]
    fn test_isa_level_kmask() {
        assert_eq!(VecIsaLevel::SSE2.kmask_registers(), 0);
        assert_eq!(VecIsaLevel::AVX512F.kmask_registers(), 8);
    }

    #[test]
    fn test_isa_level_display() {
        assert_eq!(format!("{}", VecIsaLevel::AVX2), "avx2");
        assert_eq!(format!("{}", VecIsaLevel::AVX512F), "avx512f");
    }

    #[test]
    fn test_isa_level_ordering() {
        assert!(VecIsaLevel::SSE < VecIsaLevel::AVX);
        assert!(VecIsaLevel::AVX2 < VecIsaLevel::AVX512F);
    }

    // --- VectorTypeRepr ---

    #[test]
    fn test_vec_type_new() {
        let t = VectorTypeRepr::new(32, 4, false);
        assert_eq!(t.element_bits, 32);
        assert_eq!(t.num_elements, 4);
        assert!(!t.is_floating_point);
        assert!(!t.is_scalable);
    }

    #[test]
    fn test_vec_type_scalable() {
        let t = VectorTypeRepr::new(32, 8, false).scalable();
        assert!(t.is_scalable);
    }

    #[test]
    fn test_vec_type_total_bits() {
        assert_eq!(VectorTypeRepr::new(32, 4, false).total_bits(), 128);
        assert_eq!(VectorTypeRepr::new(64, 4, false).total_bits(), 256);
    }

    #[test]
    fn test_vec_type_legal_on_sse() {
        let t = VectorTypeRepr::new(32, 4, false);
        assert!(t.is_legal_on(VecIsaLevel::SSE2));
        assert!(!t.is_legal_on(VecIsaLevel::Scalar));
    }

    #[test]
    fn test_vec_type_legal_on_avx() {
        let t = VectorTypeRepr::new(64, 4, false);
        assert!(t.is_legal_on(VecIsaLevel::AVX));
        assert!(!t.is_legal_on(VecIsaLevel::SSE2));
    }

    #[test]
    fn test_vec_type_display() {
        let t = VectorTypeRepr::new(32, 4, false);
        assert_eq!(format!("{}", t), "<4 x int>");
    }

    // --- ReductionOp ---

    #[test]
    fn test_reduction_commutative() {
        assert!(ReductionOp::Add.is_commutative());
        assert!(!ReductionOp::FMinimum.is_commutative());
    }

    #[test]
    fn test_reduction_identity_fp() {
        assert_eq!(ReductionOp::FAdd.identity_value(true), "0.0");
        assert_eq!(ReductionOp::FMul.identity_value(true), "1.0");
    }

    #[test]
    fn test_reduction_identity_int() {
        assert_eq!(ReductionOp::Add.identity_value(false), "0");
        assert_eq!(ReductionOp::Mul.identity_value(false), "1");
        assert_eq!(ReductionOp::And.identity_value(false), "-1");
    }

    #[test]
    fn test_reduction_display() {
        assert_eq!(format!("{}", ReductionOp::Add), "Add");
    }

    // --- VectorPhi ---

    #[test]
    fn test_phi_kind_display() {
        assert_eq!(format!("{}", VectorPhiKind::Induction), "induction");
        assert_eq!(format!("{}", VectorPhiKind::Reduction), "reduction");
    }

    // --- MaskedMemoryOp ---

    #[test]
    fn test_masked_mem_op_kind_display() {
        assert_eq!(format!("{}", MaskedMemOpKind::MaskedLoad), "MaskedLoad");
    }

    // --- Construction ---

    #[test]
    fn test_codegen_new_sse2() {
        let cg = make_sse2();
        assert_eq!(cg.isa_level, VecIsaLevel::SSE2);
        assert_eq!(cg.vf.width, 128);
        assert_eq!(cg.stats.total_operations(), 0);
    }

    #[test]
    fn test_codegen_new_avx2() {
        let cg = make_avx2();
        assert_eq!(cg.isa_level, VecIsaLevel::AVX2);
        assert_eq!(cg.vf.width, 256);
    }

    #[test]
    fn test_codegen_new_avx512() {
        let cg = make_avx512();
        assert_eq!(cg.isa_level, VecIsaLevel::AVX512F);
        assert_eq!(cg.vf.width, 512);
    }

    #[test]
    fn test_codegen_with_config() {
        let config = VecCodeGenConfig { enable_gather_scatter: false};
        let cg = make_avx2().with_config(config);
        assert!(!cg.config.enable_gather_scatter);
    }

    #[test]
    fn test_codegen_with_vf() {
        let vf = VectorizationFactor::new(8, 256);
        let cg = make_avx2().with_vf(vf);
        assert_eq!(cg.vf.factor, 8);
    }

    // --- Induction Phi ---

    #[test]
    fn test_create_induction_phi() {
        let mut cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        let phi = cg.create_induction_phi("i", "0", "1", 32);
        assert_eq!(phi.kind, VectorPhiKind::Induction);
        assert_eq!(phi.start_value, "0");
        assert_eq!(phi.num_lanes, 8); // 256 / 32
        assert_eq!(cg.stats.inductions_widened, 1);
    }

    #[test]
    fn test_create_induction_phi_avx512() {
        let mut cg = make_avx512().with_vf(VectorizationFactor::new(8, 512));
        let phi = cg.create_induction_phi("j", "0", "2", 64);
        assert_eq!(phi.num_lanes, 8); // 512 / 64
    }

    // --- Reduction Phi ---

    #[test]
    fn test_create_reduction_phi() {
        let mut cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        let phi = cg.create_reduction_phi("sum", ReductionOp::Add, 32, false);
        assert_eq!(phi.kind, VectorPhiKind::Reduction);
        assert_eq!(phi.start_value, "0");
    }

    #[test]
    fn test_lower_reduction_to_scalar() {
        let cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        let phi = VectorPhi {
            name: "red".into(),
            kind: VectorPhiKind::Reduction,
            vector_type: VectorTypeRepr::new(32, 8, false),
            start_value: "0".into(),
            back_edge_value: "red.acc".into(),
            num_lanes: 8,
        };
        let scalar = cg.lower_reduction_to_scalar(&phi, ReductionOp::Add);
        assert!(scalar.contains("reduce"));
        assert!(scalar.contains("Add"));
    }

    // --- Masked Load / Store ---

    #[test]
    fn test_emit_masked_load() {
        let mut cg = make_avx512().with_vf(VectorizationFactor::new(8, 512));
        let op = cg.emit_masked_load("ptr", "k1", 32, 16, 64, None);
        assert_eq!(op.kind, MaskedMemOpKind::MaskedLoad);
        assert_eq!(op.mask, "k1");
        assert_eq!(cg.stats.masked_loads_emitted, 1);
    }

    #[test]
    fn test_emit_masked_store() {
        let mut cg = make_avx512().with_vf(VectorizationFactor::new(8, 512));
        let op = cg.emit_masked_store("ptr", "val", "k2", 64, 8, 64);
        assert_eq!(op.kind, MaskedMemOpKind::MaskedStore);
        assert_eq!(op.value.as_deref(), Some("val"));
    }

    #[test]
    fn test_emit_tail_masked_memory_load() {
        let mut cg = make_avx512().with_vf(VectorizationFactor::new(8, 512));
        let op = cg.emit_tail_masked_memory("ptr", None, "k3", 32, 8, 32);
        assert!(op.is_tail);
        assert_eq!(op.kind, MaskedMemOpKind::MaskedLoad);
    }

    #[test]
    fn test_emit_tail_masked_memory_store() {
        let mut cg = make_avx512().with_vf(VectorizationFactor::new(8, 512));
        let op = cg.emit_tail_masked_memory("ptr", Some("val"), "k3", 32, 8, 32);
        assert!(op.is_tail);
        assert_eq!(op.kind, MaskedMemOpKind::MaskedStore);
    }

    // --- Gather / Scatter ---

    #[test]
    fn test_lower_gather() {
        let mut cg = make_avx512().with_vf(VectorizationFactor::new(8, 512));
        let desc = cg.lower_gather("base", "idx_vec", 4, 32, 16, Some("k1"));
        assert_eq!(desc.base_ptr, "base");
        assert_eq!(desc.scale, 4);
        assert_eq!(desc.mask.as_deref(), Some("k1"));
        assert_eq!(cg.stats.gathers_lowered, 1);
    }

    #[test]
    fn test_lower_scatter() {
        let mut cg = make_avx512().with_vf(VectorizationFactor::new(8, 512));
        let desc = cg.lower_scatter("base", "idx_vec", "data_vec", 8, 64, 8, None);
        assert_eq!(desc.data, "data_vec");
        assert_eq!(cg.stats.scatters_lowered, 1);
    }

    #[test]
    fn test_has_native_gather_scatter_avx2() {
        let cg = make_avx2();
        assert!(cg.has_native_gather_scatter());
    }

    #[test]
    fn test_has_native_gather_scatter_sse2() {
        let cg = make_sse2();
        assert!(!cg.has_native_gather_scatter());
    }

    #[test]
    fn test_simulate_gather() {
        let mut cg = make_sse2();
        let loads = cg.simulate_gather("base", &[0, 4, 8], 4, 32);
        assert_eq!(loads.len(), 3);
        assert!(loads[0].contains("base + 0*4"));
    }

    #[test]
    fn test_simulate_scatter() {
        let mut cg = make_sse2();
        let stores = cg.simulate_scatter("base", &[0, 4, 8], "v", 4, 32);
        assert_eq!(stores.len(), 3);
        assert!(stores[0].contains("store"));
        assert!(stores[0].contains("extract"));
    }

    // --- Interleave Groups ---

    #[test]
    fn test_lower_interleave_load() {
        let mut cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        let g = cg.lower_interleave_load("buf", 2, 2, 4);
        assert!(g.is_load);
        assert_eq!(g.stride, 2);
        assert_eq!(cg.stats.interleave_groups, 1);
    }

    #[test]
    fn test_lower_interleave_store() {
        let mut cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        let g = cg.lower_interleave_store("buf", 4, 4, 4);
        assert!(!g.is_load);
        assert_eq!(g.stride, 4);
    }

    #[test]
    fn test_deinterleave_shuffle_stride_2() {
        let cg = make_avx2();
        let seq = cg.deinterleave_shuffle_sequence(2, 32);
        assert_eq!(seq, vec!["unpckl", "unpckh"]);
    }

    #[test]
    fn test_deinterleave_shuffle_stride_4() {
        let cg = make_avx2();
        let seq = cg.deinterleave_shuffle_sequence(4, 32);
        assert_eq!(seq.len(), 4);
    }

    #[test]
    fn test_interleave_shuffle_sequence() {
        let cg = make_avx2();
        let seq = cg.interleave_shuffle_sequence(2, 32);
        assert_eq!(seq.len(), 2);
        assert!(seq[0].contains("rev"));
    }

    #[test]
    fn test_interleave_group_total_elements() {
        let g = InterleaveGroup::new_load("p", 3, 3, 4);
        assert_eq!(g.total_elements(), 3);
        assert_eq!(g.interleave_factor(), 3);
    }

    // --- Scalable Vectorization ---

    #[test]
    fn test_supports_scalable_avx512vl() {
        let cg = make_avx512vl();
        assert!(cg.supports_scalable_vectorization());
    }

    #[test]
    fn test_supports_scalable_no_avx2() {
        let cg = make_avx2();
        assert!(!cg.supports_scalable_vectorization());
    }

    #[test]
    fn test_scalable_vector_type() {
        let cg = make_avx512vl();
        let t = cg.scalable_vector_type(32);
        assert!(t.is_scalable);
        assert_eq!(t.num_elements, 16); // 512 / 32
    }

    #[test]
    fn test_scalable_step_vector() {
        let cg = make_avx512vl();
        let step = cg.create_scalable_step_vector(4, 32);
        assert!(step.contains("vscale_step"));
        assert!(step.contains("16"));
    }

    // --- VPlan ---

    #[test]
    fn test_vplan_new() {
        let vplan = VPlan::new("test");
        assert_eq!(vplan.name, "test");
        assert!(vplan.is_empty());
    }

    #[test]
    fn test_vplan_add_recipe() {
        let mut vplan = VPlan::new("t");
        vplan.add_recipe(VPRecipe::VPBackedgeBranch);
        assert_eq!(vplan.recipe_count(), 1);
    }

    #[test]
    fn test_vplan_add_phi() {
        let mut vplan = VPlan::new("t");
        vplan.add_phi(VectorPhi {
            name: "p".into(),
            kind: VectorPhiKind::Induction,
            vector_type: VectorTypeRepr::new(32, 4, false),
            start_value: "0".into(),
            back_edge_value: "1".into(),
            num_lanes: 4,
        });
        assert_eq!(vplan.vector_phis.len(), 1);
    }

    #[test]
    fn test_build_vplan() {
        let mut cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        let vplan = cg.build_vplan("lp", vec![VPRecipe::VPBackedgeBranch], vec![]);
        assert_eq!(vplan.name, "lp");
        assert_eq!(cg.vplan_generated, 1);
    }

    #[test]
    fn test_execute_vplan_empty() {
        let mut cg = make_avx2();
        let vplan = VPlan::new("empty");
        let ir = cg.execute_vplan(&vplan);
        assert!(ir.is_empty());
    }

    #[test]
    fn test_execute_vplan_with_phi() {
        let mut cg = make_avx2();
        let mut vplan = VPlan::new("lp");
        vplan.add_phi(VectorPhi {
            name: "ind".into(),
            kind: VectorPhiKind::Induction,
            vector_type: VectorTypeRepr::new(32, 4, false),
            start_value: "0".into(),
            back_edge_value: "ind.next".into(),
            num_lanes: 4,
        });
        let ir = cg.execute_vplan(&vplan);
        assert_eq!(ir.len(), 1);
        assert!(ir[0].contains("phi"));
    }

    #[test]
    fn test_execute_vplan_with_recipes() {
        let mut cg = make_avx2();
        let mut vplan = VPlan::new("lp");
        vplan.add_recipe(VPRecipe::Widen {
            original_opcode: "add".into(),
            result: "r".into(),
            operands: vec!["a".into(), "b".into()],
            vector_type: VectorTypeRepr::new(32, 4, false),
        });
        vplan.add_recipe(VPRecipe::VPTailMask {
            result: "tm".into(),
            num_lanes: 4,
            vf: 4,
        });
        let ir = cg.execute_vplan(&vplan);
        assert_eq!(ir.len(), 2);
        assert!(ir[0].contains("vector.add"));
        assert!(ir[1].contains("vp.tail.mask"));
    }

    #[test]
    fn test_execute_vplan_masked_op() {
        let mut cg = make_avx512();
        let mut vplan = VPlan::new("lp");
        vplan.add_recipe(VPRecipe::MaskedOp {
            opcode: "load".into(),
            result: "r".into(),
            operands: vec!["ptr".into()],
            mask: "k1".into(),
            passthru: Some("zero".into()),
        });
        let ir = cg.execute_vplan(&vplan);
        assert!(ir[0].contains("masked.load"));
        assert!(ir[0].contains("passthru=zero"));
    }

    #[test]
    fn test_execute_vplan_gather_scatter() {
        let mut cg = make_avx512();
        let mut vplan = VPlan::new("lp");
        vplan.add_recipe(VPRecipe::VPGather {
            descriptor: GatherDescriptor {
                base_ptr: "b".into(),
                indices: "i".into(),
                scale: 4,
                vector_type: VectorTypeRepr::new(32, 16, false),
                mask: None,
                is_signed_indices: true,
            },
            result: "g".into(),
        });
        vplan.add_recipe(VPRecipe::VPScatter {
            descriptor: ScatterDescriptor {
                base_ptr: "b".into(),
                indices: "i".into(),
                data: "d".into(),
                scale: 8,
                vector_type: VectorTypeRepr::new(64, 8, false),
                mask: None,
            },
        });
        let ir = cg.execute_vplan(&vplan);
        assert_eq!(ir.len(), 2);
        assert!(ir[0].contains("vp.gather"));
        assert!(ir[1].contains("vp.scatter"));
    }

    // --- Generate Vector Loop Body ---

    #[test]
    fn test_generate_vector_loop_body_empty() {
        let mut cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        let vplan = cg.generate_vector_loop_body("lp", &[], &[], None, None);
        assert_eq!(vplan.recipe_count(), 0);
        assert_eq!(vplan.vector_phis.len(), 0);
    }

    #[test]
    fn test_generate_vector_loop_body_with_induction() {
        let mut cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        let vplan = cg.generate_vector_loop_body(
            "lp", &[], &[], Some(("0", "1")), Some(16),
        );
        assert_eq!(vplan.vector_phis.len(), 1);
        assert!(vplan.vector_phis[0].name.contains("vec.ind"));
    }

    #[test]
    fn test_generate_vector_loop_body_with_reduction() {
        let mut cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        let vplan = cg.generate_vector_loop_body(
            "lp", &[],
            &[(ReductionOp::Add, "acc".into(), "x".into())],
            None, None,
        );
        assert_eq!(vplan.vector_phis.len(), 1);
        assert_eq!(vplan.vector_phis[0].kind, VectorPhiKind::Reduction);
    }

    #[test]
    fn test_generate_vector_loop_body_with_tail() {
        let mut cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        let vplan = cg.generate_vector_loop_body(
            "lp", &[], &[], None, Some(17), // 17 % 4 = 1 → tail
        );
        // Should have a tail mask recipe
        let has_tail = vplan.recipes.iter().any(|r| matches!(r, VPRecipe::VPTailMask { .. }));
        assert!(has_tail);
    }

    #[test]
    fn test_generate_vector_loop_body_no_tail() {
        let mut cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        let vplan = cg.generate_vector_loop_body(
            "lp", &[], &[], None, Some(16), // 16 % 4 = 0 → no tail
        );
        let has_tail = vplan.recipes.iter().any(|r| matches!(r, VPRecipe::VPTailMask { .. }));
        assert!(!has_tail);
    }

    // --- Vectorization Pipeline ---

    #[test]
    fn test_vectorize_loop_simple() {
        let mut cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        let result = cg.vectorize_loop(
            "test_loop",
            &["load".into(), "add".into(), "store".into()],
            &[],
            Some(("0", "1")),
            Some(16),
        );
        assert!(result.success);
        assert!(!result.emitted_ir.is_empty());
        assert_eq!(result.loop_name, "test_loop");
    }

    #[test]
    fn test_vectorize_loop_with_all_features() {
        let mut cg = make_avx512().with_vf(VectorizationFactor::new(8, 512));
        let result = cg.vectorize_loop(
            "rich",
            &["mul".into()],
            &[(ReductionOp::FAdd, "sum".into(), "elem".into())],
            Some(("0", "1")),
            Some(31),
        );
        assert!(result.success);
        assert!(cg.loops_vectorized_count > 0);
    }

    // --- Tail Mask ---

    #[test]
    fn test_generate_tail_mask_exact() {
        let mut cg = make_avx2();
        let mask = cg.generate_tail_mask(16, 4);
        assert!(mask.is_none());
    }

    #[test]
    fn test_generate_tail_mask_remainder() {
        let mut cg = make_avx2();
        let mask = cg.generate_tail_mask(17, 4);
        assert!(mask.is_some());
        assert_eq!(cg.stats.tail_masks_generated, 1);
    }

    // --- Vector Predication ---

    #[test]
    fn test_create_predication() {
        let mut cg = make_avx512();
        let vp = cg.create_predication(Some("evl"), Some("k1"), TailStrategy::PredicateLastIteration);
        assert_eq!(vp.explicit_vector_length.as_deref(), Some("evl"));
        assert_eq!(vp.tail_strategy, TailStrategy::PredicateLastIteration);
    }

    #[test]
    fn test_tail_strategy_display() {
        assert_eq!(format!("{}", TailStrategy::None), "none");
        assert_eq!(format!("{}", TailStrategy::ScalarEpilogue), "scalar-epilogue");
    }

    // --- Vector ABI ---

    #[test]
    fn test_build_vector_abi() {
        let mut cg = make_avx512();
        let abi = cg.build_vector_abi("foo", Some(8), &[0], &[(1, 2)], &[2]);
        assert_eq!(abi.function_name, "foo");
        assert_eq!(abi.simdlen, Some(8));
        assert!(abi.uniform_args.contains(&0));
        assert_eq!(abi.linear_args.get(&1), Some(&2));
        assert!(abi.aligned_args.contains(&2));
        assert!(!abi.vector_variants.is_empty());
        assert_eq!(cg.stats.vector_abis, 1);
    }

    #[test]
    fn test_vec_func_abi_best_variant() {
        let mut abi = VecFuncABI::new("bar");
        abi.add_variant(VecFuncVariant {
            mangled_name: "v1".into(),
            isa_level: VecIsaLevel::AVX2,
            vector_width: 256,
            mask_param: false,
            is_openmp_declare_simd: true,
        });
        abi.add_variant(VecFuncVariant {
            mangled_name: "v2".into(),
            isa_level: VecIsaLevel::AVX512F,
            vector_width: 512,
            mask_param: true,
            is_openmp_declare_simd: true,
        });
        let best = abi.best_variant_for(VecIsaLevel::AVX512F);
        assert_eq!(best.unwrap().vector_width, 512);
    }

    #[test]
    fn test_vec_func_abi_fallback() {
        let mut abi = VecFuncABI::new("baz");
        abi.add_variant(VecFuncVariant {
            mangled_name: "v1".into(),
            isa_level: VecIsaLevel::AVX,
            vector_width: 256,
            mask_param: false,
            is_openmp_declare_simd: true,
        });
        let best = abi.best_variant_for(VecIsaLevel::AVX512F);
        assert_eq!(best.unwrap().vector_width, 256);
    }

    // --- Available Vector Widths ---

    #[test]
    fn test_available_widths_sse2() {
        let cg = make_sse2();
        let widths = cg.available_vector_widths();
        assert_eq!(widths, vec![Some(128)]);
    }

    #[test]
    fn test_available_widths_avx512vl() {
        let cg = make_avx512vl();
        let widths = cg.available_vector_widths();
        assert!(widths.contains(&Some(512)));
        assert!(widths.contains(&Some(256)));
        assert!(widths.contains(&Some(128)));
        // scalable
        assert!(widths.contains(&None));
    }

    // --- Effective Vector Length ---

    #[test]
    fn test_effective_vector_length_scalar() {
        let cg = make_avx2().with_vf(VectorizationFactor::new(4, 256));
        assert_eq!(cg.effective_vector_length(32), 4);
    }

    #[test]
    fn test_effective_vector_length_scalable() {
        let cg = make_avx512vl().with_vf(
            VectorizationFactor::new(8, 512).with_scalable(true)
        );
        assert_eq!(cg.effective_vector_length(32), 16); // 512/32
    }

    // --- VecElementType ---

    #[test]
    fn test_element_bits() {
        assert_eq!(VecElementType::I8.bits(), 8);
        assert_eq!(VecElementType::F32.bits(), 32);
        assert_eq!(VecElementType::F64.bits(), 64);
    }

    #[test]
    fn test_element_is_float() {
        assert!(VecElementType::F32.is_float());
        assert!(!VecElementType::I32.is_float());
    }

    #[test]
    fn test_element_is_signed() {
        assert!(VecElementType::I32.is_signed());
        assert!(!VecElementType::U32.is_signed());
    }

    // --- Reset ---

    #[test]
    fn test_reset_clears_state() {
        let mut cg = make_avx2();
        cg.create_induction_phi("i", "0", "1", 32);
        cg.emit_masked_load("p", "k1", 32, 8, 32, None);
        cg.lower_gather("b", "i", 4, 32, 8, None);
        cg.lower_interleave_load("b", 2, 2, 4);
        assert!(!cg.emitted_phis.is_empty());
        cg.reset();
        assert!(cg.emitted_phis.is_empty());
        assert!(cg.emitted_masked_ops.is_empty());
        assert!(cg.emitted_gathers.is_empty());
        assert!(cg.emitted_interleave_groups.is_empty());
        assert!(cg.vector_abi.is_none());
        assert!(cg.current_vplan.is_none());
    }

    // --- Summary ---

    #[test]
    fn test_summary_returns_string() {
        let cg = make_avx2();
        let s = cg.summary();
        assert!(s.contains("X86VectorizerCodeGen"));
        assert!(s.contains("loops="));
    }

    // --- VecCodeGenResult ---

    #[test]
    fn test_result_instruction_count() {
        let result = VecCodeGenResult {
            loop_name: "t".into(),
            vf: 4,
            vplan: VPlan::new("t"),
            emitted_ir: vec!["a".into(), "b".into(), "c".into()],
            masked_ops: 0,
            gathers: 0,
            scatters: 0,
            interleave_groups: 0,
            success: true,
        };
        assert_eq!(result.instruction_count(), 3);
    }

    // --- VecCodeGenStats ---

    #[test]
    fn test_stats_total_operations() {
        let mut stats = VecCodeGenStats::default();
        stats.phi_nodes_emitted = 5;
        stats.gathers_lowered = 2;
        assert_eq!(stats.total_operations(), 7);
    }

    // --- Convenience constructors ---

    #[test]
    fn test_make_constructors() {
        let a = make_x86_vectorizer_codegen(VecIsaLevel::AVX2);
        assert_eq!(a.isa_level, VecIsaLevel::AVX2);
        let b = make_x86_vectorizer_codegen_avx2();
        assert_eq!(b.isa_level, VecIsaLevel::AVX2);
        let c = make_x86_vectorizer_codegen_avx512();
        assert_eq!(c.isa_level, VecIsaLevel::AVX512F);
        let d = make_x86_vectorizer_codegen_sse2();
        assert_eq!(d.isa_level, VecIsaLevel::SSE2);
    }

    // --- Config ---

    #[test]
    fn test_config_defaults() {
        let config = VecCodeGenConfig::default();
        assert!(config.enable_masked_load_store);
        assert!(config.enable_gather_scatter);
        assert!(!config.enable_scalable_vectors);
        assert_eq!(config.max_vf, 64);
        assert_eq!(config.min_profitable_vf, 2);
    }

    // --- Stress / Coverage ---

    #[test]
    fn test_all_reduction_ops_identity() {
        let ops = [
            ReductionOp::Add, ReductionOp::FAdd, ReductionOp::Mul, ReductionOp::FMul,
            ReductionOp::And, ReductionOp::Or, ReductionOp::Xor,
            ReductionOp::SMin, ReductionOp::SMax, ReductionOp::UMin, ReductionOp::UMax,
            ReductionOp::FMin, ReductionOp::FMax, ReductionOp::FMinimum, ReductionOp::FMaximum,
        ];
        for op in &ops {
            let fp_ident = op.identity_value(true);
            let int_ident = op.identity_value(false);
            assert!(!fp_ident.is_empty());
            assert!(!int_ident.is_empty());
        }
    }

    #[test]
    fn test_all_isa_levels_have_width() {
        let levels = [
            VecIsaLevel::Scalar, VecIsaLevel::SSE, VecIsaLevel::SSE2, VecIsaLevel::SSE3,
            VecIsaLevel::SSSE3, VecIsaLevel::SSE41, VecIsaLevel::SSE42,
            VecIsaLevel::AVX, VecIsaLevel::AVX2,
            VecIsaLevel::AVX512F, VecIsaLevel::AVX512BW, VecIsaLevel::AVX512DQ,
            VecIsaLevel::AVX512VL, VecIsaLevel::AVX512VNNI, VecIsaLevel::AVX512BF16,
            VecIsaLevel::AVX512FP16,
        ];
        for lvl in &levels {
            let w = lvl.vector_width_bits();
            if *lvl == VecIsaLevel::Scalar {
                assert_eq!(w, 0);
            } else {
                assert!(w >= 128);
            }
        }
    }

    #[test]
    fn test_vf_never_zero() {
        let cg = make_avx2();
        assert!(cg.vf.factor > 0);
        assert!(cg.vf.width > 0);
    }

    #[test]
    fn test_multiple_loops_accumulate_stats() {
        let mut cg = make_avx512().with_vf(VectorizationFactor::new(8, 512));
        cg.vectorize_loop("a", &[], &[], None, None);
        cg.vectorize_loop("b", &[], &[], None, None);
        cg.vectorize_loop("c", &[], &[], None, None);
        assert_eq!(cg.loops_vectorized_count, 3);
        assert_eq!(cg.vplan_generated, 3);
    }

    #[test]
    fn test_interleave_group_construction() {
        let g = InterleaveGroup::new_load("buf", 3, 3, 8);
        assert_eq!(g.member_indices, vec![0, 1, 2]);
        let s = InterleaveGroup::new_store("buf", 4, 4, 8);
        assert_eq!(s.member_indices, vec![0, 1, 2, 3]);
    }

    #[test]
    fn test_vplan_live_outs() {
        let mut vplan = VPlan::new("t");
        vplan.live_outs.push("sum".into());
        assert_eq!(vplan.live_outs.len(), 1);
    }

    #[test]
    fn test_config_prefer_width() {
        let mut config = VecCodeGenConfig::default();
        assert!(!config.prefer_128bit_vectors);
        config.prefer_256bit_vectors = true;
        assert!(config.prefer_256bit_vectors);
    }

    #[test]
    fn test_multiple_interleave_groups() {
        let mut cg = make_avx2();
        cg.lower_interleave_load("a", 2, 2, 4);
        cg.lower_interleave_load("b", 4, 4, 4);
        cg.lower_interleave_store("c", 3, 3, 4);
        assert_eq!(cg.stats.interleave_groups, 3);
    }

    #[test]
    fn test_masked_and_gather_combined() {
        let mut cg = make_avx512().with_vf(VectorizationFactor::new(8, 512));
        cg.emit_masked_load("p", "k1", 32, 16, 64, None);
        cg.lower_gather("b", "i", 4, 32, 16, Some("k2"));
        cg.lower_scatter("b", "i", "d", 8, 64, 8, Some("k3"));
        assert_eq!(cg.stats.masked_loads_emitted, 1);
        assert_eq!(cg.stats.gathers_lowered, 1);
        assert_eq!(cg.stats.scatters_lowered, 1);
    }

    #[test]
    fn test_no_crash_on_empty_input() {
        let mut cg = make_avx2();
        let result = cg.vectorize_loop("empty", &[], &[], None, None);
        assert!(result.success);
        assert!(result.emitted_ir.is_empty());
    }

    #[test]
    fn test_vplan_block_names() {
        let vplan = VPlan::new("t");
        assert_eq!(vplan.vector_loop_header, "vector.body");
        assert_eq!(vplan.middle_block, "middle.block");
        assert_eq!(vplan.scalar_loop_header, "scalar.body");
        assert_eq!(vplan.exit_block, "exit");
    }

    #[test]
    fn test_uniform_args_set() {
        let abi = VecFuncABI::new("f");
        assert!(abi.uniform_args.is_empty());
    }

    #[test]
    fn test_linear_args_map() {
        let abi = VecFuncABI::new("f");
        assert!(abi.linear_args.is_empty());
    }

    #[test]
    fn test_vplan_clone() {
        let vplan = VPlan::new("t");
        let cloned = vplan.clone();
        assert_eq!(cloned.name, vplan.name);
        assert_eq!(cloned.recipe_count(), vplan.recipe_count());
    }

    #[test]
    fn test_vec_codegen_result_clone() {
        let result = VecCodeGenResult {
            loop_name: "t".into(),
            vf: 4,
            vplan: VPlan::new("t"),
            emitted_ir: vec!["a".into()],
            masked_ops: 1,
            gathers: 0,
            scatters: 0,
            interleave_groups: 0,
            success: true,
        };
        let cloned = result.clone();
        assert_eq!(cloned.loop_name, result.loop_name);
        assert_eq!(cloned.vf, result.vf);
    }

    #[test]
    fn test_full_pipeline_sse2() {
        let mut cg = make_x86_vectorizer_codegen_sse2()
            .with_vf(VectorizationFactor::new(2, 128));
        let result = cg.vectorize_loop(
            "sse_loop",
            &["ld".into(), "add".into()],
            &[(ReductionOp::Add, "acc".into(), "ld".into())],
            Some(("0", "1")),
            Some(8),
        );
        assert!(result.success);
        assert!(result.instruction_count() > 0);
    }

    #[test]
    fn test_full_pipeline_avx512() {
        let mut cg = make_x86_vectorizer_codegen_avx512()
            .with_vf(VectorizationFactor::new(16, 512));
        cg.emit_masked_load("p", "k1", 64, 8, 64, Some("zero"));
        cg.lower_gather("b", "idx", 8, 64, 8, Some("k2"));
        let result = cg.vectorize_loop(
            "avx512_loop",
            &["fma".into()],
            &[(ReductionOp::FAdd, "fsum".into(), "fma".into())],
            Some(("0.0", "1.0")),
            Some(64),
        );
        assert!(result.success);
        assert!(cg.stats.total_operations() > 0);
    }
}