//! X86 Hardware Loops — hardware loop intrinsic lowering, Intel TSX HLE
//! loops, Intel CET-compatible loop patterns, loop counter optimization
//! (DEC+JNZ fusion), zero-overhead loop optimization, and hardware-loop
//! legality checking.
//!
//! On x86-64, there are no dedicated hardware-loop instructions (unlike
//! ARM's DLS/WLS or Hexagon's LOOP0/LOOP1). Instead, this module
//! recognizes and optimizes idiomatic counted-loop patterns:
//!
//! - `set.loop.iterations` / `loop.decrement.reg` / `loop.decrement`
//! intrinsic lowering
//! - Intel TSX HLE (Hardware Lock Elision) loop patterns with
//! XACQUIRE/XRELEASE prefix decoration
//! - Intel CET (Control-flow Enforcement Technology) compatible loop
//! patterns with ENDBR at loop targets
//! - Loop counter optimization: DEC+JNZ macro-fusion optimization on
//! Intel Sandy Bridge+ and AMD Zen+
//! - Zero-overhead loop micro-optimizations (LSD-friendly loop sizing
//! and alignment)
//! - Hardware loop legality checking: verifying that loops are
//! structured, have a single back-edge, known trip count, etc.
//!
//! Clean-room behavioral reconstruction from:
//! - Intel® 64 and IA-32 Architectures Optimization Reference Manual
//! - Intel® TSX (Transactional Synchronization Extensions) specification
//! - Intel® CET (Control-flow Enforcement Technology) specification
//! - Agner Fog's microarchitecture documentation
//! - AMD64 Architecture Programmer's Manual (volumes 1-5)
//!
//! Zero LLVM source code consultation. All behavior reconstructed from
//! published specifications and black-box oracle interrogation.
use std::collections::{HashMap, HashSet};
use std::fmt;
// ---------------------------------------------------------------------------
// Target microarchitecture
// ---------------------------------------------------------------------------
/// X86 microarchitecture for hardware-loop tuning decisions.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum HWLoopMicroArch {
Generic,
SandyBridge,
IvyBridge,
Haswell,
Broadwell,
Skylake,
KabyLake,
CoffeeLake,
CascadeLake,
CometLake,
IceLake,
TigerLake,
RocketLake,
AlderLakeP,
AlderLakeE,
RaptorLakeP,
SapphireRapids,
EmeraldRapids,
GraniteRapids,
Zen1,
Zen2,
Zen3,
Zen4,
Zen5,
}
impl fmt::Display for HWLoopMicroArch {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
impl HWLoopMicroArch {
/// Whether DEC+JNZ macro-fusion is available.
pub fn has_dec_jcc_fusion(&self) -> bool {
match self {
HWLoopMicroArch::SandyBridge
| HWLoopMicroArch::IvyBridge
| HWLoopMicroArch::Haswell
| HWLoopMicroArch::Broadwell
| HWLoopMicroArch::Skylake
| HWLoopMicroArch::KabyLake
| HWLoopMicroArch::CoffeeLake
| HWLoopMicroArch::CascadeLake
| HWLoopMicroArch::CometLake
| HWLoopMicroArch::IceLake
| HWLoopMicroArch::TigerLake
| HWLoopMicroArch::RocketLake
| HWLoopMicroArch::AlderLakeP
| HWLoopMicroArch::RaptorLakeP
| HWLoopMicroArch::SapphireRapids
| HWLoopMicroArch::EmeraldRapids
| HWLoopMicroArch::GraniteRapids
| HWLoopMicroArch::Zen1
| HWLoopMicroArch::Zen2
| HWLoopMicroArch::Zen3
| HWLoopMicroArch::Zen4
| HWLoopMicroArch::Zen5 => true,
_ => false,
}
}
/// Whether the LSD (Loop Stream Detector) is available.
pub fn has_lsd(&self) -> bool {
match self {
HWLoopMicroArch::SandyBridge
| HWLoopMicroArch::IvyBridge
| HWLoopMicroArch::Haswell
| HWLoopMicroArch::Broadwell
| HWLoopMicroArch::Skylake
| HWLoopMicroArch::KabyLake
| HWLoopMicroArch::CoffeeLake
| HWLoopMicroArch::CascadeLake
| HWLoopMicroArch::CometLake
| HWLoopMicroArch::IceLake
| HWLoopMicroArch::TigerLake
| HWLoopMicroArch::RocketLake => true,
// Disabled on Alder Lake+ with GD
_ => false,
}
}
/// Whether the Zen Op Cache can hold the loop (zero-overhead-like).
pub fn has_op_cache_loop(&self) -> bool {
matches!(
self,
HWLoopMicroArch::Zen1
| HWLoopMicroArch::Zen2
| HWLoopMicroArch::Zen3
| HWLoopMicroArch::Zen4
| HWLoopMicroArch::Zen5
)
}
/// Maximum uops the LSD can buffer.
pub fn lsd_capacity_uops(&self) -> u32 {
match self {
HWLoopMicroArch::SandyBridge | HWLoopMicroArch::IvyBridge => 28,
HWLoopMicroArch::Haswell | HWLoopMicroArch::Broadwell => 28,
HWLoopMicroArch::Skylake
| HWLoopMicroArch::KabyLake
| HWLoopMicroArch::CoffeeLake
| HWLoopMicroArch::CascadeLake
| HWLoopMicroArch::CometLake => 28,
HWLoopMicroArch::IceLake | HWLoopMicroArch::TigerLake | HWLoopMicroArch::RocketLake => {
28
}
_ => 0,
}
}
/// Maximum size for Zen Op Cache (instructions in the loop body).
pub fn op_cache_capacity_insts(&self) -> u32 {
match self {
HWLoopMicroArch::Zen1 | HWLoopMicroArch::Zen2 => 4096,
HWLoopMicroArch::Zen3 | HWLoopMicroArch::Zen4 | HWLoopMicroArch::Zen5 => 4096,
_ => 0,
}
}
}
// ---------------------------------------------------------------------------
// Hardware loop intrinsic kinds
// ---------------------------------------------------------------------------
/// Kinds of hardware-loop intrinsics that may be lowered.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum HWLoopIntrinsic {
/// `llvm.set.loop.iterations.i32/i64` — sets the loop iteration count.
SetLoopIterations,
/// `llvm.loop.decrement.reg.i32/i64` — decrements the loop counter
/// and returns a boolean indicating whether the loop should continue.
LoopDecrementReg,
/// `llvm.loop.decrement.i32/i64` — decrements the loop counter with
/// an explicit trip-count comparison.
LoopDecrement,
}
impl fmt::Display for HWLoopIntrinsic {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let s = match self {
HWLoopIntrinsic::SetLoopIterations => "set.loop.iterations",
HWLoopIntrinsic::LoopDecrementReg => "loop.decrement.reg",
HWLoopIntrinsic::LoopDecrement => "loop.decrement",
};
write!(f, "{}", s)
}
}
// ---------------------------------------------------------------------------
// Lowering result
// ---------------------------------------------------------------------------
/// The result of lowering a hardware-loop intrinsic.
#[derive(Debug, Clone)]
pub struct HWLoopLowering {
/// The original intrinsic kind.
pub kind: HWLoopIntrinsic,
/// The emitted X86 instruction sequence.
pub x86_sequence: Vec<String>,
/// The register(s) used for the loop counter.
pub counter_reg: Option<String>,
/// Whether a compare instruction was emitted.
pub has_compare: bool,
/// Whether the DEC+JNZ fusion pattern was applied.
pub uses_dec_jnz_fusion: bool,
/// Estimated cycles for the loop back-edge.
pub back_edge_cycles: u32,
}
impl HWLoopLowering {
pub fn new(kind: HWLoopIntrinsic) -> Self {
Self {
kind,
x86_sequence: Vec::new(),
counter_reg: None,
has_compare: false,
uses_dec_jnz_fusion: false,
back_edge_cycles: 1,
}
}
}
// ---------------------------------------------------------------------------
// TSX HLE (Hardware Lock Elision) loop descriptors
// ---------------------------------------------------------------------------
/// Describes a TSX HLE (Hardware Lock Elision) loop region.
#[derive(Debug, Clone)]
pub struct TSXHLELoop {
/// Start address of the transactional region.
pub start_label: String,
/// End address of the transactional region.
pub end_label: String,
/// Whether the region uses XACQUIRE prefix.
pub uses_xacquire: bool,
/// Whether the region uses XRELEASE prefix.
pub uses_xrelease: bool,
/// Whether an explicit XABORT was detected.
pub has_xabort: bool,
/// The abort handler label.
pub abort_handler: Option<String>,
/// Maximum retry count before falling back to a non-HLE lock.
pub max_retries: u32,
}
impl TSXHLELoop {
pub fn new(start: &str, end: &str) -> Self {
Self {
start_label: start.into(),
end_label: end.into(),
uses_xacquire: false,
uses_xrelease: false,
has_xabort: false,
abort_handler: None,
max_retries: 3,
}
}
}
// ---------------------------------------------------------------------------
// CET-compatible loop patterns
// ---------------------------------------------------------------------------
/// Describes a CET (Control-flow Enforcement Technology) compatible loop.
#[derive(Debug, Clone)]
pub struct CETLoopPattern {
/// The loop header block label.
pub header_label: String,
/// Whether the header has an ENDBR (End Branch) instruction.
pub has_endbr: bool,
/// Whether the loop is CET-compatible (indirect branches checked).
pub is_cet_compatible: bool,
/// Indirect branch targets within the loop.
pub indirect_targets: Vec<String>,
/// Whether shadow stack is enforced.
pub shadow_stack_enforced: bool,
}
impl CETLoopPattern {
pub fn new(header: &str) -> Self {
Self {
header_label: header.into(),
has_endbr: false,
is_cet_compatible: true,
indirect_targets: Vec::new(),
shadow_stack_enforced: false,
}
}
}
// ---------------------------------------------------------------------------
// Loop counter optimization descriptor
// ---------------------------------------------------------------------------
/// Decoding of a loop counter idiom for optimization.
#[derive(Debug, Clone)]
pub struct LoopCounterPattern {
/// The register used as the loop counter.
pub counter_reg: String,
/// Initial value of the counter.
pub initial_value: i64,
/// Step per iteration (negative for decrementing loops).
pub step: i64,
/// The comparison predicate.
pub predicate: LoopCounterPredicate,
/// Whether the loop is compatible with DEC+JNZ fusion.
pub fusion_compatible: bool,
/// Whether the loop counter is also used as an address index.
pub is_address_index: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LoopCounterPredicate {
EQ,
NE,
LT,
LE,
GT,
GE,
ULT,
ULE,
UGT,
UGE,
}
impl fmt::Display for LoopCounterPredicate {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
// ---------------------------------------------------------------------------
// Zero-overhead loop optimization
// ---------------------------------------------------------------------------
/// Configuration for zero-overhead loop optimizations.
#[derive(Debug, Clone)]
pub struct ZeroOverheadConfig {
/// Whether to attempt LSD-friendly loop sizing.
pub enable_lsd_optimization: bool,
/// Maximum uop count for LSD fitting.
pub lsd_max_uops: u32,
/// Whether to align loop headers to 32-byte boundaries.
pub align_loop_header: bool,
/// Alignment in bytes.
pub header_alignment: u32,
/// Whether to insert NOP padding for optimal LSD behavior.
pub insert_nop_padding: bool,
/// Whether to unroll small loops to fit uop cache.
pub unroll_for_uop_cache: bool,
}
impl Default for ZeroOverheadConfig {
fn default() -> Self {
Self {
enable_lsd_optimization: true,
lsd_max_uops: 28,
align_loop_header: true,
header_alignment: 32,
insert_nop_padding: true,
unroll_for_uop_cache: true,
}
}
}
/// Analysis of whether a loop can benefit from zero-overhead techniques.
#[derive(Debug, Clone)]
pub struct ZeroOverheadAnalysis {
/// Total uops in the loop body.
pub total_uops: u32,
/// Whether the loop fits in the LSD.
pub fits_in_lsd: bool,
/// Whether the loop fits in the uop cache.
pub fits_in_uop_cache: bool,
/// Whether the loop is already aligned.
pub is_aligned: bool,
/// Recommended action.
pub recommendation: ZeroOverheadAction,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ZeroOverheadAction {
/// No action needed — loop already optimized.
None,
/// Align the loop header.
AlignHeader,
/// Insert NOPs for LSD-friendly layout.
InsertNOPs,
/// Unroll the loop to better fill the LSD/uop cache.
Unroll,
/// The loop is too large for zero-overhead techniques.
TooLarge,
}
impl fmt::Display for ZeroOverheadAction {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
// ---------------------------------------------------------------------------
// Hardware Loop Legality Checking
// ---------------------------------------------------------------------------
/// Legality requirements for a hardware loop.
#[derive(Debug, Clone)]
pub struct HWLoopLegality {
/// Whether the loop has a single back-edge.
pub is_single_backedge: bool,
/// Whether the loop has a known trip count.
pub has_known_trip_count: bool,
/// Trip count estimate if known.
pub trip_count: Option<u64>,
/// Whether the loop body contains calls (disqualifies hardware loop).
pub contains_calls: bool,
/// Whether the loop body contains indirect branches.
pub contains_indirect_branches: bool,
/// Whether the loop is reducible.
pub is_reducible: bool,
/// Whether the loop counter is live-out (used after the loop).
pub counter_live_out: bool,
/// Overall legality verdict.
pub is_legal: bool,
/// Reason if not legal.
pub rejection_reason: Option<String>,
}
impl HWLoopLegality {
pub fn legal() -> Self {
Self {
is_single_backedge: true,
has_known_trip_count: true,
trip_count: None,
contains_calls: false,
contains_indirect_branches: false,
is_reducible: true,
counter_live_out: false,
is_legal: true,
rejection_reason: None,
}
}
pub fn reject(reason: &str) -> Self {
Self {
is_single_backedge: false,
has_known_trip_count: false,
trip_count: None,
contains_calls: false,
contains_indirect_branches: false,
is_reducible: false,
counter_live_out: false,
is_legal: false,
rejection_reason: Some(reason.into()),
}
}
}
// ---------------------------------------------------------------------------
// Main Hardware Loop Manager
// ---------------------------------------------------------------------------
#[derive(Debug)]
pub struct X86HardwareLoops {
/// Target microarchitecture for tuning.
pub microarch: HWLoopMicroArch,
/// Intrinsic lowerings performed.
pub lowerings: Vec<HWLoopLowering>,
/// TSX HLE loop regions.
pub tsx_regions: Vec<TSXHLELoop>,
/// CET-compatible loop patterns.
pub cet_patterns: Vec<CETLoopPattern>,
/// Loop counter patterns found.
pub counter_patterns: Vec<LoopCounterPattern>,
/// Zero-overhead loop analyses.
pub zero_overhead_analyses: Vec<ZeroOverheadAnalysis>,
/// Legality checks performed.
pub legality_checks: Vec<HWLoopLegality>,
/// Counter of loops optimized.
pub loops_optimized: u32,
/// Loops rejected.
pub loops_rejected: u32,
/// Global statistics.
pub stats: HWLoopStats,
/// Zero-overhead config.
pub zo_config: ZeroOverheadConfig,
/// Whether TSX HLE is available on this target.
pub has_tsx_hle: bool,
/// Whether CET is enforced.
pub cet_enforced: bool,
}
#[derive(Debug, Clone, Default)]
pub struct HWLoopStats {
pub set_loop_iterations_lowered: u32,
pub loop_decrement_reg_lowered: u32,
pub loop_decrement_lowered: u32,
pub dec_jnz_fusion_applied: u32,
pub tsx_hle_regions: u32,
pub cet_patterns_validated: u32,
pub zero_overhead_fit_lsd: u32,
pub zero_overhead_fit_uop_cache: u32,
pub zero_overhead_aligned: u32,
pub zero_overhead_unrolled: u32,
pub legal_loops: u32,
pub illegal_loops: u32,
}
impl HWLoopStats {
pub fn total_lowered(&self) -> u32 {
self.set_loop_iterations_lowered
+ self.loop_decrement_reg_lowered
+ self.loop_decrement_lowered
}
pub fn merge(&mut self, other: &HWLoopStats) {
self.set_loop_iterations_lowered += other.set_loop_iterations_lowered;
self.loop_decrement_reg_lowered += other.loop_decrement_reg_lowered;
self.loop_decrement_lowered += other.loop_decrement_lowered;
self.dec_jnz_fusion_applied += other.dec_jnz_fusion_applied;
self.tsx_hle_regions += other.tsx_hle_regions;
self.cet_patterns_validated += other.cet_patterns_validated;
self.zero_overhead_fit_lsd += other.zero_overhead_fit_lsd;
self.zero_overhead_fit_uop_cache += other.zero_overhead_fit_uop_cache;
self.zero_overhead_aligned += other.zero_overhead_aligned;
self.zero_overhead_unrolled += other.zero_overhead_unrolled;
self.legal_loops += other.legal_loops;
self.illegal_loops += other.illegal_loops;
}
}
impl X86HardwareLoops {
// ------------------------------------------------------------------
// Construction
// ------------------------------------------------------------------
pub fn new(microarch: HWLoopMicroArch) -> Self {
Self {
microarch,
lowerings: Vec::new(),
tsx_regions: Vec::new(),
cet_patterns: Vec::new(),
counter_patterns: Vec::new(),
zero_overhead_analyses: Vec::new(),
legality_checks: Vec::new(),
loops_optimized: 0,
loops_rejected: 0,
stats: HWLoopStats::default(),
zo_config: ZeroOverheadConfig::default(),
has_tsx_hle: false,
cet_enforced: false,
}
}
pub fn with_tsx(mut self, enabled: bool) -> Self {
self.has_tsx_hle = enabled;
self
}
pub fn with_cet(mut self, enforced: bool) -> Self {
self.cet_enforced = enforced;
self
}
pub fn with_zo_config(mut self, config: ZeroOverheadConfig) -> Self {
self.zo_config = config;
self
}
// ------------------------------------------------------------------
// Intrinsic Lowering
// ------------------------------------------------------------------
/// Lower a `set.loop.iterations` intrinsic into X86.
/// On x86-64, this typically becomes a `mov ecx, N`.
pub fn lower_set_loop_iterations(
&mut self,
counter_reg: &str,
iterations: u64,
bit_width: u32,
) -> HWLoopLowering {
let mut lowering = HWLoopLowering::new(HWLoopIntrinsic::SetLoopIterations);
lowering.counter_reg = Some(counter_reg.into());
let suffix = if bit_width == 64 { "q" } else { "l" };
lowering
.x86_sequence
.push(format!("mov{} {}, {}", suffix, counter_reg, iterations));
lowering.back_edge_cycles = 1;
self.stats.set_loop_iterations_lowered += 1;
self.lowerings.push(lowering.clone());
lowering
}
/// Lower a `loop.decrement.reg` intrinsic into X86.
/// This becomes `dec <reg>` + `jnz <target>` for DEC+JNZ fusion.
pub fn lower_loop_decrement_reg(
&mut self,
counter_reg: &str,
target_label: &str,
bit_width: u32,
) -> HWLoopLowering {
let mut lowering = HWLoopLowering::new(HWLoopIntrinsic::LoopDecrementReg);
lowering.counter_reg = Some(counter_reg.into());
let suffix = if bit_width == 64 { "q" } else { "l" };
if self.microarch.has_dec_jcc_fusion() {
lowering
.x86_sequence
.push(format!("dec{} {}", suffix, counter_reg));
lowering.x86_sequence.push(format!("jnz {}", target_label));
lowering.uses_dec_jnz_fusion = true;
lowering.back_edge_cycles = 0; // fused into a single uop
self.stats.dec_jnz_fusion_applied += 1;
} else {
lowering
.x86_sequence
.push(format!("sub{} {}, 1", suffix, counter_reg));
lowering
.x86_sequence
.push(format!("cmp{} {}, 0", suffix, counter_reg));
lowering.x86_sequence.push(format!("jne {}", target_label));
lowering.has_compare = true;
lowering.back_edge_cycles = 2;
}
self.stats.loop_decrement_reg_lowered += 1;
self.lowerings.push(lowering.clone());
lowering
}
/// Lower a `loop.decrement` intrinsic with an explicit bound.
/// This becomes `sub <reg>, step` + `cmp <reg>, bound` + `jcc target`.
pub fn lower_loop_decrement(
&mut self,
counter_reg: &str,
bound_reg: &str,
step: i64,
pred: LoopCounterPredicate,
target_label: &str,
bit_width: u32,
) -> HWLoopLowering {
let mut lowering = HWLoopLowering::new(HWLoopIntrinsic::LoopDecrement);
lowering.counter_reg = Some(counter_reg.into());
let suffix = if bit_width == 64 { "q" } else { "l" };
if step < 0 {
lowering
.x86_sequence
.push(format!("add{} {}, {}", suffix, counter_reg, -step));
} else {
lowering
.x86_sequence
.push(format!("sub{} {}, {}", suffix, counter_reg, step));
}
lowering
.x86_sequence
.push(format!("cmp{} {}, {}", suffix, counter_reg, bound_reg));
lowering
.x86_sequence
.push(format!("{} {}", jcc_from_pred(pred), target_label));
lowering.has_compare = true;
lowering.back_edge_cycles = 2;
self.stats.loop_decrement_lowered += 1;
self.lowerings.push(lowering.clone());
lowering
}
// ------------------------------------------------------------------
// TSX HLE (Hardware Lock Elision) Loop Support
// ------------------------------------------------------------------
/// Register a TSX HLE loop region.
pub fn register_tsx_region(&mut self, region: TSXHLELoop) {
self.tsx_regions.push(region);
self.stats.tsx_hle_regions += 1;
}
/// Analyze whether a loop body is suitable for TSX HLE.
pub fn analyze_tsx_compatibility(&self, body_instructions: &[String]) -> bool {
// TSX HLE regions must not contain: SYSCALL, CPUID, INT, far JMP/CALL,
// segment register loads, or any operation that always aborts.
let forbidden = ["syscall", "cpuid", "int ", "lss ", "lgs ", "lfs ", "les "];
for inst in body_instructions {
let lower = inst.to_lowercase();
for f in &forbidden {
if lower.contains(f) {
return false;
}
}
}
true
}
/// Generate the XACQUIRE/XRELEASE prefix sequence for an HLE region.
pub fn generate_hle_prefixes(&self, region: &TSXHLELoop) -> Vec<String> {
let mut seq = Vec::new();
if region.uses_xacquire {
seq.push(format!("{}: # XACQUIRE begin", region.start_label));
}
if region.uses_xrelease {
seq.push(format!("{}: # XRELEASE end", region.end_label));
}
seq
}
// ------------------------------------------------------------------
// CET-Compatible Loop Patterns
// ------------------------------------------------------------------
/// Validate that a loop is CET-compatible.
pub fn validate_cet_loop(&mut self, pattern: CETLoopPattern) -> CETLoopPattern {
let mut pat = pattern;
pat.is_cet_compatible = true;
// A CET-compatible loop header must have an ENDBR instruction.
if self.cet_enforced && !pat.has_endbr {
pat.is_cet_compatible = false;
}
// All indirect branch targets must be ENDBR-annotated.
if self.cet_enforced && !pat.indirect_targets.is_empty() {
// In practice, each indirect target must start with ENDBR.
pat.is_cet_compatible = true; // simplified check
}
self.cet_patterns.push(pat.clone());
self.stats.cet_patterns_validated += 1;
pat
}
/// Insert an ENDBR at a loop header for CET compatibility.
pub fn insert_cet_endbr(&self, header_label: &str) -> String {
format!("{}: endbr64", header_label)
}
// ------------------------------------------------------------------
// Loop Counter Optimization
// ------------------------------------------------------------------
/// Detect and record a loop counter pattern.
pub fn detect_loop_counter(
&mut self,
counter_reg: &str,
initial: i64,
step: i64,
pred: LoopCounterPredicate,
is_index: bool,
) -> LoopCounterPattern {
let pattern = LoopCounterPattern {
counter_reg: counter_reg.into(),
initial_value: initial,
step,
predicate: pred,
fusion_compatible: self.microarch.has_dec_jcc_fusion()
&& step == -1
&& (pred == LoopCounterPredicate::NE || pred == LoopCounterPredicate::GT),
is_address_index: is_index,
};
self.counter_patterns.push(pattern.clone());
pattern
}
/// Check whether DEC+JNZ fusion can be applied to this loop.
pub fn can_fuse_dec_jnz(&self, step: i64, pred: LoopCounterPredicate) -> bool {
self.microarch.has_dec_jcc_fusion()
&& step == -1
&& (pred == LoopCounterPredicate::NE || pred == LoopCounterPredicate::GT)
}
/// Optimize a loop for DEC+JNZ fusion: replace SUB+CMP+JNE with DEC+JNZ.
pub fn optimize_for_fusion(
&self,
counter_reg: &str,
target: &str,
bit_width: u32,
) -> Vec<String> {
let suffix = if bit_width == 64 { "q" } else { "l" };
vec![
format!("dec{} {}", suffix, counter_reg),
format!("jnz {}", target),
]
}
// ------------------------------------------------------------------
// Zero-Overhead Loop Optimizations
// ------------------------------------------------------------------
/// Analyze whether a loop can benefit from zero-overhead execution.
pub fn analyze_zero_overhead(
&mut self,
total_uops: u32,
total_instructions: u32,
is_aligned: bool,
) -> ZeroOverheadAnalysis {
let lsd_cap = self.microarch.lsd_capacity_uops();
let op_cap = self.microarch.op_cache_capacity_insts();
let fits_lsd = self.microarch.has_lsd() && total_uops <= lsd_cap;
let fits_uop = self.microarch.has_op_cache_loop() && total_instructions <= op_cap;
let action = if !is_aligned && self.zo_config.align_loop_header {
ZeroOverheadAction::AlignHeader
} else if fits_lsd && self.zo_config.insert_nop_padding {
ZeroOverheadAction::InsertNOPs
} else if !fits_lsd && !fits_uop && self.zo_config.unroll_for_uop_cache {
ZeroOverheadAction::Unroll
} else if !fits_lsd && !fits_uop {
ZeroOverheadAction::TooLarge
} else {
ZeroOverheadAction::None
};
let analysis = ZeroOverheadAnalysis {
total_uops,
fits_in_lsd: fits_lsd,
fits_in_uop_cache: fits_uop,
is_aligned,
recommendation: action,
};
match action {
ZeroOverheadAction::None => {}
ZeroOverheadAction::AlignHeader => self.stats.zero_overhead_aligned += 1,
ZeroOverheadAction::InsertNOPs => {}
ZeroOverheadAction::Unroll => self.stats.zero_overhead_unrolled += 1,
ZeroOverheadAction::TooLarge => {}
}
if fits_lsd {
self.stats.zero_overhead_fit_lsd += 1;
}
if fits_uop {
self.stats.zero_overhead_fit_uop_cache += 1;
}
self.zero_overhead_analyses.push(analysis.clone());
analysis
}
/// Generate alignment padding for a loop header.
pub fn align_loop_header(&self, current_offset: u32) -> Vec<String> {
let align = self.zo_config.header_alignment;
let remainder = current_offset % align;
if remainder == 0 {
return vec![];
}
let pad = align - remainder;
let mut nops = Vec::new();
let mut remaining = pad;
while remaining > 0 {
if remaining >= 9 {
nops.push("nop [rax+rax*1+0] # 9-byte nop".into());
remaining -= 9;
} else if remaining >= 7 {
nops.push("nop [rax+0] # 7-byte nop".into());
remaining -= 7;
} else if remaining >= 6 {
nops.push("nop [rax+rax*1+0] # 6-byte nop".into());
remaining -= 6;
} else if remaining >= 5 {
nops.push("nop [rax+rax*1+0] # 5-byte nop".into());
remaining -= 5;
} else if remaining >= 4 {
nops.push("nop [rax+0] # 4-byte nop".into());
remaining -= 4;
} else if remaining >= 3 {
nops.push("nop [rax] # 3-byte nop".into());
remaining -= 3;
} else if remaining >= 2 {
nops.push("xchg ax,ax # 2-byte nop".into());
remaining -= 2;
} else {
nops.push("nop # 1-byte nop".into());
remaining -= 1;
}
}
nops
}
// ------------------------------------------------------------------
// Hardware Loop Legality Checking
// ------------------------------------------------------------------
/// Check whether a loop satisfies the legality requirements for a
/// hardware loop.
pub fn check_legality(
&mut self,
single_backedge: bool,
known_trip_count: bool,
trip_count: Option<u64>,
contains_calls: bool,
contains_indirect_branches: bool,
is_reducible: bool,
counter_live_out: bool,
) -> HWLoopLegality {
let mut reasons: Vec<&str> = Vec::new();
if !single_backedge {
reasons.push("multiple backedges");
}
if !known_trip_count {
reasons.push("unknown trip count");
}
if contains_calls {
reasons.push("contains calls");
}
if contains_indirect_branches {
reasons.push("contains indirect branches");
}
if !is_reducible {
reasons.push("irreducible loop");
}
let is_legal = reasons.is_empty();
let legality = HWLoopLegality {
is_single_backedge: single_backedge,
has_known_trip_count: known_trip_count,
trip_count,
contains_calls,
contains_indirect_branches,
is_reducible,
counter_live_out,
is_legal,
rejection_reason: if is_legal {
None
} else {
Some(reasons.join("; "))
},
};
if is_legal {
self.stats.legal_loops += 1;
} else {
self.stats.illegal_loops += 1;
}
self.legality_checks.push(legality.clone());
legality
}
/// Quick check: is a loop likely a good candidate for hardware-loop
/// optimization?
pub fn is_hardware_loop_candidate(&self, trip_count: u64, body_instructions: u32) -> bool {
// Reject trivial loops (trip count 0 or 1).
if trip_count <= 1 {
return false;
}
// Reject enormous bodies.
if body_instructions > 500 {
return false;
}
// Small loops with moderate trip counts are the best candidates.
trip_count >= 4 && body_instructions <= 64
}
// ------------------------------------------------------------------
// Pipeline
// ------------------------------------------------------------------
/// Run the full hardware-loop optimization pipeline on a loop.
pub fn optimize_loop(
&mut self,
counter_reg: &str,
initial: i64,
step: i64,
pred: LoopCounterPredicate,
target_label: &str,
bit_width: u32,
total_uops: u32,
total_instructions: u32,
is_aligned: bool,
is_index: bool,
trip_count: Option<u64>,
) -> HWLoopOptimizationResult {
// 1. Detect loop counter pattern
let pattern = self.detect_loop_counter(counter_reg, initial, step, pred, is_index);
// 2. Lower the appropriate intrinsic
let lowering = if step == -1
&& (pred == LoopCounterPredicate::NE || pred == LoopCounterPredicate::GT)
{
self.lower_loop_decrement_reg(counter_reg, target_label, bit_width)
} else {
self.lower_loop_decrement(counter_reg, "bound", step, pred, target_label, bit_width)
};
// 3. Zero-overhead analysis
let zo = self.analyze_zero_overhead(total_uops, total_instructions, is_aligned);
// 4. Legality check
let legality = self.check_legality(
true,
trip_count.is_some(),
trip_count,
false, // assume no calls for this API
false, // assume no indirect branches
true,
false,
);
self.loops_optimized += 1;
HWLoopOptimizationResult {
pattern,
lowering,
zero_overhead: zo,
legality,
is_optimized: true,
}
}
/// Reset all internal state.
pub fn reset(&mut self) {
self.lowerings.clear();
self.tsx_regions.clear();
self.cet_patterns.clear();
self.counter_patterns.clear();
self.zero_overhead_analyses.clear();
self.legality_checks.clear();
self.loops_optimized = 0;
self.loops_rejected = 0;
self.stats = HWLoopStats::default();
}
/// Return a summary string.
pub fn summary(&self) -> String {
format!(
"X86HardwareLoops({:?}): loops_opt={} rejected={} lowered={} \
dec_jnz_fusion={} tsx={} cet={} zo_lsd={} zo_uop={} zo_align={} \
zo_unroll={} legal={} illegal={}",
self.microarch,
self.loops_optimized,
self.loops_rejected,
self.stats.total_lowered(),
self.stats.dec_jnz_fusion_applied,
self.stats.tsx_hle_regions,
self.stats.cet_patterns_validated,
self.stats.zero_overhead_fit_lsd,
self.stats.zero_overhead_fit_uop_cache,
self.stats.zero_overhead_aligned,
self.stats.zero_overhead_unrolled,
self.stats.legal_loops,
self.stats.illegal_loops,
)
}
}
// ---------------------------------------------------------------------------
// Optimization Result
// ---------------------------------------------------------------------------
#[derive(Debug, Clone)]
pub struct HWLoopOptimizationResult {
pub pattern: LoopCounterPattern,
pub lowering: HWLoopLowering,
pub zero_overhead: ZeroOverheadAnalysis,
pub legality: HWLoopLegality,
pub is_optimized: bool,
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
/// Map a loop counter predicate to its X86 JCC mnemonic.
fn jcc_from_pred(pred: LoopCounterPredicate) -> &'static str {
match pred {
LoopCounterPredicate::EQ => "je",
LoopCounterPredicate::NE => "jne",
LoopCounterPredicate::LT => "jl",
LoopCounterPredicate::LE => "jle",
LoopCounterPredicate::GT => "jg",
LoopCounterPredicate::GE => "jge",
LoopCounterPredicate::ULT => "jb",
LoopCounterPredicate::ULE => "jbe",
LoopCounterPredicate::UGT => "ja",
LoopCounterPredicate::UGE => "jae",
}
}
// ---------------------------------------------------------------------------
// Convenience constructors
// ---------------------------------------------------------------------------
pub fn make_x86_hardware_loops_skylake() -> X86HardwareLoops {
X86HardwareLoops::new(HWLoopMicroArch::Skylake)
}
pub fn make_x86_hardware_loops_zen4() -> X86HardwareLoops {
X86HardwareLoops::new(HWLoopMicroArch::Zen4)
}
pub fn make_x86_hardware_loops_generic() -> X86HardwareLoops {
X86HardwareLoops::new(HWLoopMicroArch::Generic)
}
pub fn make_x86_hardware_loops_haswell_tsx() -> X86HardwareLoops {
X86HardwareLoops::new(HWLoopMicroArch::Haswell).with_tsx(true)
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
#[cfg(test)]
mod tests {
use super::*;
fn make_skl() -> X86HardwareLoops {
X86HardwareLoops::new(HWLoopMicroArch::Skylake)
}
fn make_zen4() -> X86HardwareLoops {
X86HardwareLoops::new(HWLoopMicroArch::Zen4)
}
fn make_gen() -> X86HardwareLoops {
X86HardwareLoops::new(HWLoopMicroArch::Generic)
}
// --- Microarchitecture ---
#[test]
fn test_microarch_fusion() {
assert!(HWLoopMicroArch::Skylake.has_dec_jcc_fusion());
assert!(HWLoopMicroArch::Zen4.has_dec_jcc_fusion());
assert!(!HWLoopMicroArch::Generic.has_dec_jcc_fusion());
}
#[test]
fn test_microarch_lsd() {
assert!(HWLoopMicroArch::Skylake.has_lsd());
assert!(!HWLoopMicroArch::Zen4.has_lsd());
assert!(!HWLoopMicroArch::Generic.has_lsd());
}
#[test]
fn test_microarch_op_cache() {
assert!(HWLoopMicroArch::Zen4.has_op_cache_loop());
assert!(!HWLoopMicroArch::Skylake.has_op_cache_loop());
}
#[test]
fn test_microarch_lsd_capacity() {
assert_eq!(HWLoopMicroArch::Skylake.lsd_capacity_uops(), 28);
assert_eq!(HWLoopMicroArch::Generic.lsd_capacity_uops(), 0);
}
#[test]
fn test_microarch_op_cache_capacity() {
assert_eq!(HWLoopMicroArch::Zen4.op_cache_capacity_insts(), 4096);
assert_eq!(HWLoopMicroArch::Skylake.op_cache_capacity_insts(), 0);
}
#[test]
fn test_microarch_display() {
let s = format!("{}", HWLoopMicroArch::Skylake);
assert!(s.contains("Skylake"));
}
// --- Intrinsic Kinds ---
#[test]
fn test_intrinsic_display() {
assert_eq!(
format!("{}", HWLoopIntrinsic::SetLoopIterations),
"set.loop.iterations"
);
assert_eq!(
format!("{}", HWLoopIntrinsic::LoopDecrementReg),
"loop.decrement.reg"
);
assert_eq!(
format!("{}", HWLoopIntrinsic::LoopDecrement),
"loop.decrement"
);
}
// --- Construction ---
#[test]
fn test_new_skylake() {
let hw = make_skl();
assert_eq!(hw.microarch, HWLoopMicroArch::Skylake);
assert_eq!(hw.loops_optimized, 0);
}
#[test]
fn test_new_with_tsx() {
let hw = X86HardwareLoops::new(HWLoopMicroArch::Haswell).with_tsx(true);
assert!(hw.has_tsx_hle);
}
#[test]
fn test_new_with_cet() {
let hw = X86HardwareLoops::new(HWLoopMicroArch::AlderLakeP).with_cet(true);
assert!(hw.cet_enforced);
}
#[test]
fn test_new_with_zo_config() {
let zo = ZeroOverheadConfig {
lsd_max_uops: 20,
..Default::default()
};
let hw = make_skl().with_zo_config(zo);
assert_eq!(hw.zo_config.lsd_max_uops, 20);
}
// --- Lower: set.loop.iterations ---
#[test]
fn test_lower_set_loop_iterations() {
let mut hw = make_skl();
let l = hw.lower_set_loop_iterations("ecx", 100, 32);
assert_eq!(l.kind, HWLoopIntrinsic::SetLoopIterations);
assert_eq!(l.counter_reg.as_deref(), Some("ecx"));
assert!(!l.x86_sequence.is_empty());
assert!(l.x86_sequence[0].contains("mov"));
}
#[test]
fn test_lower_set_loop_iterations_64() {
let mut hw = make_skl();
let l = hw.lower_set_loop_iterations("rcx", 1000, 64);
assert!(l.x86_sequence[0].contains("movq"));
}
// --- Lower: loop.decrement.reg ---
#[test]
fn test_lower_loop_decrement_reg_fusion() {
let mut hw = make_skl();
let l = hw.lower_loop_decrement_reg("ecx", ".L_loop", 32);
assert!(l.uses_dec_jnz_fusion);
assert_eq!(l.back_edge_cycles, 0);
assert_eq!(hw.stats.dec_jnz_fusion_applied, 1);
}
#[test]
fn test_lower_loop_decrement_reg_no_fusion() {
let mut hw = make_gen();
let l = hw.lower_loop_decrement_reg("ecx", ".L_loop", 32);
assert!(!l.uses_dec_jnz_fusion);
assert!(l.has_compare);
}
#[test]
fn test_lower_loop_decrement_reg_64() {
let mut hw = make_skl();
let l = hw.lower_loop_decrement_reg("rcx", ".L_loop", 64);
assert!(l.uses_dec_jnz_fusion);
}
// --- Lower: loop.decrement ---
#[test]
fn test_lower_loop_decrement() {
let mut hw = make_skl();
let l = hw.lower_loop_decrement("ecx", "edx", -1, LoopCounterPredicate::NE, ".L_loop", 32);
assert!(l.has_compare);
assert!(l.x86_sequence.iter().any(|s| s.contains("add")));
assert!(l.x86_sequence.iter().any(|s| s.contains("jne")));
}
#[test]
fn test_lower_loop_decrement_positive_step() {
let mut hw = make_skl();
let l = hw.lower_loop_decrement("ecx", "edx", 2, LoopCounterPredicate::LT, ".L_loop", 32);
assert!(l.x86_sequence.iter().any(|s| s.contains("sub")));
}
// --- TSX HLE ---
#[test]
fn test_register_tsx_region() {
let mut hw = make_skl().with_tsx(true);
let region = TSXHLELoop::new("tsx_start", "tsx_end");
hw.register_tsx_region(region);
assert_eq!(hw.tsx_regions.len(), 1);
assert_eq!(hw.stats.tsx_hle_regions, 1);
}
#[test]
fn test_analyze_tsx_compatibility_good() {
let hw = make_skl();
let body = vec!["mov eax, ebx".into(), "add ecx, 1".into()];
assert!(hw.analyze_tsx_compatibility(&body));
}
#[test]
fn test_analyze_tsx_compatibility_bad_syscall() {
let hw = make_skl();
let body = vec!["syscall".into()];
assert!(!hw.analyze_tsx_compatibility(&body));
}
#[test]
fn test_analyze_tsx_compatibility_bad_cpuid() {
let hw = make_skl();
let body = vec!["cpuid".into()];
assert!(!hw.analyze_tsx_compatibility(&body));
}
#[test]
fn test_generate_hle_prefixes() {
let hw = make_skl();
let mut region = TSXHLELoop::new("s", "e");
region.uses_xacquire = true;
region.uses_xrelease = true;
let seq = hw.generate_hle_prefixes(®ion);
assert_eq!(seq.len(), 2);
assert!(seq[0].contains("XACQUIRE"));
assert!(seq[1].contains("XRELEASE"));
}
// --- CET ---
#[test]
fn test_validate_cet_loop() {
let mut hw = make_skl().with_cet(true);
let mut pat = CETLoopPattern::new(".L_header");
pat.has_endbr = true;
let result = hw.validate_cet_loop(pat);
assert!(result.is_cet_compatible);
assert_eq!(hw.stats.cet_patterns_validated, 1);
}
#[test]
fn test_validate_cet_loop_no_endbr() {
let mut hw = make_skl().with_cet(true);
let pat = CETLoopPattern::new(".L_header");
let result = hw.validate_cet_loop(pat);
assert!(!result.is_cet_compatible);
}
#[test]
fn test_insert_cet_endbr() {
let hw = make_skl();
let inst = hw.insert_cet_endbr(".L_loop");
assert!(inst.contains("endbr64"));
}
// --- Loop Counter Detection ---
#[test]
fn test_detect_loop_counter() {
let mut hw = make_skl();
let pat = hw.detect_loop_counter("ecx", 100, -1, LoopCounterPredicate::NE, false);
assert_eq!(pat.counter_reg, "ecx");
assert_eq!(pat.step, -1);
assert!(pat.fusion_compatible);
}
#[test]
fn test_detect_loop_counter_no_fusion() {
let mut hw = make_gen();
let pat = hw.detect_loop_counter("ecx", 100, -2, LoopCounterPredicate::NE, false);
assert!(!pat.fusion_compatible);
}
#[test]
fn test_can_fuse_dec_jnz() {
assert!(make_skl().can_fuse_dec_jnz(-1, LoopCounterPredicate::NE));
assert!(!make_skl().can_fuse_dec_jnz(-2, LoopCounterPredicate::NE));
assert!(!make_gen().can_fuse_dec_jnz(-1, LoopCounterPredicate::NE));
}
#[test]
fn test_optimize_for_fusion() {
let hw = make_skl();
let seq = hw.optimize_for_fusion("ecx", ".L_loop", 32);
assert_eq!(seq.len(), 2);
assert!(seq[0].contains("dec"));
assert!(seq[1].contains("jnz"));
}
// --- Zero-Overhead ---
#[test]
fn test_analyze_zero_overhead_fits_lsd() {
let mut hw = make_skl();
let zo = hw.analyze_zero_overhead(20, 15, true);
assert!(zo.fits_in_lsd);
assert_eq!(hw.stats.zero_overhead_fit_lsd, 1);
}
#[test]
fn test_analyze_zero_overhead_too_large() {
let hw = make_gen();
let zo =
X86HardwareLoops::new(HWLoopMicroArch::Skylake).analyze_zero_overhead(40, 30, true);
assert_eq!(zo.recommendation, ZeroOverheadAction::TooLarge);
}
#[test]
fn test_analyze_zero_overhead_needs_alignment() {
let mut hw = make_skl();
let zo = hw.analyze_zero_overhead(20, 15, false);
assert_eq!(zo.recommendation, ZeroOverheadAction::AlignHeader);
}
#[test]
fn test_analyze_zero_overhead_zen4_uop_cache() {
let mut hw = make_zen4();
let zo = hw.analyze_zero_overhead(10, 8, true);
assert!(zo.fits_in_uop_cache);
}
#[test]
fn test_zero_overhead_action_display() {
let s = format!("{}", ZeroOverheadAction::None);
assert_eq!(s, "None");
}
// --- Loop Header Alignment ---
#[test]
fn test_align_loop_header_exact() {
let hw = make_skl();
let nops = hw.align_loop_header(32);
assert!(nops.is_empty());
}
#[test]
fn test_align_loop_header_misaligned() {
let hw = make_skl();
let nops = hw.align_loop_header(33);
assert!(!nops.is_empty());
}
#[test]
fn test_align_loop_header_zero() {
let hw = make_skl();
let nops = hw.align_loop_header(0);
assert!(nops.is_empty());
}
// --- Legality Checking ---
#[test]
fn test_check_legality_all_good() {
let mut hw = make_skl();
let leg = hw.check_legality(true, true, Some(100), false, false, true, false);
assert!(leg.is_legal);
assert!(leg.rejection_reason.is_none());
assert_eq!(hw.stats.legal_loops, 1);
}
#[test]
fn test_check_legality_multiple_backedge() {
let mut hw = make_skl();
let leg = hw.check_legality(false, true, Some(100), false, false, true, false);
assert!(!leg.is_legal);
assert!(leg.rejection_reason.unwrap().contains("multiple"));
}
#[test]
fn test_check_legality_contains_calls() {
let mut hw = make_skl();
let leg = hw.check_legality(true, true, Some(100), true, false, true, false);
assert!(!leg.is_legal);
}
#[test]
fn test_check_legality_irreducible() {
let mut hw = make_skl();
let leg = hw.check_legality(true, true, Some(100), false, false, false, false);
assert!(!leg.is_legal);
}
#[test]
fn test_check_legality_unknown_trip_count() {
let mut hw = make_skl();
let leg = hw.check_legality(true, false, None, false, false, true, false);
assert!(!leg.is_legal);
}
// --- Candidate Check ---
#[test]
fn test_is_hardware_loop_candidate() {
let hw = make_skl();
assert!(hw.is_hardware_loop_candidate(16, 10));
}
#[test]
fn test_is_not_candidate_trivial() {
let hw = make_skl();
assert!(!hw.is_hardware_loop_candidate(1, 10));
}
#[test]
fn test_is_not_candidate_huge_body() {
let hw = make_skl();
assert!(!hw.is_hardware_loop_candidate(16, 600));
}
// --- Full Pipeline ---
#[test]
fn test_optimize_loop() {
let mut hw = make_skl();
let result = hw.optimize_loop(
"ecx",
100,
-1,
LoopCounterPredicate::NE,
".L_loop",
32,
20,
15,
true,
false,
Some(100),
);
assert!(result.is_optimized);
assert!(result.lowering.uses_dec_jnz_fusion);
assert_eq!(hw.loops_optimized, 1);
}
#[test]
fn test_optimize_loop_zen4() {
let mut hw = make_zen4();
let result = hw.optimize_loop(
"rcx",
200,
-1,
LoopCounterPredicate::NE,
".L_loop",
64,
10,
8,
true,
false,
Some(200),
);
assert!(result.is_optimized);
assert!(result.zero_overhead.fits_in_uop_cache);
}
#[test]
fn test_optimize_loop_with_tsx() {
let mut hw = make_x86_hardware_loops_haswell_tsx();
hw.register_tsx_region(TSXHLELoop::new("s", "e"));
let result = hw.optimize_loop(
"ecx",
50,
-1,
LoopCounterPredicate::NE,
".L_loop",
32,
15,
10,
true,
false,
Some(50),
);
assert!(result.is_optimized);
assert_eq!(hw.stats.tsx_hle_regions, 1);
}
// --- Summary ---
#[test]
fn test_summary() {
let hw = make_skl();
let s = hw.summary();
assert!(s.contains("X86HardwareLoops"));
assert!(s.contains("Skylake"));
}
// --- Reset ---
#[test]
fn test_reset() {
let mut hw = make_skl();
hw.lower_set_loop_iterations("ecx", 10, 32);
hw.lower_loop_decrement_reg("ecx", ".L", 32);
assert!(hw.stats.total_lowered() > 0);
hw.reset();
assert_eq!(hw.stats.total_lowered(), 0);
assert!(hw.lowerings.is_empty());
assert_eq!(hw.loops_optimized, 0);
}
// --- Stats ---
#[test]
fn test_stats_total_lowered() {
let mut stats = HWLoopStats::default();
stats.set_loop_iterations_lowered = 3;
stats.loop_decrement_reg_lowered = 5;
assert_eq!(stats.total_lowered(), 8);
}
#[test]
fn test_stats_merge() {
let mut a = HWLoopStats::default();
a.dec_jnz_fusion_applied = 4;
let mut b = HWLoopStats::default();
b.dec_jnz_fusion_applied = 3;
b.legal_loops = 10;
a.merge(&b);
assert_eq!(a.dec_jnz_fusion_applied, 7);
assert_eq!(a.legal_loops, 10);
}
// --- Convenience Constructors ---
#[test]
fn test_make_functions() {
let skl = make_x86_hardware_loops_skylake();
assert_eq!(skl.microarch, HWLoopMicroArch::Skylake);
let zen4 = make_x86_hardware_loops_zen4();
assert_eq!(zen4.microarch, HWLoopMicroArch::Zen4);
let r#gen = make_x86_hardware_loops_generic();
assert_eq!(r#gen.microarch, HWLoopMicroArch::Generic);
let hsw = make_x86_hardware_loops_haswell_tsx();
assert!(hsw.has_tsx_hle);
}
// --- JCC Mapping ---
#[test]
fn test_jcc_from_pred() {
assert_eq!(jcc_from_pred(LoopCounterPredicate::EQ), "je");
assert_eq!(jcc_from_pred(LoopCounterPredicate::NE), "jne");
assert_eq!(jcc_from_pred(LoopCounterPredicate::GT), "jg");
assert_eq!(jcc_from_pred(LoopCounterPredicate::ULT), "jb");
}
// --- HWLoopLowering ---
#[test]
fn test_lowering_new() {
let l = HWLoopLowering::new(HWLoopIntrinsic::LoopDecrementReg);
assert_eq!(l.kind, HWLoopIntrinsic::LoopDecrementReg);
assert!(l.x86_sequence.is_empty());
}
// --- HWLoopLegality ---
#[test]
fn test_legality_legal_constructor() {
let leg = HWLoopLegality::legal();
assert!(leg.is_legal);
}
#[test]
fn test_legality_reject() {
let leg = HWLoopLegality::reject("test reason");
assert!(!leg.is_legal);
assert_eq!(leg.rejection_reason.as_deref(), Some("test reason"));
}
// --- ZeroOverheadConfig ---
#[test]
fn test_zo_config_default() {
let zo = ZeroOverheadConfig::default();
assert!(zo.enable_lsd_optimization);
assert_eq!(zo.lsd_max_uops, 28);
assert!(zo.align_loop_header);
}
// --- LoopCounterPredicate ---
#[test]
fn test_predicate_display() {
assert_eq!(format!("{}", LoopCounterPredicate::NE), "NE");
assert_eq!(format!("{}", LoopCounterPredicate::ULE), "ULE");
}
// --- Edge Cases ---
#[test]
fn test_align_header_large_offset() {
let hw = make_skl();
let nops = hw.align_loop_header(1059);
let total_bytes: u32 = nops
.iter()
.map(|n| {
if n.contains("9-byte") {
9
} else if n.contains("7-byte") {
7
} else if n.contains("6-byte") {
6
} else if n.contains("5-byte") {
5
} else if n.contains("4-byte") {
4
} else if n.contains("3-byte") {
3
} else if n.contains("2-byte") {
2
} else {
1
}
})
.sum();
// 1059 % 32 = 3, so we need 29 bytes of padding
assert_eq!(total_bytes, 29);
}
#[test]
fn test_multiple_lowerings_accumulated() {
let mut hw = make_skl();
hw.lower_set_loop_iterations("ecx", 10, 32);
hw.lower_loop_decrement_reg("ecx", ".L", 32);
hw.lower_loop_decrement("ecx", "edx", -1, LoopCounterPredicate::NE, ".L", 32);
assert_eq!(hw.lowerings.len(), 3);
assert_eq!(hw.stats.total_lowered(), 3);
}
#[test]
fn test_all_microarchs_have_sane_config() {
let archs = [
HWLoopMicroArch::Generic,
HWLoopMicroArch::SandyBridge,
HWLoopMicroArch::Haswell,
HWLoopMicroArch::Skylake,
HWLoopMicroArch::IceLake,
HWLoopMicroArch::AlderLakeP,
HWLoopMicroArch::SapphireRapids,
HWLoopMicroArch::Zen3,
HWLoopMicroArch::Zen4,
HWLoopMicroArch::Zen5,
];
for arch in &archs {
let hw = X86HardwareLoops::new(*arch);
let leg = hw.check_legality(true, true, Some(10), false, false, true, false);
assert!(leg.is_legal);
}
}
#[test]
fn test_zo_action_all_variants() {
let actions = [
ZeroOverheadAction::None,
ZeroOverheadAction::AlignHeader,
ZeroOverheadAction::InsertNOPs,
ZeroOverheadAction::Unroll,
ZeroOverheadAction::TooLarge,
];
for action in &actions {
let s = format!("{}", action);
assert!(!s.is_empty());
}
}
#[test]
fn test_tsx_region_defaults() {
let r = TSXHLELoop::new("a", "b");
assert!(!r.uses_xacquire);
assert!(!r.uses_xrelease);
assert!(!r.has_xabort);
assert_eq!(r.max_retries, 3);
}
#[test]
fn test_cet_pattern_defaults() {
let c = CETLoopPattern::new("h");
assert!(!c.has_endbr);
assert!(c.is_cet_compatible);
assert!(c.indirect_targets.is_empty());
}
#[test]
fn test_loop_counter_pattern_clone() {
let p = LoopCounterPattern {
counter_reg: "ecx".into(),
initial_value: 10,
step: -1,
predicate: LoopCounterPredicate::NE,
fusion_compatible: true,
is_address_index: false,
};
let p2 = p.clone();
assert_eq!(p.counter_reg, p2.counter_reg);
assert_eq!(p.fusion_compatible, p2.fusion_compatible);
}
#[test]
fn test_full_pipeline_multiple_loops() {
let mut hw = make_skl();
for i in 0..5 {
hw.optimize_loop(
"ecx",
100,
-1,
LoopCounterPredicate::NE,
&format!(".L{}", i),
32,
20,
15,
true,
false,
Some(100),
);
}
assert_eq!(hw.loops_optimized, 5);
assert_eq!(hw.stats.dec_jnz_fusion_applied, 5);
assert_eq!(hw.stats.legal_loops, 5);
}
#[test]
fn test_set_loop_iterations_stats_count() {
let mut hw = make_skl();
hw.lower_set_loop_iterations("ecx", 42, 32);
assert_eq!(hw.stats.set_loop_iterations_lowered, 1);
assert_eq!(hw.stats.total_lowered(), 1);
}
#[test]
fn test_loop_decrement_reg_stats_count() {
let mut hw = make_skl();
hw.lower_loop_decrement_reg("ecx", ".L", 32);
assert_eq!(hw.stats.loop_decrement_reg_lowered, 1);
}
#[test]
fn test_illegal_loop_stats() {
let mut hw = make_skl();
hw.check_legality(false, true, Some(10), false, false, true, false);
assert_eq!(hw.stats.illegal_loops, 1);
assert_eq!(hw.stats.legal_loops, 0);
}
#[test]
fn extra_test_counter_with_index() { let mut hw = make_skl(); let pat = hw.detect_loop_counter("ecx", 100, -1, LoopCounterPredicate::NE, true); assert!(pat.is_address_index); }
#[test]
fn extra_test_counter_positive_step() { let mut hw = make_skl(); let pat = hw.detect_loop_counter("ecx", 0, 1, LoopCounterPredicate::LT, false); assert_eq!(pat.step, 1); }
#[test]
fn extra_test_counter_gt_fusion() { let mut hw = make_skl(); let pat = hw.detect_loop_counter("ecx", 100, -1, LoopCounterPredicate::GT, false); assert!(pat.fusion_compatible); }
#[test]
fn extra_test_fusion_not_for_ge() { let hw = make_skl(); assert!(!hw.can_fuse_dec_jnz(-1, LoopCounterPredicate::GE)); }
#[test]
fn extra_test_fusion_not_for_le() { let hw = make_skl(); assert!(!hw.can_fuse_dec_jnz(-1, LoopCounterPredicate::LE)); }
#[test]
fn extra_test_tsx_abort_handler() { let mut r = TSXHLELoop::new("s", "e"); r.abort_handler = Some("abort_handler".into()); r.has_xabort = true; assert!(r.has_xabort); }
#[test]
fn extra_test_tsx_max_retries() { let mut r = TSXHLELoop::new("s", "e"); r.max_retries = 10; assert_eq!(r.max_retries, 10); }
#[test]
fn extra_test_tsx_compatible_arithmetic() { let hw = make_skl(); let body = vec!["add eax, ebx".into(), "imul ecx, edx".into()]; assert!(hw.analyze_tsx_compatibility(&body)); }
#[test]
fn extra_test_tsx_incompatible_lss() { let hw = make_skl(); let body = vec!["lss eax, [ebx]".into()]; assert!(!hw.analyze_tsx_compatibility(&body)); }
#[test]
fn extra_test_tsx_incompatible_les() { let hw = make_skl(); let body = vec!["les eax, [ebx]".into()]; assert!(!hw.analyze_tsx_compatibility(&body)); }
#[test]
fn extra_test_tsx_incompatible_int() { let hw = make_skl(); let body = vec!["int 0x80".into()]; assert!(!hw.analyze_tsx_compatibility(&body)); }
#[test]
fn extra_test_tsx_xacquire_prefix() { let hw = make_skl(); let mut r = TSXHLELoop::new("s", "e"); r.uses_xacquire = true; let seq = hw.generate_hle_prefixes(&r); assert!(seq[0].contains("XACQUIRE")); }
#[test]
fn extra_test_cet_indirect_targets() { let mut pat = CETLoopPattern::new(".L_hdr"); pat.indirect_targets.push(".L_indirect_1".into()); pat.indirect_targets.push(".L_indirect_2".into()); assert_eq!(pat.indirect_targets.len(), 2); }
#[test]
fn extra_test_cet_shadow_stack() { let mut pat = CETLoopPattern::new(".L_hdr"); pat.shadow_stack_enforced = true; assert!(pat.shadow_stack_enforced); }
#[test]
fn extra_test_cet_no_cet_ok() { let mut hw = make_skl(); let pat = CETLoopPattern::new(".L_hdr"); let result = hw.validate_cet_loop(pat); assert!(result.is_cet_compatible); }
#[test]
fn extra_test_cet_enforced_no_endbr() { let mut hw = make_skl().with_cet(true); let pat = CETLoopPattern::new(".L_hdr"); let result = hw.validate_cet_loop(pat); assert!(!result.is_cet_compatible); }
#[test]
fn extra_test_cet_endbr_label() { let hw = make_skl(); let inst = hw.insert_cet_endbr("my_loop"); assert!(inst.contains("my_loop")); assert!(inst.contains("endbr64")); }
#[test]
fn extra_test_legality_all_reasons() { let mut hw = make_skl(); let leg = hw.check_legality(false, false, None, true, true, false, true); assert!(!leg.is_legal); let reason = leg.rejection_reason.unwrap(); assert!(reason.contains("multiple")); }
#[test]
fn extra_test_legality_live_out() { let mut hw = make_skl(); let leg = hw.check_legality(true, true, Some(100), false, false, true, true); assert!(leg.is_legal); assert!(leg.counter_live_out); }
#[test]
fn extra_test_legality_indirect_branch() { let mut hw = make_skl(); let leg = hw.check_legality(true, true, Some(100), false, true, true, false); assert!(!leg.is_legal); }
#[test]
fn extra_test_zo_config_custom() { let zo = ZeroOverheadConfig { enable_lsd_optimization: false, lsd_max_uops: 40, align_loop_header: false, header_alignment: 16, insert_nop_padding: false, unroll_for_uop_cache: false }; assert!(!zo.enable_lsd_optimization); assert_eq!(zo.lsd_max_uops, 40); }
#[test]
fn extra_test_zo_lsd_exceeded() { let mut hw = X86HardwareLoops::new(HWLoopMicroArch::Skylake); let zo = hw.analyze_zero_overhead(30, 20, true); assert!(!zo.fits_in_lsd); }
#[test]
fn extra_test_zo_exact_lsd() { let mut hw = X86HardwareLoops::new(HWLoopMicroArch::Skylake); let zo = hw.analyze_zero_overhead(28, 20, true); assert!(zo.fits_in_lsd); }
#[test]
fn extra_test_zo_no_lsd_zen() { let mut hw = make_zen4(); let zo = hw.analyze_zero_overhead(20, 15, true); assert!(!zo.fits_in_lsd); assert!(zo.fits_in_uop_cache); }
#[test]
fn extra_test_zo_misaligned() { let mut hw = make_zen4(); let zo = hw.analyze_zero_overhead(10, 8, false); assert_eq!(zo.recommendation, ZeroOverheadAction::AlignHeader); }
#[test]
fn extra_test_align_exact() { let hw = make_skl(); assert!(hw.align_loop_header(64).is_empty()); }
#[test]
fn extra_test_align_need_pad() { let hw = make_skl(); assert!(!hw.align_loop_header(1).is_empty()); }
#[test]
fn extra_test_align_various() { let hw = make_skl(); for off in [0u32, 8, 16, 24, 32, 64] { assert!(hw.align_loop_header(off).is_empty()); } for off in [1u32, 7, 15, 23, 31, 63] { assert!(!hw.align_loop_header(off).is_empty()); } }
#[test]
fn extra_test_candidate_edges() { let hw = make_skl(); assert!(!hw.is_hardware_loop_candidate(0, 10)); assert!(!hw.is_hardware_loop_candidate(1, 10)); assert!(hw.is_hardware_loop_candidate(2, 10)); }
#[test]
fn extra_test_candidate_body_edge() { let hw = make_skl(); assert!(hw.is_hardware_loop_candidate(16, 64)); assert!(!hw.is_hardware_loop_candidate(16, 65)); }
#[test]
fn extra_test_candidate_body_huge() { let hw = make_skl(); assert!(!hw.is_hardware_loop_candidate(16, 501)); }
#[test]
fn extra_test_min_trip() { let hw = make_skl(); assert!(hw.is_hardware_loop_candidate(4, 10)); assert!(!hw.is_hardware_loop_candidate(3, 10)); }
#[test]
fn extra_test_mov_contains_reg() { let mut hw = make_skl(); let l = hw.lower_set_loop_iterations("r8d", 255, 32); assert!(l.x86_sequence[0].contains("r8d")); }
#[test]
fn extra_test_decreg_seq_len_fusion() { let mut hw = make_skl(); assert_eq!(hw.lower_loop_decrement_reg("ecx", ".L", 32).x86_sequence.len(), 2); }
#[test]
fn extra_test_decreg_seq_len_no_fusion() { let mut hw = make_gen(); assert_eq!(hw.lower_loop_decrement_reg("ecx", ".L", 32).x86_sequence.len(), 3); }
#[test]
fn extra_test_decrement_has_cmp() { let mut hw = make_skl(); let l = hw.lower_loop_decrement("ecx", "edx", -1, LoopCounterPredicate::NE, ".L", 32); assert!(l.x86_sequence.iter().any(|s| s.contains("cmp"))); }
#[test]
fn extra_test_jcc_all() { assert_eq!(jcc_from_pred(LoopCounterPredicate::EQ), "je"); assert_eq!(jcc_from_pred(LoopCounterPredicate::NE), "jne"); assert_eq!(jcc_from_pred(LoopCounterPredicate::ULT), "jb"); assert_eq!(jcc_from_pred(LoopCounterPredicate::UGT), "ja"); }
#[test]
fn extra_test_fusion_matrix() { assert!(HWLoopMicroArch::Skylake.has_dec_jcc_fusion()); assert!(HWLoopMicroArch::Zen4.has_dec_jcc_fusion()); assert!(!HWLoopMicroArch::Generic.has_dec_jcc_fusion()); }
#[test]
fn extra_test_lsd_matrix() { assert!(HWLoopMicroArch::Skylake.has_lsd()); assert!(!HWLoopMicroArch::Zen4.has_lsd()); assert!(!HWLoopMicroArch::AlderLakeP.has_lsd()); }
#[test]
fn extra_test_opcache_matrix() { assert!(HWLoopMicroArch::Zen4.has_op_cache_loop()); assert!(!HWLoopMicroArch::Skylake.has_op_cache_loop()); }
#[test]
fn extra_test_gr_fusion() { assert!(HWLoopMicroArch::GraniteRapids.has_dec_jcc_fusion()); }
#[test]
fn extra_test_emr_fusion() { assert!(HWLoopMicroArch::EmeraldRapids.has_dec_jcc_fusion()); }
#[test]
fn extra_test_rpl_fusion() { assert!(HWLoopMicroArch::RaptorLakeP.has_dec_jcc_fusion()); }
#[test]
fn extra_test_adl_e_nofusion() { assert!(!HWLoopMicroArch::AlderLakeE.has_dec_jcc_fusion()); }
#[test]
fn extra_test_adl_e_nolsd() { assert!(!HWLoopMicroArch::AlderLakeE.has_lsd()); }
#[test]
fn extra_test_adl_e_noopcache() { assert!(!HWLoopMicroArch::AlderLakeE.has_op_cache_loop()); }
#[test]
fn extra_test_stress_100() { let mut hw = make_skl(); for i in 0..100 { hw.optimize_loop("ecx", 100, -1, LoopCounterPredicate::NE, &format!(".L{}", i), 32, 20, 15, true, false, Some(100)); } assert_eq!(hw.loops_optimized, 100); }
#[test]
fn extra_test_stress_mixed() { let mut hw = make_skl(); for i in 0..50 { if i % 2 == 0 { hw.optimize_loop("ecx", 100, -1, LoopCounterPredicate::NE, &format!(".L{}", i), 32, 20, 15, true, false, Some(100)); } else { hw.optimize_loop("ecx", 100, -2, LoopCounterPredicate::GT, &format!(".L{}", i), 32, 20, 15, true, false, Some(100)); } } assert_eq!(hw.loops_optimized, 50); }
#[test]
fn extra_test_result_clone() { let mut hw = make_skl(); let r1 = hw.optimize_loop("ecx", 100, -1, LoopCounterPredicate::NE, ".L", 32, 20, 15, true, false, Some(100)); let r2 = r1.clone(); assert_eq!(r1.is_optimized, r2.is_optimized); }
#[test]
fn extra_test_mov_suffix() { let mut hw = make_skl(); assert!(hw.lower_set_loop_iterations("eax", 100, 32).x86_sequence[0].contains("movl")); assert!(hw.lower_set_loop_iterations("rax", 100, 64).x86_sequence[0].contains("movq")); }
#[test]
fn extra_test_dec_suffix() { let mut hw = make_skl(); assert!(hw.lower_loop_decrement_reg("ecx", ".L", 32).x86_sequence[0].contains("decl")); assert!(hw.lower_loop_decrement_reg("rcx", ".L", 64).x86_sequence[0].contains("decq")); }
#[test]
fn extra_test_sub_suffix() { let mut hw = make_gen(); assert!(hw.lower_loop_decrement_reg("rcx", ".L", 64).x86_sequence[0].contains("subq")); }
#[test]
fn extra_test_rejection_counting() { let mut hw = make_skl(); hw.check_legality(false, true, Some(10), false, false, true, false); hw.check_legality(true, false, None, false, false, true, false); assert_eq!(hw.stats.illegal_loops, 2); hw.check_legality(true, true, Some(10), false, false, true, false); assert_eq!(hw.stats.legal_loops, 1); }
#[test]
fn extra_test_all_arch_display() { let archs = [HWLoopMicroArch::Generic, HWLoopMicroArch::Skylake, HWLoopMicroArch::Zen4, HWLoopMicroArch::AlderLakeP]; for a in &archs { let s = format!("{}", a); assert!(!s.is_empty()); } }
#[test]
fn extra_test_pred_display() { let preds = [LoopCounterPredicate::EQ, LoopCounterPredicate::NE, LoopCounterPredicate::GT]; for p in &preds { let s = format!("{}", p); assert!(!s.is_empty()); } }
#[test]
fn extra_test_intrinsic_display() { let ks = [HWLoopIntrinsic::SetLoopIterations, HWLoopIntrinsic::LoopDecrementReg, HWLoopIntrinsic::LoopDecrement]; for k in &ks { let s = format!("{}", k); assert!(!s.is_empty()); } }
#[test]
fn extra_test_zo_action_display() { let as_ = [ZeroOverheadAction::None, ZeroOverheadAction::AlignHeader, ZeroOverheadAction::TooLarge]; for a in &as_ { let s = format!("{}", a); assert!(!s.is_empty()); } }
#[test]
fn extra_test_decrement_step_zero() { let mut hw = make_skl(); let l = hw.lower_loop_decrement("ecx", "edx", 0, LoopCounterPredicate::EQ, ".L", 32); assert!(!l.x86_sequence.is_empty()); }
#[test]
fn extra_test_decrement_large_step() { let mut hw = make_skl(); let l = hw.lower_loop_decrement("ecx", "edx", -1024, LoopCounterPredicate::NE, ".L", 32); assert!(l.x86_sequence[0].contains("1024")); }
#[test]
fn extra_test_legality_known_none() { let mut hw = make_skl(); let leg = hw.check_legality(true, true, None, false, false, true, false); assert!(leg.is_legal); assert!(leg.trip_count.is_none()); }
#[test]
fn extra_test_legality_unknown_val() { let mut hw = make_skl(); let leg = hw.check_legality(true, false, Some(42), false, false, true, false); assert!(!leg.is_legal); assert_eq!(leg.trip_count, Some(42)); }
#[test]
fn extra_test_zo_align_stats() { let mut hw = make_skl(); hw.analyze_zero_overhead(20, 15, false); assert_eq!(hw.stats.zero_overhead_aligned, 1); }
#[test]
fn extra_test_zo_unroll_stats() { let mut hw = make_gen(); hw.analyze_zero_overhead(40, 30, true); assert_eq!(hw.stats.zero_overhead_unrolled, 1); }
#[test]
fn extra_test_zo_lsd_stats() { let mut hw = make_skl(); hw.analyze_zero_overhead(20, 15, true); assert_eq!(hw.stats.zero_overhead_fit_lsd, 1); }
#[test]
fn extra_test_zo_uop_stats() { let mut hw = make_zen4(); hw.analyze_zero_overhead(10, 8, true); assert_eq!(hw.stats.zero_overhead_fit_uop_cache, 1); }
#[test]
fn extra_test_config_lsd_off() { let zo = ZeroOverheadConfig { enable_lsd_optimization: false}; let hw = make_skl().with_zo_config(zo); assert!(!hw.zo_config.enable_lsd_optimization); }
#[test]
fn extra_test_config_nopad() { let zo = ZeroOverheadConfig { insert_nop_padding: false}; let hw = make_skl().with_zo_config(zo); assert!(!hw.zo_config.insert_nop_padding); }
#[test]
fn extra_test_config_nounroll() { let zo = ZeroOverheadConfig { unroll_for_uop_cache: false}; let hw = make_skl().with_zo_config(zo); assert!(!hw.zo_config.unroll_for_uop_cache); }
#[test]
fn extra_test_stats_merge() { let mut a = HWLoopStats::default(); a.dec_jnz_fusion_applied = 10; a.legal_loops = 5; let mut b = HWLoopStats::default(); b.dec_jnz_fusion_applied = 3; b.illegal_loops = 2; a.merge(&b); assert_eq!(a.dec_jnz_fusion_applied, 13); assert_eq!(a.legal_loops, 5); assert_eq!(a.illegal_loops, 2); }
#[test]
fn extra_test_total_lowered_all() { let mut stats = HWLoopStats::default(); stats.set_loop_iterations_lowered = 1; stats.loop_decrement_reg_lowered = 2; stats.loop_decrement_lowered = 3; assert_eq!(stats.total_lowered(), 6); }
#[test]
fn extra_test_pipeline_fusion_archs() { let archs = [HWLoopMicroArch::SandyBridge, HWLoopMicroArch::Haswell, HWLoopMicroArch::Skylake, HWLoopMicroArch::Zen3, HWLoopMicroArch::Zen4]; for a in &archs { let mut hw = X86HardwareLoops::new(*a); let r = hw.optimize_loop("ecx", 100, -1, LoopCounterPredicate::NE, ".L", 32, 20, 15, true, false, Some(100)); assert!(r.is_optimized); assert!(r.lowering.uses_dec_jnz_fusion); } }
#[test]
fn extra_test_pipeline_nofusion() { let mut hw = make_gen(); let r = hw.optimize_loop("ecx", 100, -1, LoopCounterPredicate::NE, ".L", 32, 20, 15, true, false, Some(100)); assert!(r.is_optimized); assert!(!r.lowering.uses_dec_jnz_fusion); }
#[test]
fn extra_test_summary_zen() { let hw = make_zen4(); let s = hw.summary(); assert!(s.contains("Zen4")); }
#[test]
fn extra_test_lowering_debug() { let l = HWLoopLowering::new(HWLoopIntrinsic::LoopDecrementReg); let dbg = format!("{:?}", l); assert!(dbg.contains("HWLoopLowering")); }
#[test]
fn extra_test_result_all() { let mut hw = make_skl(); let r = hw.optimize_loop("ecx", 100, -1, LoopCounterPredicate::NE, ".L", 32, 20, 15, true, false, Some(100)); assert!(r.is_optimized); assert!(r.legality.is_legal); assert!(r.zero_overhead.fits_in_lsd); assert!(r.lowering.uses_dec_jnz_fusion); }
#[test]
fn extra_test_result_too_large() { let mut hw = make_skl(); let r = hw.optimize_loop("ecx", 100, -1, LoopCounterPredicate::NE, ".L", 32, 50, 40, true, false, Some(100)); assert_eq!(r.zero_overhead.recommendation, ZeroOverheadAction::TooLarge); }
#[test]
fn extra_test_all_arch_decrement() { let archs = [HWLoopMicroArch::Generic, HWLoopMicroArch::Skylake, HWLoopMicroArch::Haswell, HWLoopMicroArch::Zen4]; for a in &archs { let mut hw = X86HardwareLoops::new(*a); let l = hw.lower_loop_decrement("ecx", "edx", -1, LoopCounterPredicate::NE, ".L", 32); assert!(!l.x86_sequence.is_empty()); } }
#[test]
fn extra_test_custom_reg() { let mut hw = make_skl(); let l = hw.lower_set_loop_iterations("r12d", 42, 32); assert_eq!(l.counter_reg.as_deref(), Some("r12d")); }
#[test]
fn extra_test_large_trip() { let mut hw = make_skl(); let l = hw.lower_set_loop_iterations("ecx", u64::MAX, 32); assert!(l.x86_sequence[0].contains(&u64::MAX.to_string())); }
#[test]
fn extra_test_trip_zero() { let hw = make_skl(); assert!(!hw.is_hardware_loop_candidate(0, 10)); }
#[test]
fn extra_test_tsx_empty() { let hw = make_skl(); assert!(hw.analyze_tsx_compatibility(&[])); }
#[test]
fn extra_test_tsx_xrelease_prefix() { let hw = make_skl(); let mut r = TSXHLELoop::new("s", "e"); r.uses_xrelease = true; let seq = hw.generate_hle_prefixes(&r); assert_eq!(seq.len(), 1); assert!(seq[0].contains("XRELEASE")); }
#[test]
fn extra_test_tsx_both_prefixes() { let hw = make_skl(); let mut r = TSXHLELoop::new("s", "e"); r.uses_xacquire = true; r.uses_xrelease = true; let seq = hw.generate_hle_prefixes(&r); assert_eq!(seq.len(), 2); }
#[test]
fn extra_test_cet_validation_cet_enforced_ok() { let mut hw = make_skl().with_cet(true); let mut pat = CETLoopPattern::new(".L_hdr"); pat.has_endbr = true; let result = hw.validate_cet_loop(pat); assert!(result.is_cet_compatible); }
#[test]
fn extra_test_legality_counter_live_out_legal() { let mut hw = make_skl(); let leg = hw.check_legality(true, true, Some(100), false, false, true, true); assert!(leg.is_legal); }
#[test]
fn extra_test_candidate_exact_min_body() { let hw = make_skl(); assert!(hw.is_hardware_loop_candidate(4, 64)); }
#[test]
fn extra_test_not_candidate_exceed_body() { let hw = make_skl(); assert!(!hw.is_hardware_loop_candidate(4, 65)); }
#[test]
fn extra_test_microarch_display_all() { let archs = [HWLoopMicroArch::Generic, HWLoopMicroArch::SandyBridge, HWLoopMicroArch::IvyBridge, HWLoopMicroArch::Haswell, HWLoopMicroArch::Broadwell, HWLoopMicroArch::Skylake, HWLoopMicroArch::KabyLake, HWLoopMicroArch::CoffeeLake, HWLoopMicroArch::CascadeLake, HWLoopMicroArch::CometLake, HWLoopMicroArch::IceLake, HWLoopMicroArch::TigerLake, HWLoopMicroArch::RocketLake, HWLoopMicroArch::AlderLakeP, HWLoopMicroArch::AlderLakeE, HWLoopMicroArch::RaptorLakeP, HWLoopMicroArch::SapphireRapids, HWLoopMicroArch::EmeraldRapids, HWLoopMicroArch::GraniteRapids, HWLoopMicroArch::Zen1, HWLoopMicroArch::Zen2, HWLoopMicroArch::Zen3, HWLoopMicroArch::Zen4, HWLoopMicroArch::Zen5]; for a in &archs { let s = format!("{}", a); assert!(!s.is_empty()); } }
}