use crate::clang::{
ClangCodeGen, ClangDriver, ClangOptions, ClangSourceLocation, DiagID, DiagSeverity,
DiagnosticBuilder, DiagnosticEngine, FixItHint, Parser, Sema, Token, TokenKind,
};
use crate::openmp::{
MapType, MappedVariable, OMPRuntimeCall, OpenMPLoopLowering, OpenMPParallelLowering,
OpenMPRuntime, OpenMPSIMDLowering, TargetDataDirective, TargetDevice, TargetOffloadEngine,
TargetRegion, OMP_DEFAULT_CHUNK_SIZE, OMP_MAX_THREADS,
};
use crate::openmp_x86::{
current_gtid, default_ident, get_runtime, omp_initialize, omp_parse_environment,
with_current_icv, CriticalLockTable, ICVStorage, Ident, OMPLock, OMPNestLock, ProcBind,
ReductionContext, ReductionEntry, ReductionOp, ReductionTable, ReductionTypeInfo,
TaskDescriptor, TaskExecutionState, TaskQueue, TeamBarrier, ThreadPool, WorkerThread,
WorkshareState, X86OpenMP, BARRIER_PATTERN_FOR, BARRIER_PATTERN_PARALLEL,
BARRIER_PATTERN_REDUCTION, BARRIER_PATTERN_SECTIONS, BARRIER_PATTERN_SINGLE,
BARRIER_PATTERN_TASKGROUP, CANCEL_LOOP, CANCEL_PARALLEL, CANCEL_SECTIONS, CANCEL_TASKGROUP,
DEPEND_IN, DEPEND_INOUT, DEPEND_MUTEXINOUTSET, DEPEND_OUT, GLOBAL_RUNTIME,
OMP_DEFAULT_CHUNK_SIZE_X86, OMP_DEFAULT_GRAINSIZE, OMP_DEFAULT_NUM_TASKS,
OMP_DEFAULT_NUM_THREADS, OMP_DEFAULT_STACK_SIZE, OMP_MAX_ACTIVE_LEVELS,
OMP_MAX_NEST_LOCK_DEPTH, OMP_MAX_TEAMS, OMP_MAX_THREADS_PER_TEAM, OMP_MAX_THREADS_X86,
TASK_FLAG_DETACHABLE, TASK_FLAG_FINAL, TASK_FLAG_IMPLICIT, TASK_FLAG_MERGABLE,
TASK_FLAG_NONDEFERRED, TASK_FLAG_PRIORITY, TASK_FLAG_TARGET, TASK_FLAG_UNTIED,
};
use crate::x86::{
x86_avx10, x86_full_instr_info, x86_intrinsics_full, x86_microarch_full, x86_optimize,
x86_schedule_model, AsmSyntax, X86AsmPrinter, X86FullInstrInfo, X86InstrInfo, X86MCDecoder,
X86MCEncoder, X86OptStats, X86PeepholeOptimizer, X86RegisterInfo, X86Subtarget,
X86TargetMachine, X86_64_REG_COUNT, X86_ENDIANNESS, X86_MAX_ALIGNMENT, X86_PAGE_SIZE,
X86_RED_ZONE_SIZE_64, X86_STACK_ALIGNMENT_64,
};
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
use std::fmt;
use std::mem;
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, OnceLock, RwLock};
pub type OMPContext = X86OpenMPFull;
pub type OMPCodeGen = X86OpenMPCodeGen;
pub type OMPIRGen = X86OpenMPIRGen;
pub type OMPRuntime = X86OpenMPRuntime;
pub type OMPParser = X86OmpDirectiveParser;
pub const OMP_VERSION_MAJOR: u32 = 5;
pub const OMP_VERSION_MINOR: u32 = 2;
pub const OMP_VERSION: u32 = OMP_VERSION_MAJOR * 100 + OMP_VERSION_MINOR;
pub const X86_CACHE_LINE_SIZE: usize = 64;
pub const X86_OMP_MAX_THREADS: u32 = 1024;
pub const X86_DEFAULT_SIMD_WIDTH: u32 = 256;
pub const X86_MAX_SIMD_WIDTH: u32 = 512;
pub const X86_MIN_SIMD_WIDTH: u32 = 128;
pub const X86_PAUSE_HINT: u8 = 0;
pub const X86_MAX_SPIN_COUNT: u32 = 1000;
pub mod kmpc_x86 {
pub const FORK_CALL: &str = "__kmpc_fork_call";
pub const FORK_TEAMS: &str = "__kmpc_fork_teams";
pub const PUSH_NUM_THREADS: &str = "__kmpc_push_num_threads";
pub const PUSH_PROC_BIND: &str = "__kmpc_push_proc_bind";
pub const GLOBAL_THREAD_NUM: &str = "__kmpc_global_thread_num";
pub const BOUND_THREAD_NUM: &str = "__kmpc_bound_thread_num";
pub const FOR_STATIC_INIT_4: &str = "__kmpc_for_static_init_4";
pub const FOR_STATIC_INIT_8: &str = "__kmpc_for_static_init_8";
pub const FOR_STATIC_INIT_4U: &str = "__kmpc_for_static_init_4u";
pub const FOR_STATIC_INIT_8U: &str = "__kmpc_for_static_init_8u";
pub const FOR_STATIC_NEXT: &str = "__kmpc_for_static_next";
pub const FOR_STATIC_FINI: &str = "__kmpc_for_static_fini";
pub const DISPATCH_INIT_4: &str = "__kmpc_dispatch_init_4";
pub const DISPATCH_INIT_8: &str = "__kmpc_dispatch_init_8";
pub const DISPATCH_NEXT_4: &str = "__kmpc_dispatch_next_4";
pub const DISPATCH_NEXT_8: &str = "__kmpc_dispatch_next_8";
pub const DISPATCH_FINI_4: &str = "__kmpc_dispatch_fini_4";
pub const DISPATCH_FINI_8: &str = "__kmpc_dispatch_fini_8";
pub const DIST_FOR_STATIC_INIT_4: &str = "__kmpc_dist_for_static_init_4";
pub const DIST_FOR_STATIC_INIT_8: &str = "__kmpc_dist_for_static_init_8";
pub const DISTRIBUTE_STATIC_INIT: &str = "__kmpc_distribute_static_init";
pub const DISTRIBUTE_STATIC_FINI: &str = "__kmpc_distribute_static_fini";
pub const BARRIER: &str = "__kmpc_barrier";
pub const CANCEL_BARRIER: &str = "__kmpc_cancel_barrier";
pub const CRITICAL: &str = "__kmpc_critical";
pub const END_CRITICAL: &str = "__kmpc_end_critical";
pub const MASTER_BEGIN: &str = "__kmpc_master";
pub const MASTER_END: &str = "__kmpc_end_master";
pub const SINGLE_BEGIN: &str = "__kmpc_single";
pub const SINGLE_END: &str = "__kmpc_end_single";
pub const ORDERED_BEGIN: &str = "__kmpc_ordered";
pub const ORDERED_END: &str = "__kmpc_end_ordered";
pub const ATOMIC_RD_4: &str = "__kmpc_atomic_rd_4";
pub const ATOMIC_WR_4: &str = "__kmpc_atomic_wr_4";
pub const ATOMIC_RD_8: &str = "__kmpc_atomic_rd_8";
pub const ATOMIC_WR_8: &str = "__kmpc_atomic_wr_8";
pub const ATOMIC_RD_16: &str = "__kmpc_atomic_rd_16";
pub const ATOMIC_WR_16: &str = "__kmpc_atomic_wr_16";
pub const ATOMIC_FLOAT_RD: &str = "__kmpc_atomic_float4_rd";
pub const ATOMIC_FLOAT_WR: &str = "__kmpc_atomic_float4_wr";
pub const ATOMIC_DOUBLE_RD: &str = "__kmpc_atomic_float8_rd";
pub const ATOMIC_DOUBLE_WR: &str = "__kmpc_atomic_float8_wr";
pub const ATOMIC_ADD_4: &str = "__kmpc_atomic_4_add";
pub const ATOMIC_ADD_8: &str = "__kmpc_atomic_8_add";
pub const ATOMIC_SUB_4: &str = "__kmpc_atomic_4_sub";
pub const ATOMIC_SUB_8: &str = "__kmpc_atomic_8_sub";
pub const ATOMIC_AND_4: &str = "__kmpc_atomic_4_and";
pub const ATOMIC_AND_8: &str = "__kmpc_atomic_8_and";
pub const ATOMIC_OR_4: &str = "__kmpc_atomic_4_or";
pub const ATOMIC_OR_8: &str = "__kmpc_atomic_8_or";
pub const ATOMIC_XOR_4: &str = "__kmpc_atomic_4_xor";
pub const ATOMIC_XOR_8: &str = "__kmpc_atomic_8_xor";
pub const ATOMIC_MIN_4: &str = "__kmpc_atomic_4_min";
pub const ATOMIC_MIN_8: &str = "__kmpc_atomic_8_min";
pub const ATOMIC_MAX_4: &str = "__kmpc_atomic_4_max";
pub const ATOMIC_MAX_8: &str = "__kmpc_atomic_8_max";
pub const ATOMIC_CAS_4: &str = "__kmpc_atomic_4_cas";
pub const ATOMIC_CAS_8: &str = "__kmpc_atomic_8_cas";
pub const ATOMIC_SWAP_4: &str = "__kmpc_atomic_4_swap";
pub const ATOMIC_SWAP_8: &str = "__kmpc_atomic_8_swap";
pub const FLUSH: &str = "__kmpc_flush";
pub const CANCEL: &str = "__kmpc_cancel";
pub const CANCELLATION_POINT: &str = "__kmpc_cancellationpoint";
pub const THREADPRIVATE_REGISTER: &str = "__kmpc_threadprivate_register";
pub const THREADPRIVATE_CACHED: &str = "__kmpc_threadprivate_cached";
pub const TASK_ALLOC: &str = "__kmpc_omp_task_alloc";
pub const TASK: &str = "__kmpc_omp_task";
pub const TASK_WITH_DEPS: &str = "__kmpc_omp_task_with_deps";
pub const TASK_BEGIN_IF0: &str = "__kmpc_omp_task_begin_if0";
pub const TASK_COMPLETE_IF0: &str = "__kmpc_omp_task_complete_if0";
pub const TASK_MODIFY_FLAGS: &str = "__kmpc_omp_task_modify_flags";
pub const TASKWAIT: &str = "__kmpc_omp_taskwait";
pub const TASKYIELD: &str = "__kmpc_omp_taskyield";
pub const TASKGROUP: &str = "__kmpc_taskgroup";
pub const TASKGROUP_END: &str = "__kmpc_end_taskgroup";
pub const TASKLOOP: &str = "__kmpc_taskloop";
pub const TASKLOOP_TASK: &str = "__kmpc_omp_taskloop_task";
pub const TARGET_INIT: &str = "__kmpc_target_init";
pub const TARGET_DEINIT: &str = "__kmpc_target_deinit";
pub const TARGET_DATA_BEGIN: &str = "__kmpc_data_target_begin";
pub const TARGET_DATA_END: &str = "__kmpc_data_target_end";
pub const TARGET: &str = "__kmpc_target";
pub const TARGET_UPDATE: &str = "__kmpc_target_update";
pub const TARGET_MEMCPY: &str = "__kmpc_target_memcpy";
pub const TARGET_TASK_ALLOC: &str = "__kmpc_omp_target_task_alloc";
pub const REDUCE: &str = "__kmpc_reduce";
pub const END_REDUCE: &str = "__kmpc_end_reduce";
pub const REDUCE_NOWAIT: &str = "__kmpc_reduce_nowait";
pub const SIMD: &str = "__kmpc_simd";
pub const ALLOC_SHARED: &str = "__kmpc_alloc_shared";
pub const FREE_SHARED: &str = "__kmpc_free_shared";
pub const SECTIONS_INIT: &str = "__kmpc_sections_init";
pub const NEXT_SECTION: &str = "__kmpc_next_section";
}
pub mod tgt_x86 {
pub const REGISTER_LIB: &str = "__tgt_register_lib";
pub const UNREGISTER_LIB: &str = "__tgt_unregister_lib";
pub const TARGET: &str = "__tgt_target";
pub const TARGET_NOWAIT: &str = "__tgt_target_nowait";
pub const TARGET_TEAMS: &str = "__tgt_target_teams";
pub const TARGET_TEAMS_NOWAIT: &str = "__tgt_target_teams_nowait";
pub const TARGET_DATA_BEGIN: &str = "__tgt_target_data_begin";
pub const TARGET_DATA_END: &str = "__tgt_target_data_end";
pub const TARGET_DATA_UPDATE: &str = "__tgt_target_data_update";
pub const TARGET_ALLOC: &str = "omp_target_alloc";
pub const TARGET_FREE: &str = "omp_target_free";
pub const TARGET_MEMCPY: &str = "omp_target_memcpy";
pub const TARGET_MEMCPY_RECT: &str = "omp_target_memcpy_rect";
pub const TARGET_ASSOCIATE_PTR: &str = "omp_target_associate_ptr";
pub const TARGET_DISASSOCIATE_PTR: &str = "omp_target_disassociate_ptr";
pub const IS_INITIAL_DEVICE: &str = "omp_is_initial_device";
pub const GET_INITIAL_DEVICE: &str = "omp_get_initial_device";
pub const GET_NUM_DEVICES: &str = "omp_get_num_devices";
pub const GET_DEFAULT_DEVICE: &str = "omp_get_default_device";
pub const SET_DEFAULT_DEVICE: &str = "omp_set_default_device";
pub const OFFLOADING_ENTRY: &str = "__omp_offloading_";
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum X86OmpDirectiveKind {
Parallel,
ParallelFor,
ParallelSections,
ParallelForSimd,
For,
ForSimd,
Sections,
Section,
Single,
Master,
Masked,
Simd,
DeclareSimd,
Task,
TaskWait,
TaskYield,
TaskGroup,
TaskLoop,
TaskLoopSimd,
Barrier,
Critical,
Atomic,
Flush,
Ordered,
Target,
TargetData,
TargetEnterData,
TargetExitData,
TargetUpdate,
DeclareTarget,
TargetTeams,
TeamsDistribute,
TeamsDistributeSimd,
TeamsDistributeParallelFor,
TeamsDistributeParallelForSimd,
Tile,
Unroll,
Interchange,
Scan,
Allocate,
Cancel,
CancellationPoint,
ThreadPrivate,
CopyPrivate,
Distribute,
DistributeParallelFor,
DistributeSimd,
DistributeParallelForSimd,
Teams,
Requires,
DeclareVariant,
DeclareMapper,
TargetParallel,
TargetParallelFor,
TargetParallelForSimd,
TargetSimd,
Loop,
Scope,
Nothing,
Error,
Metadirective,
}
impl X86OmpDirectiveKind {
pub fn is_parallel(&self) -> bool {
matches!(
self,
Self::Parallel
| Self::ParallelFor
| Self::ParallelSections
| Self::ParallelForSimd
| Self::TargetParallel
| Self::TargetParallelFor
| Self::TargetParallelForSimd
)
}
pub fn is_worksharing(&self) -> bool {
matches!(
self,
Self::For
| Self::ForSimd
| Self::Sections
| Self::Single
| Self::Distribute
| Self::DistributeParallelFor
| Self::DistributeSimd
| Self::DistributeParallelForSimd
)
}
pub fn is_target(&self) -> bool {
matches!(
self,
Self::Target
| Self::TargetData
| Self::TargetEnterData
| Self::TargetExitData
| Self::TargetUpdate
| Self::TargetTeams
| Self::TargetParallel
| Self::TargetParallelFor
| Self::TargetParallelForSimd
| Self::TargetSimd
| Self::TeamsDistribute
| Self::TeamsDistributeSimd
| Self::TeamsDistributeParallelFor
| Self::TeamsDistributeParallelForSimd
)
}
pub fn is_task(&self) -> bool {
matches!(self, Self::Task | Self::TaskLoop | Self::TaskLoopSimd)
}
pub fn is_synchronization(&self) -> bool {
matches!(
self,
Self::Barrier | Self::Critical | Self::Atomic | Self::Flush | Self::Ordered
)
}
pub fn is_simd(&self) -> bool {
matches!(
self,
Self::Simd
| Self::ForSimd
| Self::ParallelForSimd
| Self::TaskLoopSimd
| Self::DistributeSimd
| Self::DistributeParallelForSimd
| Self::TargetSimd
| Self::TargetParallelForSimd
| Self::TeamsDistributeSimd
| Self::TeamsDistributeParallelForSimd
| Self::DeclareSimd
)
}
pub fn is_loop_transform(&self) -> bool {
matches!(
self,
Self::Tile | Self::Unroll | Self::Interchange | Self::Scan | Self::Loop
)
}
pub fn name(&self) -> &'static str {
match self {
Self::Parallel => "parallel",
Self::ParallelFor => "parallel for",
Self::ParallelSections => "parallel sections",
Self::ParallelForSimd => "parallel for simd",
Self::For => "for",
Self::ForSimd => "for simd",
Self::Sections => "sections",
Self::Section => "section",
Self::Single => "single",
Self::Master => "master",
Self::Masked => "masked",
Self::Simd => "simd",
Self::DeclareSimd => "declare simd",
Self::Task => "task",
Self::TaskWait => "taskwait",
Self::TaskYield => "taskyield",
Self::TaskGroup => "taskgroup",
Self::TaskLoop => "taskloop",
Self::TaskLoopSimd => "taskloop simd",
Self::Barrier => "barrier",
Self::Critical => "critical",
Self::Atomic => "atomic",
Self::Flush => "flush",
Self::Ordered => "ordered",
Self::Target => "target",
Self::TargetData => "target data",
Self::TargetEnterData => "target enter data",
Self::TargetExitData => "target exit data",
Self::TargetUpdate => "target update",
Self::DeclareTarget => "declare target",
Self::TargetTeams => "target teams",
Self::TeamsDistribute => "teams distribute",
Self::TeamsDistributeSimd => "teams distribute simd",
Self::TeamsDistributeParallelFor => "teams distribute parallel for",
Self::TeamsDistributeParallelForSimd => "teams distribute parallel for simd",
Self::Tile => "tile",
Self::Unroll => "unroll",
Self::Interchange => "interchange",
Self::Scan => "scan",
Self::Allocate => "allocate",
Self::Cancel => "cancel",
Self::CancellationPoint => "cancellation point",
Self::ThreadPrivate => "threadprivate",
Self::CopyPrivate => "copyprivate",
Self::Distribute => "distribute",
Self::DistributeParallelFor => "distribute parallel for",
Self::DistributeSimd => "distribute simd",
Self::DistributeParallelForSimd => "distribute parallel for simd",
Self::Teams => "teams",
Self::Requires => "requires",
Self::DeclareVariant => "declare variant",
Self::DeclareMapper => "declare mapper",
Self::TargetParallel => "target parallel",
Self::TargetParallelFor => "target parallel for",
Self::TargetParallelForSimd => "target parallel for simd",
Self::TargetSimd => "target simd",
Self::Loop => "loop",
Self::Scope => "scope",
Self::Nothing => "nothing",
Self::Error => "error",
Self::Metadirective => "metadirective",
}
}
pub fn from_pragma_name(name: &str) -> Option<Self> {
match name {
"parallel" => Some(Self::Parallel),
"parallel for" => Some(Self::ParallelFor),
"parallel sections" => Some(Self::ParallelSections),
"parallel for simd" => Some(Self::ParallelForSimd),
"for" => Some(Self::For),
"for simd" => Some(Self::ForSimd),
"sections" => Some(Self::Sections),
"section" => Some(Self::Section),
"single" => Some(Self::Single),
"master" => Some(Self::Master),
"masked" => Some(Self::Masked),
"simd" => Some(Self::Simd),
"declare simd" => Some(Self::DeclareSimd),
"task" => Some(Self::Task),
"taskwait" => Some(Self::TaskWait),
"taskyield" => Some(Self::TaskYield),
"taskgroup" => Some(Self::TaskGroup),
"taskloop" => Some(Self::TaskLoop),
"taskloop simd" => Some(Self::TaskLoopSimd),
"barrier" => Some(Self::Barrier),
"critical" => Some(Self::Critical),
"atomic" => Some(Self::Atomic),
"flush" => Some(Self::Flush),
"ordered" => Some(Self::Ordered),
"target" => Some(Self::Target),
"target data" => Some(Self::TargetData),
"target enter data" => Some(Self::TargetEnterData),
"target exit data" => Some(Self::TargetExitData),
"target update" => Some(Self::TargetUpdate),
"declare target" => Some(Self::DeclareTarget),
"target teams" => Some(Self::TargetTeams),
"teams distribute" => Some(Self::TeamsDistribute),
"teams distribute simd" => Some(Self::TeamsDistributeSimd),
"teams distribute parallel for" => Some(Self::TeamsDistributeParallelFor),
"teams distribute parallel for simd" => Some(Self::TeamsDistributeParallelForSimd),
"tile" => Some(Self::Tile),
"unroll" => Some(Self::Unroll),
"interchange" => Some(Self::Interchange),
"scan" => Some(Self::Scan),
"allocate" => Some(Self::Allocate),
"cancel" => Some(Self::Cancel),
"cancellation point" => Some(Self::CancellationPoint),
"threadprivate" => Some(Self::ThreadPrivate),
"copyprivate" => Some(Self::CopyPrivate),
"distribute" => Some(Self::Distribute),
"distribute parallel for" => Some(Self::DistributeParallelFor),
"distribute simd" => Some(Self::DistributeSimd),
"distribute parallel for simd" => Some(Self::DistributeParallelForSimd),
"teams" => Some(Self::Teams),
"requires" => Some(Self::Requires),
"declare variant" => Some(Self::DeclareVariant),
"declare mapper" => Some(Self::DeclareMapper),
"target parallel" => Some(Self::TargetParallel),
"target parallel for" => Some(Self::TargetParallelFor),
"target parallel for simd" => Some(Self::TargetParallelForSimd),
"target simd" => Some(Self::TargetSimd),
"loop" => Some(Self::Loop),
"scope" => Some(Self::Scope),
"nothing" => Some(Self::Nothing),
"error" => Some(Self::Error),
"metadirective" => Some(Self::Metadirective),
_ => None,
}
}
}
impl fmt::Display for X86OmpDirectiveKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.name())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86OmpScheduleKind {
Static,
Dynamic,
Guided,
Auto,
Runtime,
StaticChunked,
DynamicChunked,
GuidedChunked,
Monotonic,
Nonmonotonic,
}
impl X86OmpScheduleKind {
pub fn to_runtime_id(&self) -> i32 {
match self {
Self::Static | Self::StaticChunked => 1,
Self::Dynamic | Self::DynamicChunked => 2,
Self::Guided | Self::GuidedChunked => 3,
Self::Auto => 4,
Self::Runtime => 5,
Self::Monotonic => 0x80000000u32 as i32,
Self::Nonmonotonic => 0x40000000u32 as i32,
}
}
pub fn from_name(name: &str) -> Option<Self> {
match name {
"static" => Some(Self::Static),
"dynamic" => Some(Self::Dynamic),
"guided" => Some(Self::Guided),
"auto" => Some(Self::Auto),
"runtime" => Some(Self::Runtime),
_ => None,
}
}
pub fn name(&self) -> &'static str {
match self {
Self::Static => "static",
Self::Dynamic => "dynamic",
Self::Guided => "guided",
Self::Auto => "auto",
Self::Runtime => "runtime",
Self::StaticChunked => "static",
Self::DynamicChunked => "dynamic",
Self::GuidedChunked => "guided",
Self::Monotonic => "monotonic",
Self::Nonmonotonic => "nonmonotonic",
}
}
pub fn has_modifier(&self) -> bool {
matches!(self, Self::Monotonic | Self::Nonmonotonic)
}
}
impl Default for X86OmpScheduleKind {
fn default() -> Self {
Self::Static
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86OmpMapType {
To,
From,
ToFrom,
Alloc,
Release,
Delete,
Present,
CloseMapper,
AlwaysMapper,
}
impl X86OmpMapType {
pub fn from_name(name: &str) -> Option<Self> {
match name {
"to" => Some(Self::To),
"from" => Some(Self::From),
"tofrom" => Some(Self::ToFrom),
"alloc" => Some(Self::Alloc),
"release" => Some(Self::Release),
"delete" => Some(Self::Delete),
"present" => Some(Self::Present),
"close" => Some(Self::CloseMapper),
"always" => Some(Self::AlwaysMapper),
_ => None,
}
}
pub fn name(&self) -> &'static str {
match self {
Self::To => "to",
Self::From => "from",
Self::ToFrom => "tofrom",
Self::Alloc => "alloc",
Self::Release => "release",
Self::Delete => "delete",
Self::Present => "present",
Self::CloseMapper => "close",
Self::AlwaysMapper => "always",
}
}
pub fn is_data_movement(&self) -> bool {
matches!(self, Self::To | Self::From | Self::ToFrom)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86OmpAtomicKind {
Read,
Write,
Update,
Capture,
Compare,
WeakCompare,
Hint,
}
impl X86OmpAtomicKind {
pub fn name(&self) -> &'static str {
match self {
Self::Read => "read",
Self::Write => "write",
Self::Update => "update",
Self::Capture => "capture",
Self::Compare => "compare",
Self::WeakCompare => "weak compare",
Self::Hint => "hint",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum X86ReductionOp {
Add,
Sub,
Mul,
Min,
Max,
And,
Or,
Xor,
BAnd,
BOr,
LogicalAnd,
LogicalOr,
Custom,
}
impl X86ReductionOp {
pub fn from_name(name: &str) -> Option<Self> {
match name {
"+" => Some(Self::Add),
"-" => Some(Self::Sub),
"*" => Some(Self::Mul),
"min" => Some(Self::Min),
"max" => Some(Self::Max),
"&&" => Some(Self::LogicalAnd),
"||" => Some(Self::LogicalOr),
"&" => Some(Self::BAnd),
"|" => Some(Self::BOr),
"^" => Some(Self::Xor),
_ => None,
}
}
pub fn name(&self) -> &'static str {
match self {
Self::Add => "+",
Self::Sub => "-",
Self::Mul => "*",
Self::Min => "min",
Self::Max => "max",
Self::And => "&&",
Self::Or => "||",
Self::Xor => "^",
Self::BAnd => "&",
Self::BOr => "|",
Self::LogicalAnd => "&&",
Self::LogicalOr => "||",
Self::Custom => "custom",
}
}
pub fn is_commutative(&self) -> bool {
matches!(
self,
Self::Add
| Self::Mul
| Self::Min
| Self::Max
| Self::And
| Self::Or
| Self::Xor
| Self::BAnd
| Self::BOr
| Self::LogicalAnd
| Self::LogicalOr
)
}
pub fn neutral_value(&self) -> Option<i64> {
match self {
Self::Add | Self::Sub | Self::Or | Self::BOr | Self::Xor => Some(0),
Self::Mul => Some(1),
Self::And | Self::BAnd => Some(-1), Self::Max => Some(i64::MIN),
Self::Min => Some(i64::MAX),
Self::LogicalAnd => Some(1),
Self::LogicalOr => Some(0),
Self::Custom => None,
}
}
pub fn neutral_value_f64(&self) -> Option<f64> {
match self {
Self::Add | Self::Sub => Some(0.0),
Self::Mul => Some(1.0),
Self::Max => Some(f64::NEG_INFINITY),
Self::Min => Some(f64::INFINITY),
_ => None,
}
}
pub fn neutral_value_f32(&self) -> Option<f32> {
match self {
Self::Add | Self::Sub => Some(0.0f32),
Self::Mul => Some(1.0f32),
Self::Max => Some(f32::NEG_INFINITY),
Self::Min => Some(f32::INFINITY),
_ => None,
}
}
}
impl fmt::Display for X86ReductionOp {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.name())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86DataSharingAttr {
Shared,
Private,
FirstPrivate,
LastPrivate,
Linear(i32),
Reduction(X86ReductionOp),
CopyIn,
CopyPrivate,
}
impl X86DataSharingAttr {
pub fn name(&self) -> &'static str {
match self {
Self::Shared => "shared",
Self::Private => "private",
Self::FirstPrivate => "firstprivate",
Self::LastPrivate => "lastprivate",
Self::Linear(_) => "linear",
Self::Reduction(_) => "reduction",
Self::CopyIn => "copyin",
Self::CopyPrivate => "copyprivate",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86DefaultAttr {
None,
Shared,
Private,
FirstPrivate,
}
impl X86DefaultAttr {
pub fn from_name(name: &str) -> Option<Self> {
match name {
"none" => Some(Self::None),
"shared" => Some(Self::Shared),
"private" => Some(Self::Private),
"firstprivate" => Some(Self::FirstPrivate),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86ProcBind {
False,
True,
Master,
Close,
Spread,
Primary,
}
impl X86ProcBind {
pub fn name(&self) -> &'static str {
match self {
Self::False => "false",
Self::True => "true",
Self::Master => "master",
Self::Close => "close",
Self::Spread => "spread",
Self::Primary => "primary",
}
}
pub fn from_name(name: &str) -> Option<Self> {
match name {
"false" => Some(Self::False),
"true" => Some(Self::True),
"master" => Some(Self::Master),
"close" => Some(Self::Close),
"spread" => Some(Self::Spread),
"primary" => Some(Self::Primary),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86OrderModifier {
Concurrent,
Unconstrained,
Reproducible,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86OrderedKind {
None,
Ordered,
OrderedN(u32),
}
#[derive(Debug, Clone)]
pub struct X86SimdClause {
pub safelen: Option<u32>,
pub simdlen: Option<u32>,
pub aligned_vars: Vec<String>,
pub aligned_alignments: Vec<u32>,
pub nontemporal_vars: Vec<String>,
pub linear_vars: Vec<(String, i32)>,
pub uniform_vars: Vec<String>,
pub private_vars: Vec<String>,
pub reductions: Vec<(String, X86ReductionOp)>,
}
impl X86SimdClause {
pub fn new() -> Self {
Self {
safelen: None,
simdlen: None,
aligned_vars: Vec::new(),
aligned_alignments: Vec::new(),
nontemporal_vars: Vec::new(),
linear_vars: Vec::new(),
uniform_vars: Vec::new(),
private_vars: Vec::new(),
reductions: Vec::new(),
}
}
pub fn has_any(&self) -> bool {
self.safelen.is_some()
|| self.simdlen.is_some()
|| !self.aligned_vars.is_empty()
|| !self.nontemporal_vars.is_empty()
|| !self.linear_vars.is_empty()
|| !self.uniform_vars.is_empty()
}
pub fn effective_vector_width(&self, hw_max: u32) -> u32 {
self.simdlen
.or(self.safelen)
.map(|w| w.min(hw_max))
.unwrap_or(hw_max)
}
}
impl Default for X86SimdClause {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum X86DependType {
In,
Out,
InOut,
MutexInOutSet,
DepObj,
Source,
Sink,
}
impl X86DependType {
pub fn from_name(name: &str) -> Option<Self> {
match name {
"in" => Some(Self::In),
"out" => Some(Self::Out),
"inout" => Some(Self::InOut),
"mutexinoutset" => Some(Self::MutexInOutSet),
"depobj" => Some(Self::DepObj),
"source" => Some(Self::Source),
"sink" => Some(Self::Sink),
_ => None,
}
}
pub fn runtime_id(&self) -> i32 {
match self {
Self::In => 1,
Self::Out => 2,
Self::InOut => 3,
Self::MutexInOutSet => 4,
Self::DepObj => 5,
Self::Source => 6,
Self::Sink => 7,
}
}
}
#[derive(Debug, Clone)]
pub struct X86MapClause {
pub variable_name: String,
pub map_type: X86OmpMapType,
pub array_section: bool,
pub array_start: Option<String>,
pub array_length: Option<String>,
pub pointer_kind: Option<String>,
pub present_modifier: bool,
pub always_modifier: bool,
pub close_modifier: bool,
}
impl X86MapClause {
pub fn new(name: &str, map_type: X86OmpMapType) -> Self {
Self {
variable_name: name.to_string(),
map_type,
array_section: false,
array_start: None,
array_length: None,
pointer_kind: None,
present_modifier: false,
always_modifier: false,
close_modifier: false,
}
}
pub fn with_array(mut self, start: &str, length: &str) -> Self {
self.array_section = true;
self.array_start = Some(start.to_string());
self.array_length = Some(length.to_string());
self
}
}
#[derive(Debug, Clone)]
pub struct X86LoopTransformClause {
pub tile_sizes: Option<Vec<u32>>,
pub unroll_factor: Option<u32>,
pub unroll_full: bool,
pub interchange_indices: Option<Vec<u32>>,
pub scan_inclusive: bool,
pub scan_exclusive: bool,
}
impl Default for X86LoopTransformClause {
fn default() -> Self {
Self {
tile_sizes: None,
unroll_factor: None,
unroll_full: false,
interchange_indices: None,
scan_inclusive: false,
scan_exclusive: false,
}
}
}
#[derive(Debug, Clone)]
pub struct X86OmpDirective {
pub kind: X86OmpDirectiveKind,
pub location: Option<ClangSourceLocation>,
pub critical_name: Option<String>,
pub schedule: X86OmpScheduleKind,
pub chunk_size: Option<u64>,
pub nowait: bool,
pub num_threads: Option<String>,
pub if_clause: Option<String>,
pub final_clause: Option<String>,
pub priority_clause: Option<String>,
pub default_attr: X86DefaultAttr,
pub proc_bind: Option<X86ProcBind>,
pub collapse: Option<u32>,
pub ordered: X86OrderedKind,
pub order_modifier: Option<X86OrderModifier>,
pub private_vars: Vec<String>,
pub firstprivate_vars: Vec<String>,
pub lastprivate_vars: Vec<String>,
pub shared_vars: Vec<String>,
pub copyin_vars: Vec<String>,
pub copyprivate_vars: Vec<String>,
pub reductions: Vec<(String, X86ReductionOp)>,
pub linear_vars: Vec<(String, i32)>,
pub map_clauses: Vec<X86MapClause>,
pub depend_clauses: Vec<(X86DependType, String)>,
pub device_id: Option<String>,
pub is_device_ptr: Vec<String>,
pub has_device_addr: Vec<String>,
pub uses_allocators: Vec<(String, String)>,
pub untied: bool,
pub mergeable: bool,
pub grainsize: Option<u64>,
pub num_tasks: Option<u32>,
pub simd_clause: X86SimdClause,
pub loop_transform: X86LoopTransformClause,
pub filter: Option<String>,
pub allocate_vars: Vec<(String, String)>,
pub affinity: Option<String>,
pub detach: Option<String>,
pub is_combined: bool,
pub sub_directives: Vec<X86OmpDirective>,
}
impl X86OmpDirective {
pub fn new(kind: X86OmpDirectiveKind) -> Self {
Self {
kind,
location: None,
critical_name: None,
schedule: X86OmpScheduleKind::default(),
chunk_size: None,
nowait: false,
num_threads: None,
if_clause: None,
final_clause: None,
priority_clause: None,
default_attr: X86DefaultAttr::Shared,
proc_bind: None,
collapse: None,
ordered: X86OrderedKind::None,
order_modifier: None,
private_vars: Vec::new(),
firstprivate_vars: Vec::new(),
lastprivate_vars: Vec::new(),
shared_vars: Vec::new(),
copyin_vars: Vec::new(),
copyprivate_vars: Vec::new(),
reductions: Vec::new(),
linear_vars: Vec::new(),
map_clauses: Vec::new(),
depend_clauses: Vec::new(),
device_id: None,
is_device_ptr: Vec::new(),
has_device_addr: Vec::new(),
uses_allocators: Vec::new(),
untied: false,
mergeable: false,
grainsize: None,
num_tasks: None,
simd_clause: X86SimdClause::default(),
loop_transform: X86LoopTransformClause::default(),
filter: None,
allocate_vars: Vec::new(),
affinity: None,
detach: None,
is_combined: false,
sub_directives: Vec::new(),
}
}
pub fn with_schedule(mut self, sched: X86OmpScheduleKind, chunk: Option<u64>) -> Self {
self.schedule = sched;
self.chunk_size = chunk;
self
}
pub fn with_num_threads(mut self, expr: &str) -> Self {
self.num_threads = Some(expr.to_string());
self
}
pub fn with_nowait(mut self) -> Self {
self.nowait = true;
self
}
pub fn with_collapse(mut self, n: u32) -> Self {
self.collapse = Some(n);
self
}
pub fn add_private(mut self, var: &str) -> Self {
self.private_vars.push(var.to_string());
self
}
pub fn add_shared(mut self, var: &str) -> Self {
self.shared_vars.push(var.to_string());
self
}
pub fn add_reduction(mut self, var: &str, op: X86ReductionOp) -> Self {
self.reductions.push((var.to_string(), op));
self
}
pub fn add_map(mut self, map: X86MapClause) -> Self {
self.map_clauses.push(map);
self
}
pub fn add_depend(mut self, dep_type: X86DependType, var: &str) -> Self {
self.depend_clauses.push((dep_type, var.to_string()));
self
}
pub fn get_runtime_call(&self) -> &'static str {
match self.kind {
X86OmpDirectiveKind::Parallel | X86OmpDirectiveKind::TargetParallel => {
kmpc_x86::FORK_CALL
}
X86OmpDirectiveKind::Teams => kmpc_x86::FORK_TEAMS,
X86OmpDirectiveKind::For | X86OmpDirectiveKind::ParallelFor => {
kmpc_x86::FOR_STATIC_INIT_4
}
X86OmpDirectiveKind::Barrier => kmpc_x86::BARRIER,
X86OmpDirectiveKind::Critical => kmpc_x86::CRITICAL,
X86OmpDirectiveKind::Single => kmpc_x86::SINGLE_BEGIN,
X86OmpDirectiveKind::Master => kmpc_x86::MASTER_BEGIN,
X86OmpDirectiveKind::Task => kmpc_x86::TASK,
X86OmpDirectiveKind::TaskWait => kmpc_x86::TASKWAIT,
X86OmpDirectiveKind::TaskGroup => kmpc_x86::TASKGROUP,
X86OmpDirectiveKind::Flush => kmpc_x86::FLUSH,
X86OmpDirectiveKind::Target => kmpc_x86::TARGET,
_ => kmpc_x86::BARRIER,
}
}
pub fn has_data_environment(&self) -> bool {
!self.private_vars.is_empty()
|| !self.firstprivate_vars.is_empty()
|| !self.lastprivate_vars.is_empty()
|| !self.shared_vars.is_empty()
|| !self.reductions.is_empty()
|| !self.copyin_vars.is_empty()
|| !self.copyprivate_vars.is_empty()
|| !self.map_clauses.is_empty()
|| !self.allocate_vars.is_empty()
}
pub fn all_captured_vars(&self) -> Vec<String> {
let mut vars = Vec::new();
vars.extend(self.private_vars.iter().cloned());
vars.extend(self.firstprivate_vars.iter().cloned());
vars.extend(self.lastprivate_vars.iter().cloned());
vars.extend(self.shared_vars.iter().cloned());
vars.extend(self.copyin_vars.iter().cloned());
for (v, _) in &self.reductions {
vars.push(v.clone());
}
vars
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum X86SimdLevel {
None,
SSE,
SSE2,
SSE3,
SSSE3,
SSE41,
SSE42,
AVX,
AVX2,
AVX512F,
AVX512BW,
AVX512DQ,
AVX512VL,
AVX512CD,
AVX512ER,
AVX512PF,
AVX512BF16,
AVX512VPOPCNTDQ,
AVX512VNNI,
AMX,
}
impl X86SimdLevel {
pub fn vector_width_bits(&self) -> u32 {
match self {
Self::None => 0,
Self::SSE | Self::SSE2 | Self::SSE3 | Self::SSSE3 | Self::SSE41 | Self::SSE42 => 128,
Self::AVX | Self::AVX2 => 256,
Self::AVX512F
| Self::AVX512BW
| Self::AVX512DQ
| Self::AVX512VL
| Self::AVX512CD
| Self::AVX512ER
| Self::AVX512PF
| Self::AVX512BF16
| Self::AVX512VPOPCNTDQ
| Self::AVX512VNNI => 512,
Self::AMX => 512,
}
}
pub fn from_features(features: &[String]) -> Self {
let feat_set: HashSet<&str> = features.iter().map(|s| s.as_str()).collect();
if feat_set.contains("avx512f") || feat_set.contains("avx512") {
if feat_set.contains("avx512bw") {
Self::AVX512BW
} else {
Self::AVX512F
}
} else if feat_set.contains("avx2") {
Self::AVX2
} else if feat_set.contains("avx") {
Self::AVX
} else if feat_set.contains("sse4.2") {
Self::SSE42
} else if feat_set.contains("sse4.1") {
Self::SSE41
} else if feat_set.contains("ssse3") {
Self::SSSE3
} else if feat_set.contains("sse3") {
Self::SSE3
} else if feat_set.contains("sse2") {
Self::SSE2
} else if feat_set.contains("sse") {
Self::SSE
} else {
Self::None
}
}
pub fn f32_element_count(&self) -> u32 {
self.vector_width_bits() / 32
}
pub fn f64_element_count(&self) -> u32 {
self.vector_width_bits() / 64
}
pub fn i32_element_count(&self) -> u32 {
self.vector_width_bits() / 32
}
pub fn i64_element_count(&self) -> u32 {
self.vector_width_bits() / 64
}
pub fn i8_element_count(&self) -> u32 {
self.vector_width_bits() / 8
}
pub fn i16_element_count(&self) -> u32 {
self.vector_width_bits() / 16
}
pub fn supports_masked_operations(&self) -> bool {
self.vector_width_bits() >= 256
}
pub fn name(&self) -> &'static str {
match self {
Self::None => "none",
Self::SSE => "sse",
Self::SSE2 => "sse2",
Self::SSE3 => "sse3",
Self::SSSE3 => "ssse3",
Self::SSE41 => "sse4.1",
Self::SSE42 => "sse4.2",
Self::AVX => "avx",
Self::AVX2 => "avx2",
Self::AVX512F => "avx512f",
Self::AVX512BW => "avx512bw",
Self::AVX512DQ => "avx512dq",
Self::AVX512VL => "avx512vl",
Self::AVX512CD => "avx512cd",
Self::AVX512ER => "avx512er",
Self::AVX512PF => "avx512pf",
Self::AVX512BF16 => "avx512bf16",
Self::AVX512VPOPCNTDQ => "avx512vpopcntdq",
Self::AVX512VNNI => "avx512vnni",
Self::AMX => "amx",
}
}
}
impl fmt::Display for X86SimdLevel {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.name())
}
}
#[repr(C, align(64))]
#[derive(Debug, Clone)]
pub struct X86ICVStorage {
pub nthreads_var: u32,
pub dyn_var: bool,
pub nest_var: bool,
pub run_sched_var: i32,
pub run_sched_chunk: i64,
pub thread_limit_var: u32,
pub max_active_levels_var: u32,
pub active_levels_var: u32,
pub levels_var: u32,
pub place_partition_var: i32,
pub proc_bind_var: i32,
pub cancellation_var: bool,
pub def_device_var: i32,
pub stacksize_var: u64,
pub wait_policy_var: i32,
pub default_device_var: i32,
_pad: [u8; 16],
}
impl X86ICVStorage {
pub fn new() -> Self {
Self {
nthreads_var: OMP_DEFAULT_NUM_THREADS as u32,
dyn_var: false,
nest_var: false,
run_sched_var: 1, run_sched_chunk: 1,
thread_limit_var: X86_OMP_MAX_THREADS,
max_active_levels_var: 1,
active_levels_var: 0,
levels_var: 0,
place_partition_var: 0,
proc_bind_var: 0,
cancellation_var: false,
def_device_var: 0,
stacksize_var: OMP_DEFAULT_STACK_SIZE as u64,
wait_policy_var: 0,
default_device_var: 0,
_pad: [0u8; 16],
}
}
pub fn effective_num_threads(&self) -> u32 {
if self.dyn_var {
self.nthreads_var
} else {
self.nthreads_var.max(1)
}
}
pub fn from_environment() -> Self {
let mut icv = Self::new();
if let Ok(val) = std::env::var("OMP_NUM_THREADS") {
if let Ok(n) = val.parse::<u32>() {
icv.nthreads_var = n.min(X86_OMP_MAX_THREADS);
}
}
if let Ok(val) = std::env::var("OMP_DYNAMIC") {
icv.dyn_var = val == "TRUE" || val == "true" || val == "1";
}
if let Ok(val) = std::env::var("OMP_NESTED") {
icv.nest_var = val == "TRUE" || val == "true" || val == "1";
}
if let Ok(val) = std::env::var("OMP_SCHEDULE") {
let parts: Vec<&str> = val.split(',').collect();
if let Some(kind_str) = parts.first() {
icv.run_sched_var = match *kind_str {
"static" => 1,
"dynamic" => 2,
"guided" => 3,
"auto" => 4,
_ => 1,
};
}
if parts.len() > 1 {
if let Ok(chunk) = parts[1].trim().parse::<i64>() {
icv.run_sched_chunk = chunk;
}
}
}
if let Ok(val) = std::env::var("OMP_THREAD_LIMIT") {
if let Ok(n) = val.parse::<u32>() {
icv.thread_limit_var = n;
}
}
if let Ok(val) = std::env::var("OMP_MAX_ACTIVE_LEVELS") {
if let Ok(n) = val.parse::<u32>() {
icv.max_active_levels_var = n;
}
}
if let Ok(val) = std::env::var("OMP_CANCELLATION") {
icv.cancellation_var = val == "TRUE" || val == "true" || val == "1";
}
icv
}
pub fn inherit(&self) -> Self {
let mut child = self.clone();
child.nthreads_var = self.nthreads_var;
child.dyn_var = self.dyn_var;
child.nest_var = self.nest_var;
child
}
}
impl Default for X86ICVStorage {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug)]
pub struct X86OpenMPFull {
pub target_triple: String,
pub simd_level: X86SimdLevel,
pub omp_version: u32,
pub directive_count: u64,
pub codegen_count: u64,
pub simd_count: u64,
pub offload_count: u64,
pub icv: X86ICVStorage,
pub runtime: OpenMPRuntime,
pub offload_engine: Option<TargetOffloadEngine>,
pub parallel_contexts: Vec<X86OmpDirective>,
pub declare_target_fns: Vec<String>,
pub declare_simd_fns: Vec<(String, X86SimdClause)>,
pub reduction_registry: Vec<X86ReductionOp>,
pub critical_names: HashSet<String>,
pub x86_optimizations: bool,
pub verbose: bool,
}
impl X86OpenMPFull {
pub fn new(target_triple: &str) -> Self {
let simd_level = Self::detect_simd_level_for(target_triple);
let icv = X86ICVStorage::from_environment();
Self {
target_triple: target_triple.to_string(),
simd_level,
omp_version: OMP_VERSION,
directive_count: 0,
codegen_count: 0,
simd_count: 0,
offload_count: 0,
icv,
runtime: OpenMPRuntime::default(),
offload_engine: None,
parallel_contexts: Vec::new(),
declare_target_fns: Vec::new(),
declare_simd_fns: Vec::new(),
reduction_registry: Vec::new(),
critical_names: HashSet::new(),
x86_optimizations: true,
verbose: false,
}
}
pub fn with_simd_level(mut self, level: X86SimdLevel) -> Self {
self.simd_level = level;
self
}
pub fn new_x86_64() -> Self {
Self::new("x86_64-unknown-linux-gnu").with_simd_level(X86SimdLevel::AVX2)
}
pub fn new_x86_64_sse2() -> Self {
Self::new("x86_64-unknown-linux-gnu").with_simd_level(X86SimdLevel::SSE2)
}
pub fn new_x86_64_avx512() -> Self {
Self::new("x86_64-unknown-linux-gnu").with_simd_level(X86SimdLevel::AVX512F)
}
pub fn new_i386() -> Self {
Self::new("i386-unknown-linux-gnu").with_simd_level(X86SimdLevel::SSE2)
}
fn detect_simd_level_for(triple: &str) -> X86SimdLevel {
if triple.contains("x86_64") || triple.contains("amd64") {
X86SimdLevel::AVX2
} else if triple.contains("i386")
|| triple.contains("i486")
|| triple.contains("i586")
|| triple.contains("i686")
{
X86SimdLevel::SSE2
} else {
X86SimdLevel::None
}
}
pub fn set_verbose(&mut self, verbose: bool) {
self.verbose = verbose;
}
pub fn disable_x86_optimizations(&mut self) {
self.x86_optimizations = false;
}
pub fn vector_width_bits(&self) -> u32 {
self.simd_level.vector_width_bits()
}
pub fn f32_vector_len(&self) -> u32 {
self.simd_level.f32_element_count()
}
pub fn f64_vector_len(&self) -> u32 {
self.simd_level.f64_element_count()
}
pub fn i32_vector_len(&self) -> u32 {
self.simd_level.i32_element_count()
}
}
impl Default for X86OpenMPFull {
fn default() -> Self {
Self::new_x86_64()
}
}
pub struct X86OmpDirectiveParser {
pub recognized_count: u64,
pub parse_errors: Vec<String>,
pub verbose: bool,
}
impl X86OmpDirectiveParser {
pub fn new() -> Self {
Self {
recognized_count: 0,
parse_errors: Vec::new(),
verbose: false,
}
}
pub fn parse_directive(&mut self, pragma_text: &str) -> Result<X86OmpDirective, String> {
let text = pragma_text.trim();
if text.is_empty() {
return Err("empty pragma".into());
}
let tokens = self.tokenize_pragma(text);
if tokens.is_empty() {
return Err("no tokens".into());
}
let kind = self.identify_directive(&tokens)?;
let mut directive = X86OmpDirective::new(kind);
let mut i = 0;
while i < tokens.len() {
let token = &tokens[i];
match token.as_str() {
"num_threads" => {
i += 1;
if i < tokens.len() {
directive.num_threads = Some(tokens[i].clone());
}
}
"if" => {
i += 1;
if i < tokens.len() {
directive.if_clause = Some(tokens[i].clone());
}
}
"final" => {
i += 1;
if i < tokens.len() {
directive.final_clause = Some(tokens[i].clone());
}
}
"priority" => {
i += 1;
if i < tokens.len() {
directive.priority_clause = Some(tokens[i].clone());
}
}
"default" => {
i += 1;
if i < tokens.len() {
let attr = tokens[i].trim_matches('(').trim_matches(')');
if let Some(d) = X86DefaultAttr::from_name(attr) {
directive.default_attr = d;
}
}
}
"private" => {
i += 1;
if i < tokens.len() {
let vars = self.parse_var_list(&tokens[i]);
directive.private_vars.extend(vars);
}
}
"firstprivate" => {
i += 1;
if i < tokens.len() {
let vars = self.parse_var_list(&tokens[i]);
directive.firstprivate_vars.extend(vars);
}
}
"lastprivate" => {
i += 1;
if i < tokens.len() {
let vars = self.parse_var_list(&tokens[i]);
directive.lastprivate_vars.extend(vars);
}
}
"shared" => {
i += 1;
if i < tokens.len() {
let vars = self.parse_var_list(&tokens[i]);
directive.shared_vars.extend(vars);
}
}
"copyin" => {
i += 1;
if i < tokens.len() {
let vars = self.parse_var_list(&tokens[i]);
directive.copyin_vars.extend(vars);
}
}
"copyprivate" => {
i += 1;
if i < tokens.len() {
let vars = self.parse_var_list(&tokens[i]);
directive.copyprivate_vars.extend(vars);
}
}
"reduction" => {
i += 1;
if i < tokens.len() {
let parsed = self.parse_reduction_clause(&tokens[i]);
directive.reductions.extend(parsed);
}
}
"schedule" => {
i += 1;
if i < tokens.len() {
let (sched, chunk) = self.parse_schedule_clause(&tokens[i]);
directive.schedule = sched;
directive.chunk_size = chunk;
}
}
"collapse" => {
i += 1;
if i < tokens.len() {
if let Ok(n) = tokens[i].trim_matches('(').trim_matches(')').parse::<u32>()
{
directive.collapse = Some(n);
}
}
}
"ordered" => {
i += 1;
if i < tokens.len() {
if let Ok(n) = tokens[i].trim_matches('(').trim_matches(')').parse::<u32>()
{
directive.ordered = X86OrderedKind::OrderedN(n);
} else {
directive.ordered = X86OrderedKind::Ordered;
}
} else {
directive.ordered = X86OrderedKind::Ordered;
}
}
"nowait" => {
directive.nowait = true;
}
"untied" => {
directive.untied = true;
}
"mergeable" => {
directive.mergeable = true;
}
"proc_bind" => {
i += 1;
if i < tokens.len() {
let name = tokens[i].trim_matches('(').trim_matches(')');
directive.proc_bind = X86ProcBind::from_name(name);
}
}
"map" => {
i += 1;
if i < tokens.len() {
if let Some(map) = self.parse_map_clause(&tokens[i]) {
directive.map_clauses.push(map);
}
}
}
"depend" => {
i += 1;
if i < tokens.len() {
if let Some((dt, var)) = self.parse_depend_clause(&tokens[i]) {
directive.depend_clauses.push((dt, var));
}
}
}
"device" => {
i += 1;
if i < tokens.len() {
directive.device_id = Some(tokens[i].clone());
}
}
"safelen" => {
i += 1;
if i < tokens.len() {
if let Ok(n) = tokens[i].trim_matches('(').trim_matches(')').parse::<u32>()
{
directive.simd_clause.safelen = Some(n);
}
}
}
"simdlen" => {
i += 1;
if i < tokens.len() {
if let Ok(n) = tokens[i].trim_matches('(').trim_matches(')').parse::<u32>()
{
directive.simd_clause.simdlen = Some(n);
}
}
}
"aligned" => {
i += 1;
if i < tokens.len() {
let inner = tokens[i].trim_matches('(').trim_matches(')');
for part in inner.split(',') {
let part = part.trim();
if let Some(colon_idx) = part.find(':') {
let var = part[..colon_idx].trim().to_string();
let align_str = part[colon_idx + 1..].trim();
if let Ok(a) = align_str.parse::<u32>() {
directive.simd_clause.aligned_vars.push(var);
directive.simd_clause.aligned_alignments.push(a);
}
} else {
directive.simd_clause.aligned_vars.push(part.to_string());
directive.simd_clause.aligned_alignments.push(16);
}
}
}
}
"linear" => {
i += 1;
if i < tokens.len() {
let inner = tokens[i].trim_matches('(').trim_matches(')');
for part in inner.split(',') {
let part = part.trim();
if let Some(colon_idx) = part.find(':') {
let var = part[..colon_idx].trim().to_string();
let step_str = part[colon_idx + 1..].trim();
if let Ok(step) = step_str.parse::<i32>() {
directive.simd_clause.linear_vars.push((var, step));
}
} else {
directive
.simd_clause
.linear_vars
.push((part.to_string(), 1));
}
}
}
}
"uniform" => {
i += 1;
if i < tokens.len() {
let vars = self.parse_var_list(&tokens[i]);
directive.simd_clause.uniform_vars.extend(vars);
}
}
"grainsize" => {
i += 1;
if i < tokens.len() {
if let Ok(n) = tokens[i].trim_matches('(').trim_matches(')').parse::<u64>()
{
directive.grainsize = Some(n);
}
}
}
"num_tasks" => {
i += 1;
if i < tokens.len() {
if let Ok(n) = tokens[i].trim_matches('(').trim_matches(')').parse::<u32>()
{
directive.num_tasks = Some(n);
}
}
}
"tile" => {
i += 1;
if i < tokens.len() {
let inner = tokens[i].trim_matches('(').trim_matches(')');
let sizes: Vec<u32> = inner
.split(',')
.filter_map(|s| s.trim().parse::<u32>().ok())
.collect();
if !sizes.is_empty() {
directive.loop_transform.tile_sizes = Some(sizes);
}
}
}
"unroll" => {
i += 1;
if i < tokens.len() && tokens[i] != "full" {
if let Ok(n) = tokens[i].trim_matches('(').trim_matches(')').parse::<u32>()
{
directive.loop_transform.unroll_factor = Some(n);
}
} else if i < tokens.len() && tokens[i] == "full" {
directive.loop_transform.unroll_full = true;
} else {
directive.loop_transform.unroll_full = true;
i -= 1;
}
}
"filter" => {
i += 1;
if i < tokens.len() {
directive.filter = Some(tokens[i].clone());
}
}
"allocate" => {
i += 1;
if i < tokens.len() {
let inner = tokens[i].trim_matches('(').trim_matches(')');
for part in inner.split(',') {
let part = part.trim();
if let Some(colon_idx) = part.find(':') {
let var = part[..colon_idx].trim().to_string();
let alloc = part[colon_idx + 1..].trim().to_string();
directive.allocate_vars.push((var, alloc));
}
}
}
}
"affinity" => {
i += 1;
if i < tokens.len() {
directive.affinity = Some(tokens[i].clone());
}
}
"detach" => {
i += 1;
if i < tokens.len() {
directive.detach = Some(tokens[i].clone());
}
}
"nontemporal" => {
i += 1;
if i < tokens.len() {
let vars = self.parse_var_list(&tokens[i]);
directive.simd_clause.nontemporal_vars.extend(vars);
}
}
_ => {
if self.verbose {
self.parse_errors.push(format!("unknown clause: {}", token));
}
}
}
i += 1;
}
self.recognized_count += 1;
Ok(directive)
}
fn tokenize_pragma(&self, text: &str) -> Vec<String> {
let mut tokens = Vec::new();
let mut current = String::new();
let mut paren_depth: i32 = 0;
let chars: Vec<char> = text.chars().collect();
let mut i = 0;
while i < chars.len() {
let c = chars[i];
match c {
' ' | '\t' if paren_depth == 0 => {
if !current.is_empty() {
tokens.push(current.clone());
current.clear();
}
}
'(' => {
paren_depth += 1;
current.push(c);
}
')' => {
paren_depth = paren_depth.saturating_sub(1);
current.push(c);
}
_ => {
current.push(c);
}
}
i += 1;
}
if !current.is_empty() {
tokens.push(current);
}
tokens
}
fn identify_directive(&self, tokens: &[String]) -> Result<X86OmpDirectiveKind, String> {
if tokens.is_empty() {
return Err("empty directive".into());
}
let combined = tokens.join(" ");
if let Some(kind) = X86OmpDirectiveKind::from_pragma_name(&combined) {
return Ok(kind);
}
if let Some(kind) = X86OmpDirectiveKind::from_pragma_name(&tokens[0]) {
return Ok(kind);
}
if tokens.len() >= 2 {
let first_two = format!("{} {}", tokens[0], tokens[1]);
if let Some(kind) = X86OmpDirectiveKind::from_pragma_name(&first_two) {
return Ok(kind);
}
}
if tokens.len() >= 3 {
let first_three = format!("{} {} {}", tokens[0], tokens[1], tokens[2]);
if let Some(kind) = X86OmpDirectiveKind::from_pragma_name(&first_three) {
return Ok(kind);
}
}
if tokens.len() >= 4 {
let first_four = format!("{} {} {} {}", tokens[0], tokens[1], tokens[2], tokens[3]);
if let Some(kind) = X86OmpDirectiveKind::from_pragma_name(&first_four) {
return Ok(kind);
}
}
if tokens.len() >= 5 {
let first_five = format!(
"{} {} {} {} {}",
tokens[0], tokens[1], tokens[2], tokens[3], tokens[4]
);
if let Some(kind) = X86OmpDirectiveKind::from_pragma_name(&first_five) {
return Ok(kind);
}
}
Err(format!("unrecognized directive: {}", combined))
}
fn parse_var_list(&self, token: &str) -> Vec<String> {
let inner = token.trim_matches('(').trim_matches(')');
inner
.split(',')
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect()
}
fn parse_reduction_clause(&self, token: &str) -> Vec<(String, X86ReductionOp)> {
let inner = token.trim_matches('(').trim_matches(')');
if let Some(colon_idx) = inner.find(':') {
let op_str = inner[..colon_idx].trim();
let vars_str = &inner[colon_idx + 1..];
if let Some(op) = X86ReductionOp::from_name(op_str) {
return vars_str
.split(',')
.map(|s| (s.trim().to_string(), op))
.filter(|(v, _)| !v.is_empty())
.collect();
}
}
Vec::new()
}
fn parse_schedule_clause(&self, token: &str) -> (X86OmpScheduleKind, Option<u64>) {
let inner = token.trim_matches('(').trim_matches(')');
let parts: Vec<&str> = inner.split(',').map(|s| s.trim()).collect();
let kind = if let Some(first) = parts.first() {
X86OmpScheduleKind::from_name(first).unwrap_or_default()
} else {
X86OmpScheduleKind::default()
};
let chunk = parts.get(1).and_then(|s| s.parse::<u64>().ok());
(kind, chunk)
}
fn parse_map_clause(&self, token: &str) -> Option<X86MapClause> {
let inner = token.trim_matches('(').trim_matches(')');
if let Some(colon_idx) = inner.find(':') {
let type_part = inner[..colon_idx].trim();
let var_part = inner[colon_idx + 1..].trim();
let map_type = X86OmpMapType::from_name(type_part)?;
let mut map = X86MapClause::new(var_part, map_type);
if let Some(bracket_idx) = var_part.find('[') {
let var_name = &var_part[..bracket_idx];
let slice = &var_part[bracket_idx + 1..];
if let Some(end_idx) = slice.rfind(']') {
let slice_inner = &slice[..end_idx];
map.variable_name = var_name.to_string();
map.array_section = true;
if let Some(sep_idx) = slice_inner.find(':') {
map.array_start = Some(slice_inner[..sep_idx].trim().to_string());
map.array_length = Some(slice_inner[sep_idx + 1..].trim().to_string());
} else {
map.array_start = Some(slice_inner.to_string());
}
}
}
Some(map)
} else {
None
}
}
fn parse_depend_clause(&self, token: &str) -> Option<(X86DependType, String)> {
let inner = token.trim_matches('(').trim_matches(')');
if let Some(colon_idx) = inner.find(':') {
let dep_str = inner[..colon_idx].trim();
let var = inner[colon_idx + 1..].trim().to_string();
let dt = X86DependType::from_name(dep_str)?;
Some((dt, var))
} else {
None
}
}
pub fn recognized_count(&self) -> u64 {
self.recognized_count
}
pub fn has_errors(&self) -> bool {
!self.parse_errors.is_empty()
}
pub fn errors(&self) -> &[String] {
&self.parse_errors
}
}
impl Default for X86OmpDirectiveParser {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct X86OmpParallelRegion {
pub directive: X86OmpDirective,
pub outlined_fn_name: String,
pub captured_vars: Vec<X86CapturedVariable>,
pub num_threads_expr: Option<String>,
pub if_expr: Option<String>,
}
#[derive(Debug, Clone)]
pub struct X86CapturedVariable {
pub name: String,
pub sharing: X86DataSharingAttr,
pub var_type: String,
pub is_pointer: bool,
pub reduction_op: Option<X86ReductionOp>,
pub linear_step: Option<i64>,
}
#[derive(Debug, Clone)]
pub struct X86OmpLoopLowering {
pub directive: X86OmpDirective,
pub loop_var: String,
pub lower_bound: String,
pub upper_bound: String,
pub stride: String,
pub chunk_size: u64,
pub schedule: X86OmpScheduleKind,
pub ordered_clause: X86OrderedKind,
pub collapse_depth: u32,
pub simd_clause: X86SimdClause,
pub nowait: bool,
pub body_label: String,
pub epilogue_label: String,
}
#[derive(Debug, Clone)]
pub struct X86OmpTaskLowering {
pub directive: X86OmpDirective,
pub outlined_fn_name: String,
pub captured_vars: Vec<X86CapturedVariable>,
pub depend_vars: Vec<(X86DependType, String)>,
pub is_untied: bool,
pub is_mergeable: bool,
pub priority: Option<String>,
pub if_expr: Option<String>,
pub final_expr: Option<String>,
}
#[derive(Debug, Clone)]
pub struct X86OmpSimdLowering {
pub directive: X86OmpDirective,
pub loop_var: String,
pub lower_bound: String,
pub upper_bound: String,
pub stride: String,
pub simd_clause: X86SimdClause,
pub vector_width: u32,
pub element_type_width: u32,
pub masked: bool,
}
#[derive(Debug, Clone)]
pub struct X86OmpTargetLowering {
pub directive: X86OmpDirective,
pub outlined_fn_name: String,
pub captured_vars: Vec<X86CapturedVariable>,
pub map_clauses: Vec<X86MapClause>,
pub device_id: Option<String>,
pub is_target_data: bool,
pub device_ptr_vars: Vec<String>,
}
#[derive(Debug, Clone)]
pub struct X86OmpTaskLoopLowering {
pub directive: X86OmpDirective,
pub loop_var: String,
pub lower_bound: String,
pub upper_bound: String,
pub stride: String,
pub grainsize: Option<u64>,
pub num_tasks: Option<u32>,
pub outlined_fn_name: String,
pub captured_vars: Vec<X86CapturedVariable>,
pub priority: Option<String>,
}
#[derive(Debug, Clone)]
pub struct X86OmpReductionLowering {
pub reduction_op: X86ReductionOp,
pub target_var: String,
pub local_var: String,
pub var_type: String,
pub init_value: String,
pub combiner_fn: String,
pub finalizer_fn: String,
}
#[derive(Debug, Clone)]
pub struct X86OmpCriticalLowering {
pub name: Option<String>,
pub lock_name: String,
pub hint: Option<String>,
}
#[derive(Debug)]
pub struct X86OpenMPCodeGen {
pub context: X86OpenMPFull,
pub parallel_count: u64,
pub loop_count: u64,
pub simd_count: u64,
pub task_count: u64,
pub target_count: u64,
pub section_count: u64,
outlined_fn_counter: u64,
lock_counter: u64,
pub parallel_regions: Vec<X86OmpParallelRegion>,
pub loop_lowerings: Vec<X86OmpLoopLowering>,
pub task_lowerings: Vec<X86OmpTaskLowering>,
pub simd_lowerings: Vec<X86OmpSimdLowering>,
pub target_lowerings: Vec<X86OmpTargetLowering>,
pub taskloop_lowerings: Vec<X86OmpTaskLoopLowering>,
pub reduction_lowerings: Vec<X86OmpReductionLowering>,
pub critical_lowerings: Vec<X86OmpCriticalLowering>,
pub cancel_flags: Vec<String>,
pub verbose: bool,
}
impl X86OpenMPCodeGen {
pub fn new(context: X86OpenMPFull) -> Self {
Self {
context,
parallel_count: 0,
loop_count: 0,
simd_count: 0,
task_count: 0,
target_count: 0,
section_count: 0,
outlined_fn_counter: 0,
lock_counter: 0,
parallel_regions: Vec::new(),
loop_lowerings: Vec::new(),
task_lowerings: Vec::new(),
simd_lowerings: Vec::new(),
target_lowerings: Vec::new(),
taskloop_lowerings: Vec::new(),
reduction_lowerings: Vec::new(),
critical_lowerings: Vec::new(),
cancel_flags: Vec::new(),
verbose: false,
}
}
pub fn set_verbose(&mut self, v: bool) {
self.verbose = v;
self.context.verbose = v;
}
fn gen_outlined_name(&mut self, prefix: &str) -> String {
let id = self.outlined_fn_counter;
self.outlined_fn_counter += 1;
format!("__omp_{}_{}", prefix, id)
}
fn gen_lock_name(&mut self, prefix: &str) -> String {
let id = self.lock_counter;
self.lock_counter += 1;
format!("_omp_lock_{}_{}", prefix, id)
}
pub fn lower_parallel(&mut self, directive: &X86OmpDirective) -> X86OmpParallelRegion {
self.parallel_count += 1;
let outlined_fn = self.gen_outlined_name("parallel");
let captured_vars = self.collect_captured_variables(directive);
if self.verbose {
eprintln!(
"[X86OpenMPCodeGen] Lowering parallel region #{} -> {}",
self.parallel_count, outlined_fn
);
}
let region = X86OmpParallelRegion {
directive: directive.clone(),
outlined_fn_name: outlined_fn,
captured_vars,
num_threads_expr: directive.num_threads.clone(),
if_expr: directive.if_clause.clone(),
};
self.parallel_regions.push(region.clone());
region
}
pub fn lower_parallel_for(
&mut self,
directive: &X86OmpDirective,
) -> (X86OmpParallelRegion, X86OmpLoopLowering) {
let parallel_region = self.lower_parallel(directive);
let loop_lowering = self.lower_for_loop(directive);
(parallel_region, loop_lowering)
}
pub fn lower_parallel_sections(&mut self, directive: &X86OmpDirective) -> X86OmpParallelRegion {
self.lower_parallel(directive)
}
pub fn lower_parallel_for_simd(
&mut self,
directive: &X86OmpDirective,
) -> (X86OmpParallelRegion, X86OmpSimdLowering) {
let parallel_region = self.lower_parallel(directive);
let simd_lowering = self.lower_simd_loop(directive);
(parallel_region, simd_lowering)
}
pub fn lower_for_loop(&mut self, directive: &X86OmpDirective) -> X86OmpLoopLowering {
self.loop_count += 1;
let loop_var = "i".to_string();
let lower = "0".to_string();
let upper = "N".to_string();
let stride = "1".to_string();
let chunk = directive.chunk_size.unwrap_or(1);
let schedule = directive.schedule;
let ordered = if matches!(directive.ordered, X86OrderedKind::OrderedN(0))
|| matches!(directive.ordered, X86OrderedKind::Ordered)
{
directive.ordered
} else {
X86OrderedKind::None
};
let body_label = format!(".omp_for_body.{}", self.loop_count);
let epilogue_label = format!(".omp_for_epilogue.{}", self.loop_count);
if self.verbose {
eprintln!(
"[X86OpenMPCodeGen] Lowering for loop #{} with schedule {:?} chunk={}",
self.loop_count, schedule, chunk
);
}
let lowering = X86OmpLoopLowering {
directive: directive.clone(),
loop_var,
lower_bound: lower,
upper_bound: upper,
stride,
chunk_size: chunk,
schedule,
ordered_clause: ordered,
collapse_depth: directive.collapse.unwrap_or(1),
simd_clause: directive.simd_clause.clone(),
nowait: directive.nowait,
body_label,
epilogue_label,
};
self.loop_lowerings.push(lowering.clone());
lowering
}
pub fn generate_static_schedule(&self, loop_info: &X86OmpLoopLowering) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!(
"; Static schedule init: {}",
kmpc_x86::FOR_STATIC_INIT_4
));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i32 34, i32* %plastiter, i32* %plower, i32* %pupper, i32* %pstride, i32 1, i32 {})",
kmpc_x86::FOR_STATIC_INIT_4,
loop_info.chunk_size
));
ir.push(format!(
"; Loop body: {} to upper bound",
loop_info.body_label
));
ir.push(format!("; Loop epilogue: {}", loop_info.epilogue_label));
ir.push(format!("call void @{}()", kmpc_x86::FOR_STATIC_FINI));
ir
}
pub fn generate_dynamic_schedule(&self, loop_info: &X86OmpLoopLowering) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!(
"; Dynamic schedule init: {}",
kmpc_x86::DISPATCH_INIT_4
));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i32 35, i32 {}, i32 {}, i32 1, i32 {})",
kmpc_x86::DISPATCH_INIT_4,
loop_info.lower_bound,
loop_info.upper_bound,
loop_info.chunk_size
));
ir.push(format!(
"; Dynamic dispatch loop: {}",
kmpc_x86::DISPATCH_NEXT_4
));
ir.push(format!(
"%has_work = call i32 @{}(@ident_t* @.loc, i32 %gtid, i32* %plastiter, i32* %plower, i32* %pupper, i32* %pstride)",
kmpc_x86::DISPATCH_NEXT_4
));
ir.push("br i1 %has_work, label %loop_body, label %loop_exit".into());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::DISPATCH_FINI_4
));
ir
}
pub fn generate_guided_schedule(&self, loop_info: &X86OmpLoopLowering) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!(
"; Guided schedule init: {}",
kmpc_x86::DISPATCH_INIT_4
));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i32 36, i32 {}, i32 {}, i32 1, i32 {})",
kmpc_x86::DISPATCH_INIT_4,
loop_info.lower_bound,
loop_info.upper_bound,
loop_info.chunk_size.max(1)
));
ir.push(format!(
"%has_work_guided = call i32 @{}(@ident_t* @.loc, i32 %gtid, i32* %plastiter, i32* %plower, i32* %pupper, i32* %pstride)",
kmpc_x86::DISPATCH_NEXT_4
));
ir.push("br i1 %has_work_guided, label %loop_body, label %loop_exit".into());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::DISPATCH_FINI_4
));
ir
}
pub fn generate_auto_schedule(&self, loop_info: &X86OmpLoopLowering) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Auto schedule: delegate to runtime ICV run-sched-var".into());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i32 %run_sched, i32 {}, i32 {}, i32 1, i32 %run_sched_chunk)",
kmpc_x86::DISPATCH_INIT_4,
loop_info.lower_bound,
loop_info.upper_bound
));
ir.push(format!(
"%has_work_auto = call i32 @{}(@ident_t* @.loc, i32 %gtid, i32* %plastiter, i32* %plower, i32* %pupper, i32* %pstride)",
kmpc_x86::DISPATCH_NEXT_4
));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::DISPATCH_FINI_4
));
ir
}
pub fn generate_schedule_code(&self, loop_info: &X86OmpLoopLowering) -> Vec<String> {
match loop_info.schedule {
X86OmpScheduleKind::Static => self.generate_static_schedule(loop_info),
X86OmpScheduleKind::Dynamic => self.generate_dynamic_schedule(loop_info),
X86OmpScheduleKind::Guided => self.generate_guided_schedule(loop_info),
X86OmpScheduleKind::Auto | X86OmpScheduleKind::Runtime => {
self.generate_auto_schedule(loop_info)
}
_ => self.generate_static_schedule(loop_info),
}
}
pub fn lower_sections(
&mut self,
directive: &X86OmpDirective,
num_sections: u32,
) -> Vec<String> {
self.section_count += 1;
let mut ir = Vec::new();
ir.push(format!(
"; Sections region #{} with {} sections",
self.section_count, num_sections
));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::SECTIONS_INIT
));
for i in 0..num_sections {
ir.push(format!("; Section {}: check __kmpc_next_section", i));
ir.push(format!(
"%has_section{} = call i32 @{}(@ident_t* @.loc, i32 %gtid)",
i,
kmpc_x86::NEXT_SECTION
));
ir.push(format!(
"br i1 %has_section{}, label %section{}_body, label %after_section{}",
i, i, i
));
ir.push(format!("section{}_body:", i));
}
if !directive.nowait {
ir.push(format!("call void @{}()", kmpc_x86::BARRIER));
}
ir
}
pub fn lower_section(&self, section_index: u32) -> Vec<String> {
vec![
format!("; Section #{} body", section_index),
format!("br label %after_section{}", section_index),
format!("after_section{}:", section_index),
]
}
pub fn lower_single(&self, directive: &X86OmpDirective) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Single region: one thread executes".into());
ir.push(format!(
"%is_single = call i32 @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::SINGLE_BEGIN
));
ir.push("br i1 %is_single, label %single_body, label %single_end".into());
ir.push("single_body:".into());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::SINGLE_END
));
ir.push("single_end:".into());
if directive.nowait {
ir.push("; nowait: no barrier after single".into());
} else {
ir.push(format!(
"call void @{}() ; implicit barrier",
kmpc_x86::BARRIER
));
}
ir
}
pub fn lower_master(&self) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Master region: master thread only".into());
ir.push(format!(
"%is_master = call i32 @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::MASTER_BEGIN
));
ir.push("br i1 %is_master, label %master_body, label %master_end".into());
ir.push("master_body:".into());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::MASTER_END
));
ir.push("master_end:".into());
ir
}
pub fn lower_critical(&mut self, directive: &X86OmpDirective) -> X86OmpCriticalLowering {
let name = directive.critical_name.clone();
let lock_name = name.clone().unwrap_or_else(|| self.gen_lock_name("crit"));
if let Some(ref n) = name {
self.context.critical_names.insert(n.clone());
}
let lowering = X86OmpCriticalLowering {
name: name.clone(),
lock_name,
hint: None,
};
self.critical_lowerings.push(lowering.clone());
lowering
}
pub fn generate_critical_code(&self, lowering: &X86OmpCriticalLowering) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!(
"; Critical region '{}'",
lowering.name.as_deref().unwrap_or("(anonymous)")
));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i8* @{})",
kmpc_x86::CRITICAL,
lowering.lock_name
));
ir.push("critical_body:".into());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i8* @{})",
kmpc_x86::END_CRITICAL,
lowering.lock_name
));
ir
}
pub fn lower_atomic(
&self,
directive: &X86OmpDirective,
atomic_kind: X86OmpAtomicKind,
) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!("; Atomic operation: {:?}", atomic_kind));
match atomic_kind {
X86OmpAtomicKind::Read => {
ir.push(format!(
"%atomic_val = call i32 @{}(@ident_t* @.loc, i32 %gtid, i32* %ptr)",
kmpc_x86::ATOMIC_RD_4
));
}
X86OmpAtomicKind::Write => {
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i32* %ptr, i32 %val)",
kmpc_x86::ATOMIC_WR_4
));
}
X86OmpAtomicKind::Update => {
ir.push("; Atomic update using X86 LOCK CMPXCHG or similar".into());
ir.push("%atomic_result = atomicrmw add i32* %ptr, i32 %val seq_cst".into());
}
X86OmpAtomicKind::Capture => {
ir.push("; Atomic capture: old_value = atomicrmw; new_val = old_val op val".into());
ir.push("%old_val = atomicrmw add i32* %ptr, i32 %val seq_cst".into());
ir.push("%new_val = add i32 %old_val, %val".into());
}
X86OmpAtomicKind::Compare => {
ir.push(
"%cmp_result = cmpxchg i32* %ptr, i32 %expected, i32 %desired seq_cst seq_cst"
.into(),
);
}
X86OmpAtomicKind::WeakCompare => {
ir.push("%weak_cmp = cmpxchg weak i32* %ptr, i32 %expected, i32 %desired seq_cst seq_cst".into());
}
X86OmpAtomicKind::Hint => {
ir.push("; Atomic with hint: use X86 LOCK prefix with hint".into());
ir.push("%atomic_hint = atomicrmw xchg i32* %ptr, i32 %val seq_cst".into());
}
}
ir
}
pub fn generate_atomic_float(&self, is_read: bool, is_double: bool) -> Vec<String> {
let mut ir = Vec::new();
if is_read {
if is_double {
ir.push(format!(
"%f64_val = call double @{}(@ident_t* @.loc, i32 %gtid, double* %ptr)",
kmpc_x86::ATOMIC_DOUBLE_RD
));
} else {
ir.push(format!(
"%f32_val = call float @{}(@ident_t* @.loc, i32 %gtid, float* %ptr)",
kmpc_x86::ATOMIC_FLOAT_RD
));
}
} else {
if is_double {
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, double* %ptr, double %val)",
kmpc_x86::ATOMIC_DOUBLE_WR
));
} else {
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, float* %ptr, float %val)",
kmpc_x86::ATOMIC_FLOAT_WR
));
}
}
ir
}
pub fn lower_barrier(&self) -> Vec<String> {
vec![
"; Barrier: all threads synchronize".to_string(),
format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::BARRIER
),
]
}
pub fn lower_task(&mut self, directive: &X86OmpDirective) -> X86OmpTaskLowering {
self.task_count += 1;
let outlined_fn = self.gen_outlined_name("task");
let captured_vars = self.collect_captured_variables(directive);
if self.verbose {
eprintln!(
"[X86OpenMPCodeGen] Lowering task #{} -> {}",
self.task_count, outlined_fn
);
}
let lowering = X86OmpTaskLowering {
directive: directive.clone(),
outlined_fn_name: outlined_fn,
captured_vars,
depend_vars: Vec::new(),
is_untied: directive.untied,
is_mergeable: directive.mergeable,
priority: directive.priority_clause.clone(),
if_expr: directive.if_clause.clone(),
final_expr: directive.final_clause.clone(),
};
self.task_lowerings.push(lowering.clone());
lowering
}
pub fn generate_task_code(&self, task: &X86OmpTaskLowering) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!("; Task creation -> {}", task.outlined_fn_name));
ir.push(format!(
"%task_ptr = call i8* @{}(@ident_t* @.loc, i32 %gtid, i32 {}, i64 0, i64 0, i8* @{})",
kmpc_x86::TASK_ALLOC,
0,
task.outlined_fn_name
));
ir.push(format!(
"call i32 @{}(@ident_t* @.loc, i32 %gtid, i8* %task_ptr)",
kmpc_x86::TASK
));
if task.if_expr.as_deref() == Some("0") {
ir.push("; if(0) clause: execute task immediately".into());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i8* %task_ptr)",
kmpc_x86::TASK_BEGIN_IF0
));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i8* %task_ptr)",
kmpc_x86::TASK_COMPLETE_IF0
));
}
ir
}
pub fn generate_task_with_deps_code(&self, task: &X86OmpTaskLowering) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!(
"; Task with dependences -> {}",
task.outlined_fn_name
));
let ndeps = task.depend_vars.len();
ir.push(format!(
"call i32 @{}(@ident_t* @.loc, i32 %gtid, i8* %task_ptr, i32 {}, i8* %dep_vars, i8* %dep_types, i8* %no_aliases)",
kmpc_x86::TASK_WITH_DEPS, ndeps
));
ir
}
pub fn lower_taskwait(&self) -> Vec<String> {
vec![
"; Taskwait: wait for all child tasks".to_string(),
format!(
"call i32 @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::TASKWAIT
),
]
}
pub fn lower_taskyield(&self) -> Vec<String> {
vec![
"; Taskyield: yield execution to another task".to_string(),
format!(
"call i32 @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::TASKYIELD
),
]
}
pub fn lower_taskgroup_begin(&self) -> Vec<String> {
vec![
"; Taskgroup: begin group of tasks".to_string(),
format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::TASKGROUP
),
]
}
pub fn lower_taskgroup_end(&self) -> Vec<String> {
vec![
"; Taskgroup: end — wait for all tasks in group".to_string(),
format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::TASKGROUP_END
),
]
}
pub fn lower_taskloop(&mut self, directive: &X86OmpDirective) -> X86OmpTaskLoopLowering {
let outlined_fn = self.gen_outlined_name("taskloop_body");
let captured_vars = self.collect_captured_variables(directive);
let loop_var = "i_taskloop".to_string();
let lower = "0".to_string();
let upper = "N".to_string();
let stride = "1".to_string();
let lowering = X86OmpTaskLoopLowering {
directive: directive.clone(),
loop_var,
lower_bound: lower,
upper_bound: upper,
stride,
grainsize: directive.grainsize,
num_tasks: directive.num_tasks,
outlined_fn_name: outlined_fn,
captured_vars,
priority: directive.priority_clause.clone(),
};
self.taskloop_lowerings.push(lowering.clone());
lowering
}
pub fn generate_taskloop_code(&self, taskloop: &X86OmpTaskLoopLowering) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!("; Taskloop: {}", taskloop.outlined_fn_name));
let grainsize = taskloop.grainsize.unwrap_or(0);
let num_tasks = taskloop.num_tasks.unwrap_or(0);
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i8* %task_ptr, i32 1, i64 {}, i64 {}, i64 1, i64 {}, i64 {}, i8* @{})",
kmpc_x86::TASKLOOP,
taskloop.lower_bound,
taskloop.upper_bound,
grainsize,
num_tasks,
taskloop.outlined_fn_name
));
ir
}
pub fn lower_simd_loop(&mut self, directive: &X86OmpDirective) -> X86OmpSimdLowering {
self.simd_count += 1;
let simd_clause = directive.simd_clause.clone();
let vector_width =
simd_clause.effective_vector_width(self.context.simd_level.f32_element_count() as u32);
if self.verbose {
eprintln!(
"[X86OpenMPCodeGen] Lowering SIMD loop #{} with vector width {}",
self.simd_count, vector_width
);
}
let lowering = X86OmpSimdLowering {
directive: directive.clone(),
loop_var: "i".to_string(),
lower_bound: "0".to_string(),
upper_bound: "N".to_string(),
stride: "1".to_string(),
simd_clause,
vector_width,
element_type_width: 32, masked: false,
};
self.simd_lowerings.push(lowering.clone());
lowering
}
pub fn generate_simd_code(&self, simd: &X86OmpSimdLowering) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!(
"; SIMD loop vectorized with width {} using {:?}",
simd.vector_width, self.context.simd_level
));
let (vec_type, element_count) = self.get_simd_type_info(simd);
ir.push(format!(
"; Vector type: {} x {} elements",
vec_type, element_count
));
ir.push(format!(
"; SIMD loop with safelen={:?}",
simd.simd_clause.safelen
));
ir.push(format!("{}:", simd.directive.kind.name()));
ir.push(format!(
"%vec_ptr = getelementptr {}, {}* %base, i64 %iv",
vec_type, vec_type
));
ir.push(format!(
"%vec_load = load <{} x {}>, <{} x {}>* %vec_ptr, align {}",
element_count,
vec_type,
element_count,
vec_type,
self.context.simd_level.vector_width_bits() / 8
));
if !simd.simd_clause.aligned_vars.is_empty() {
ir.push("; Aligned access: using aligned load".into());
ir.push(format!(
"%aligned_load = load <{} x {}>, <{} x {}>* %vec_ptr, align {}",
element_count,
vec_type,
element_count,
vec_type,
self.context.simd_level.vector_width_bits() / 8
));
}
if !simd.simd_clause.nontemporal_vars.is_empty() {
ir.push("; Nontemporal store hint".into());
ir.push("store <x> %vec_result, <x>* %ptr, !nontemporal !{}".into());
}
ir
}
fn get_simd_type_info(&self, simd: &X86OmpSimdLowering) -> (&'static str, u32) {
match self.context.simd_level.vector_width_bits() {
512 => {
if simd.element_type_width <= 32 {
("float", 16)
} else {
("double", 8)
}
}
256 => {
if simd.element_type_width <= 32 {
("float", 8)
} else {
("double", 4)
}
}
128 => {
if simd.element_type_width <= 32 {
("float", 4)
} else {
("double", 2)
}
}
_ => ("float", 1),
}
}
pub fn lower_target(&mut self, directive: &X86OmpDirective) -> X86OmpTargetLowering {
self.target_count += 1;
let outlined_fn = self.gen_outlined_name("target");
let captured_vars = self.collect_captured_variables(directive);
if self.verbose {
eprintln!(
"[X86OpenMPCodeGen] Lowering target region #{} -> {}",
self.target_count, outlined_fn
);
}
let lowering = X86OmpTargetLowering {
directive: directive.clone(),
outlined_fn_name: outlined_fn,
captured_vars,
map_clauses: directive.map_clauses.clone(),
device_id: directive.device_id.clone(),
is_target_data: false,
device_ptr_vars: directive.is_device_ptr.clone(),
};
self.target_lowerings.push(lowering.clone());
lowering
}
pub fn generate_target_code(&self, target: &X86OmpTargetLowering) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!("; Target offload: {}", target.outlined_fn_name));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::TARGET_INIT
));
let device = target
.device_id
.as_ref()
.and_then(|s| s.parse().ok())
.unwrap_or(0);
ir.push(format!(
"%target_rc = call i32 @{}(@ident_t* @.loc, i32 {}, i8* @binary, i32 0, i8** %args_base, i8** %args, i64* %arg_sizes, i64* %arg_types, i32 0, i32 0)",
kmpc_x86::TARGET, device
));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::TARGET_DEINIT
));
ir
}
pub fn lower_target_data(&mut self, directive: &X86OmpDirective) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Target data region: manage device data environment".into());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 {}, i32 0, i8** %arg_sizes, i8** %arg_types, i8** %args)",
kmpc_x86::TARGET_DATA_BEGIN,
0
));
ir.push("target_data_body:".into());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 {}, i32 0, i8** %arg_sizes, i8** %arg_types, i8** %args)",
kmpc_x86::TARGET_DATA_END,
0
));
ir
}
pub fn lower_target_update(&self, directive: &X86OmpDirective) -> Vec<String> {
vec![
"; Target update: sync data between host and device".to_string(),
format!(
"call void @{}(@ident_t* @.loc, i32 {}, i32 0, i8** %arg_sizes, i8** %arg_types, i8** %args)",
kmpc_x86::TARGET_UPDATE,
0
),
]
}
pub fn lower_target_enter_data(&self, directive: &X86OmpDirective) -> Vec<String> {
vec![
"; Target enter data: map data to device only".to_string(),
format!(
"call void @{}(@ident_t* @.loc, i32 {}, i32 1, i8** %args)",
kmpc_x86::TARGET_DATA_BEGIN,
0
),
]
}
pub fn lower_target_exit_data(&self, directive: &X86OmpDirective) -> Vec<String> {
vec![
"; Target exit data: unmap data from device".to_string(),
format!(
"call void @{}(@ident_t* @.loc, i32 {}, i32 1, i8** %args)",
kmpc_x86::TARGET_DATA_END,
0
),
]
}
pub fn lower_teams(&mut self, directive: &X86OmpDirective) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Teams region: launch multiple teams".into());
let num_teams = directive.num_threads.as_deref().unwrap_or("0");
let thread_limit = directive.num_threads.as_deref().unwrap_or("0");
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i32 {}, i32 {})",
kmpc_x86::FORK_TEAMS,
num_teams,
thread_limit
));
ir
}
pub fn lower_distribute(&self, directive: &X86OmpDirective) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Distribute region: static distribution among teams".into());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i32 33, i32* %plastiter, i64* %plower, i64* %pupper, i64* %pstride, i64 1, i64 1)",
kmpc_x86::DISTRIBUTE_STATIC_INIT
));
ir.push("distribute_loop_body:".into());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::DISTRIBUTE_STATIC_FINI
));
ir
}
pub fn lower_ordered(&self) -> Vec<String> {
vec![
"; Ordered region: execute in iteration order".to_string(),
format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::ORDERED_BEGIN
),
"ordered_body:".to_string(),
format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::ORDERED_END
),
]
}
pub fn lower_ordered_depend(
&self,
depend_type: X86DependType,
dependencies: &[String],
) -> Vec<String> {
let mut ir = Vec::new();
match depend_type {
X86DependType::Source => {
ir.push("; Ordered depend(source): signal completion".into());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i64 0) ; source signal",
kmpc_x86::ORDERED_BEGIN
));
}
X86DependType::Sink => {
ir.push("; Ordered depend(sink): wait for prior iteration".into());
for dep in dependencies {
ir.push(format!("; Waiting for sink depend on {}", dep));
}
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i64 1) ; sink wait",
kmpc_x86::ORDERED_BEGIN
));
}
_ => {
ir.push("; Unsupported depend type for ordered".into());
}
}
ir
}
pub fn lower_cancel(&self, directive: &X86OmpDirective) -> Vec<String> {
let cancel_kind = if directive.kind == X86OmpDirectiveKind::Parallel {
1u32 } else if matches!(
directive.kind,
X86OmpDirectiveKind::For | X86OmpDirectiveKind::ParallelFor
) {
2u32 } else if directive.kind == X86OmpDirectiveKind::Sections
|| directive.kind == X86OmpDirectiveKind::ParallelSections
{
4u32 } else if directive.kind == X86OmpDirectiveKind::TaskGroup {
8u32 } else {
0u32
};
vec![
format!("; Cancel: {}", directive.kind.name()),
format!(
"%cancel_rc = call i32 @{}(@ident_t* @.loc, i32 %gtid, i32 {})",
kmpc_x86::CANCEL,
cancel_kind
),
]
}
pub fn lower_cancellation_point(&self, directive: &X86OmpDirective) -> Vec<String> {
let cancel_kind = if directive.kind == X86OmpDirectiveKind::Parallel {
1u32
} else {
2u32
};
vec![
format!("; Cancellation point: {}", directive.kind.name()),
format!(
"%cancel_pt_rc = call i32 @{}(@ident_t* @.loc, i32 %gtid, i32 {})",
kmpc_x86::CANCELLATION_POINT,
cancel_kind
),
]
}
pub fn lower_threadprivate(&self, vars: &[String]) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Threadprivate: thread-local storage for variables".into());
for var in vars {
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i8* @{})",
kmpc_x86::THREADPRIVATE_REGISTER,
var
));
}
ir
}
pub fn lower_copyin(&self, vars: &[String]) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Copyin: master thread copies values to all threads".into());
for var in vars {
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i8* @{})",
kmpc_x86::THREADPRIVATE_CACHED,
var
));
}
ir
}
pub fn lower_copyprivate(&self, vars: &[String]) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Copyprivate: broadcast values after single region".into());
for var in vars {
ir.push(format!(
"; Copyprivate for {}: broadcast to all threads",
var
));
}
ir.push(format!(
"call void @{}() ; sync before copyprivate",
kmpc_x86::BARRIER
));
ir
}
pub fn lower_reduction(
&mut self,
op: X86ReductionOp,
var: &str,
var_type: &str,
) -> X86OmpReductionLowering {
let local_var = format!("_{}_omp_local", var);
let combiner_fn = format!("__omp_reduction_combiner_{}_{}", var, op.name());
let finalizer_fn = format!("__omp_reduction_finalizer_{}_{}", var, op.name());
let init_value = if var_type.contains("float") || var_type.contains("double") {
if matches!(op, X86ReductionOp::Add | X86ReductionOp::Sub) {
"0.0".to_string()
} else if matches!(op, X86ReductionOp::Mul) {
"1.0".to_string()
} else {
"0.0".to_string()
}
} else {
if matches!(op, X86ReductionOp::Add | X86ReductionOp::Sub) {
"0".to_string()
} else if matches!(op, X86ReductionOp::Mul) {
"1".to_string()
} else if matches!(op, X86ReductionOp::And | X86ReductionOp::BAnd) {
"-1".to_string()
} else if matches!(op, X86ReductionOp::Or | X86ReductionOp::BOr) {
"0".to_string()
} else if matches!(op, X86ReductionOp::LogicalAnd) {
"true".to_string()
} else if matches!(op, X86ReductionOp::LogicalOr) {
"false".to_string()
} else {
"0".to_string()
}
};
let lowering = X86OmpReductionLowering {
reduction_op: op,
target_var: var.to_string(),
local_var,
var_type: var_type.to_string(),
init_value,
combiner_fn,
finalizer_fn,
};
self.reduction_lowerings.push(lowering.clone());
lowering
}
pub fn generate_reduction_init_code(&self, red: &X86OmpReductionLowering) -> Vec<String> {
vec![
format!(
"; Reduction init: {} variable {} as {}",
red.reduction_op.name(),
red.target_var,
red.reduction_op.name()
),
format!("%{} = alloca {}", red.local_var, red.var_type),
format!(
"store {} {}, {}* %{}",
red.var_type, red.init_value, red.var_type, red.local_var
),
]
}
pub fn generate_reduction_combiner_code(&self, red: &X86OmpReductionLowering) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!(
"; Reduction combiner: {}_{}",
red.target_var,
red.reduction_op.name()
));
let op_ir = match red.reduction_op {
X86ReductionOp::Add => "fadd",
X86ReductionOp::Mul => "fmul",
X86ReductionOp::Min => {
ir.push("%cmp_min = fcmp olt %a, %b".into());
ir.push("select i1 %cmp_min, %a, %b".into());
return ir;
}
X86ReductionOp::Max => {
ir.push("%cmp_max = fcmp ogt %a, %b".into());
ir.push("select i1 %cmp_max, %a, %b".into());
return ir;
}
X86ReductionOp::And | X86ReductionOp::BAnd => "and",
X86ReductionOp::Or | X86ReductionOp::BOr => "or",
X86ReductionOp::Xor => "xor",
X86ReductionOp::LogicalAnd => {
ir.push("%conv_a = trunc %a to i1".into());
ir.push("%conv_b = trunc %b to i1".into());
ir.push("%res_logand = and i1 %conv_a, %conv_b".into());
ir.push("%res_zext = zext i1 %res_logand to i32".into());
return ir;
}
X86ReductionOp::LogicalOr => {
ir.push("%conv_a = trunc %a to i1".into());
ir.push("%conv_b = trunc %b to i1".into());
ir.push("%res_logor = or i1 %conv_a, %conv_b".into());
ir.push("%res_zext = zext i1 %res_logor to i32".into());
return ir;
}
X86ReductionOp::Sub => "fsub",
X86ReductionOp::Custom => {
ir.push("; Custom reduction: use user-defined combiner".into());
return ir;
}
};
ir.push(format!(
"%combined = {} {} %local_val, %remote_val",
op_ir, red.var_type
));
ir
}
pub fn generate_reduction_finalize_code(&self, red: &X86OmpReductionLowering) -> Vec<String> {
vec![
format!(
"; Reduction finalize: {} -> {}",
red.local_var, red.target_var
),
format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, {}* %{}_final, {}* %{})",
kmpc_x86::REDUCE,
red.var_type,
red.target_var,
red.var_type,
red.local_var
),
format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, {}* %{}_final, {}* %{})",
kmpc_x86::END_REDUCE,
red.var_type,
red.target_var,
red.var_type,
red.local_var
),
]
}
pub fn lower_flush(&self, vars: Option<&[String]>) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Flush: memory consistency".into());
if let Some(vars) = vars {
for var in vars {
ir.push(format!("; Flush variable: {}", var));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i8* @{})",
kmpc_x86::FLUSH,
var
));
}
} else {
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i8* null) ; flush all",
kmpc_x86::FLUSH
));
}
ir
}
pub fn lower_allocate(&self, vars: &[String], allocator: Option<&str>) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Allocate clause: custom memory allocator for variables".into());
for var in vars {
ir.push(format!(
"; Allocate {} with allocator {}",
var,
allocator.unwrap_or("omp_default_mem_alloc")
));
ir.push(format!(
"call i8* @{}(@ident_t* @.loc, i32 %gtid, i64 sizeof({}))",
kmpc_x86::ALLOC_SHARED,
var
));
}
ir
}
fn collect_captured_variables(&self, directive: &X86OmpDirective) -> Vec<X86CapturedVariable> {
let mut vars = Vec::new();
for var in &directive.shared_vars {
vars.push(X86CapturedVariable {
name: var.clone(),
sharing: X86DataSharingAttr::Shared,
var_type: "i8*".to_string(),
is_pointer: true,
reduction_op: None,
linear_step: Some(0),
});
}
for var in &directive.private_vars {
vars.push(X86CapturedVariable {
name: var.clone(),
sharing: X86DataSharingAttr::Private,
var_type: "i32".to_string(),
is_pointer: false,
reduction_op: None,
linear_step: Some(0),
});
}
for var in &directive.firstprivate_vars {
vars.push(X86CapturedVariable {
name: var.clone(),
sharing: X86DataSharingAttr::FirstPrivate,
var_type: "i32".to_string(),
is_pointer: false,
reduction_op: None,
linear_step: Some(0),
});
}
for var in &directive.lastprivate_vars {
vars.push(X86CapturedVariable {
name: var.clone(),
sharing: X86DataSharingAttr::LastPrivate,
var_type: "i32".to_string(),
is_pointer: false,
reduction_op: None,
linear_step: Some(0),
});
}
for (var_name, op) in &directive.reductions {
vars.push(X86CapturedVariable {
name: var_name.clone(),
sharing: X86DataSharingAttr::Reduction(X86ReductionOp::Add),
var_type: "i32".to_string(),
is_pointer: false,
reduction_op: Some(*op),
linear_step: Some(0),
});
}
for (var_name, step) in &directive.linear_vars {
vars.push(X86CapturedVariable {
name: var_name.clone(),
sharing: X86DataSharingAttr::Linear(0),
var_type: "i32".to_string(),
is_pointer: false,
reduction_op: None,
linear_step: Some(*step as i64),
});
}
for var in &directive.copyin_vars {
vars.push(X86CapturedVariable {
name: var.clone(),
sharing: X86DataSharingAttr::CopyIn,
var_type: "i8*".to_string(),
is_pointer: true,
reduction_op: None,
linear_step: Some(0),
});
}
vars
}
pub fn generate_outlined_func_prologue(
&self,
fn_name: &str,
captured_vars: &[X86CapturedVariable],
) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!("define internal void @{}(", fn_name));
ir.push(" i32* noalias %.global_tid.,".into());
ir.push(" i32* noalias %.bound_tid.,".into());
for var in captured_vars {
let ptr_sigil = if var.is_pointer { "*" } else { "" };
ir.push(format!(
" {} {}{}%.captured.{}, ; {}",
if var.is_pointer { "i8*" } else { &var.var_type },
ptr_sigil,
if var.is_pointer { "" } else { "" },
var.name,
var.sharing.name()
));
}
ir.push(") {".into());
ir.push("entry:".into());
ir.push(" %gtid = load i32, i32* %.global_tid.".into());
ir
}
pub fn generate_outlined_func_epilogue(&self) -> Vec<String> {
vec![" ret void".to_string(), "}".to_string()]
}
pub fn generate_fork_call(
&self,
region: &X86OmpParallelRegion,
outlined_fn: &str,
) -> Vec<String> {
let num_args = region.captured_vars.len();
let mut ir = Vec::new();
ir.push(format!(
"; Fork call: {}, {} captured vars",
outlined_fn, num_args
));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 {}, void (i32*, i32*, ...)* @{})",
kmpc_x86::FORK_CALL,
num_args,
outlined_fn
));
if let Some(ref nt) = region.num_threads_expr {
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i32 {})",
kmpc_x86::PUSH_NUM_THREADS,
nt
));
}
if let Some(ref proc_bind) = region.directive.proc_bind {
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i32 {})",
kmpc_x86::PUSH_PROC_BIND,
proc_bind.name()
));
}
ir
}
pub fn lower_directive(&mut self, directive: &X86OmpDirective) -> Vec<String> {
match directive.kind {
X86OmpDirectiveKind::Parallel => {
let region = self.lower_parallel(directive);
self.generate_fork_call(®ion, ®ion.outlined_fn_name)
}
X86OmpDirectiveKind::For => {
let loop_info = self.lower_for_loop(directive);
self.generate_schedule_code(&loop_info)
}
X86OmpDirectiveKind::ParallelFor => {
let (region, loop_info) = self.lower_parallel_for(directive);
let mut ir = self.generate_fork_call(®ion, ®ion.outlined_fn_name);
ir.extend(self.generate_schedule_code(&loop_info));
ir
}
X86OmpDirectiveKind::Sections => self.lower_sections(directive, 1),
X86OmpDirectiveKind::Single => self.lower_single(directive),
X86OmpDirectiveKind::Master => self.lower_master(),
X86OmpDirectiveKind::Critical => {
let crit = self.lower_critical(directive);
self.generate_critical_code(&crit)
}
X86OmpDirectiveKind::Atomic => self.lower_atomic(directive, X86OmpAtomicKind::Update),
X86OmpDirectiveKind::Barrier => self.lower_barrier(),
X86OmpDirectiveKind::Task => {
let task = self.lower_task(directive);
self.generate_task_code(&task)
}
X86OmpDirectiveKind::TaskWait => self.lower_taskwait(),
X86OmpDirectiveKind::TaskYield => self.lower_taskyield(),
X86OmpDirectiveKind::TaskGroup => self.lower_taskgroup_begin(),
X86OmpDirectiveKind::TaskLoop => {
let taskloop = self.lower_taskloop(directive);
self.generate_taskloop_code(&taskloop)
}
X86OmpDirectiveKind::Simd => {
let simd = self.lower_simd_loop(directive);
self.generate_simd_code(&simd)
}
X86OmpDirectiveKind::Target => {
let target = self.lower_target(directive);
self.generate_target_code(&target)
}
X86OmpDirectiveKind::TargetData => self.lower_target_data(directive),
X86OmpDirectiveKind::TargetUpdate => self.lower_target_update(directive),
X86OmpDirectiveKind::TargetEnterData => self.lower_target_enter_data(directive),
X86OmpDirectiveKind::TargetExitData => self.lower_target_exit_data(directive),
X86OmpDirectiveKind::Teams => self.lower_teams(directive),
X86OmpDirectiveKind::Distribute => self.lower_distribute(directive),
X86OmpDirectiveKind::Ordered => self.lower_ordered(),
X86OmpDirectiveKind::Cancel => self.lower_cancel(directive),
X86OmpDirectiveKind::CancellationPoint => self.lower_cancellation_point(directive),
X86OmpDirectiveKind::Flush => self.lower_flush(None),
X86OmpDirectiveKind::ThreadPrivate => self.lower_threadprivate(&directive.private_vars),
_ => {
vec![format!(
"; Unsupported directive: {:?} (not yet implemented)",
directive.kind
)]
}
}
}
pub fn stats(&self) -> X86OpenMPCodeGenStats {
X86OpenMPCodeGenStats {
parallel_regions: self.parallel_count,
worksharing_loops: self.loop_count,
simd_loops: self.simd_count,
tasks: self.task_count,
target_regions: self.target_count,
sections: self.section_count,
total_outlined_functions: self.outlined_fn_counter,
}
}
}
impl Default for X86OpenMPCodeGen {
fn default() -> Self {
Self::new(X86OpenMPFull::default())
}
}
#[derive(Debug, Clone, Default)]
pub struct X86OpenMPCodeGenStats {
pub parallel_regions: u64,
pub worksharing_loops: u64,
pub simd_loops: u64,
pub tasks: u64,
pub target_regions: u64,
pub sections: u64,
pub total_outlined_functions: u64,
}
impl fmt::Display for X86OpenMPCodeGenStats {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"OpenMP CodeGen Stats: parallel={}, loops={}, simd={}, tasks={}, targets={}, sections={}, outlined_fns={}",
self.parallel_regions,
self.worksharing_loops,
self.simd_loops,
self.tasks,
self.target_regions,
self.sections,
self.total_outlined_functions
)
}
}
#[derive(Debug, Clone)]
pub enum X86SharedVarMapping {
SharedByRef,
PrivateByCopy,
FirstPrivateByCopy,
LastPrivateByCopy,
ReductionPrivate,
LinearPrivate,
}
#[derive(Debug, Clone)]
pub struct X86OmpIRLoopInfo {
pub induction_var: String,
pub lower_bound: i64,
pub upper_bound: i64,
pub stride: i64,
pub chunk_size: i64,
pub schedule_id: i32,
pub is_static: bool,
pub has_nowait: bool,
pub has_ordered: bool,
pub collapse_count: u32,
}
#[derive(Debug, Clone)]
pub struct X86OmpIRReduction {
pub kind: X86ReductionOp,
pub original_var: String,
pub local_var: String,
pub type_name: String,
pub init_expr: Option<String>,
pub combiner_label: String,
pub finalize_label: String,
}
#[derive(Debug, Clone)]
pub struct X86OmpIROutlinedFunction {
pub name: String,
pub shared_vars: Vec<(String, X86SharedVarMapping)>,
pub body_label: String,
pub has_barrier: bool,
pub has_reduction: bool,
pub reduction_ops: Vec<X86OmpIRReduction>,
pub nowait: bool,
}
#[derive(Debug, Clone)]
pub struct X86OmpIRSimd {
pub vector_width: u32,
pub element_type: String,
pub safelen_clause: Option<u32>,
pub simdlen_clause: Option<u32>,
pub aligned_ptrs: Vec<String>,
pub nontemporal_stores: Vec<String>,
pub linear_steps: Vec<(String, i64)>,
pub private_vars: Vec<String>,
pub reductions: Vec<X86OmpIRReduction>,
}
#[derive(Debug)]
pub struct X86OpenMPIRGen {
pub context: X86OpenMPFull,
pub module_name: String,
pub outlined_functions: Vec<X86OmpIROutlinedFunction>,
pub simd_regions: Vec<X86OmpIRSimd>,
pub loop_infos: Vec<X86OmpIRLoopInfo>,
pub reductions: Vec<X86OmpIRReduction>,
pub instruction_counter: u64,
pub label_counter: u64,
pub var_counter: u64,
pub verbose: bool,
}
impl X86OpenMPIRGen {
pub fn new(context: X86OpenMPFull) -> Self {
Self {
context,
module_name: "omp_x86_module".to_string(),
outlined_functions: Vec::new(),
simd_regions: Vec::new(),
loop_infos: Vec::new(),
reductions: Vec::new(),
instruction_counter: 0,
label_counter: 0,
var_counter: 0,
verbose: false,
}
}
pub fn set_verbose(&mut self, v: bool) {
self.verbose = v;
}
fn gen_label(&mut self, prefix: &str) -> String {
let id = self.label_counter;
self.label_counter += 1;
format!("__omp_{}_{}", prefix, id)
}
fn gen_var(&mut self, prefix: &str) -> String {
let id = self.var_counter;
self.var_counter += 1;
format!("%__omp_{}_{}", prefix, id)
}
pub fn generate_outlined_parallel_function(
&mut self,
fn_name: &str,
captured_vars: &[X86CapturedVariable],
has_reduction: bool,
reduction_ops: &[X86OmpIRReduction],
) -> X86OmpIROutlinedFunction {
let entry_label = self.gen_label("entry");
let mut shared_var_mappings = Vec::new();
for var in captured_vars {
let mapping = match var.sharing {
X86DataSharingAttr::Shared => X86SharedVarMapping::SharedByRef,
X86DataSharingAttr::Private => X86SharedVarMapping::PrivateByCopy,
X86DataSharingAttr::FirstPrivate => X86SharedVarMapping::FirstPrivateByCopy,
X86DataSharingAttr::LastPrivate => X86SharedVarMapping::LastPrivateByCopy,
X86DataSharingAttr::Reduction(_) => X86SharedVarMapping::ReductionPrivate,
X86DataSharingAttr::Linear(_) => X86SharedVarMapping::LinearPrivate,
X86DataSharingAttr::CopyIn => X86SharedVarMapping::SharedByRef,
X86DataSharingAttr::CopyPrivate => X86SharedVarMapping::PrivateByCopy,
};
shared_var_mappings.push((var.name.clone(), mapping));
}
let outlined = X86OmpIROutlinedFunction {
name: fn_name.to_string(),
shared_vars: shared_var_mappings,
body_label: entry_label,
has_barrier: true,
has_reduction,
reduction_ops: reduction_ops.to_vec(),
nowait: false,
};
self.outlined_functions.push(outlined.clone());
outlined
}
pub fn emit_outlined_fn_signature(&self, func: &X86OmpIROutlinedFunction) -> Vec<String> {
let mut ir = Vec::new();
let mut sig = format!("define internal void @{}(", func.name);
sig.push_str("i32* noalias %.global_tid., i32* noalias %.bound_tid.");
for (var_name, mapping) in &func.shared_vars {
match mapping {
X86SharedVarMapping::SharedByRef => {
sig.push_str(&format!(", i8* %.captured.{}", var_name));
}
X86SharedVarMapping::PrivateByCopy
| X86SharedVarMapping::FirstPrivateByCopy
| X86SharedVarMapping::LastPrivateByCopy => {
sig.push_str(&format!(", i64 %.captured.{}", var_name));
}
X86SharedVarMapping::ReductionPrivate => {
sig.push_str(&format!(", i8* %.captured.{}", var_name));
}
X86SharedVarMapping::LinearPrivate => {
sig.push_str(&format!(", i64 %.captured.{}", var_name));
}
}
}
sig.push_str(") {");
ir.push(sig);
ir.push("entry:".to_string());
ir.push(" %gtid = load i32, i32* %.global_tid.".to_string());
ir.push(" %btid = load i32, i32* %.bound_tid.".to_string());
for (var_name, mapping) in &func.shared_vars {
if matches!(mapping, X86SharedVarMapping::SharedByRef) {
ir.push(format!(
" %{}.addr = bitcast i8* %.captured.{} to i64*",
var_name, var_name
));
ir.push(format!(
" %{}.val = load i64, i64* %{}.addr",
var_name, var_name
));
}
}
ir
}
pub fn emit_outlined_fn_epilogue(&self, func: &X86OmpIROutlinedFunction) -> Vec<String> {
let mut ir = Vec::new();
if func.has_barrier && !func.nowait {
ir.push(" ; Implicit barrier at end of parallel region".to_string());
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::BARRIER
));
}
ir.push(" ret void".to_string());
ir.push("}".to_string());
ir
}
pub fn generate_shared_var_ref(&self, var_name: &str) -> String {
format!(
" %{}.ptr = bitcast i8* %.captured.{} to i64*",
var_name, var_name
)
}
pub fn generate_private_var_init(
&self,
var_name: &str,
var_type: &str,
init_val: &str,
) -> Vec<String> {
vec![
format!(" ; Private variable: {}", var_name),
format!(" %{}.private = alloca {}", var_name, var_type),
format!(
" store {} {}, {}* %{}.private",
var_type, init_val, var_type, var_name
),
]
}
pub fn generate_firstprivate_init(&self, var_name: &str, var_type: &str) -> Vec<String> {
vec![
format!(" ; Firstprivate: {}", var_name),
format!(
" %{}.fp.val = load {}, {}* %.captured.{}",
var_name, var_type, var_type, var_name
),
format!(" %{}.fp.ptr = alloca {}", var_name, var_type),
format!(
" store {} %{}.fp.val, {}* %{}.fp.ptr",
var_type, var_name, var_type, var_name
),
]
}
pub fn generate_lastprivate_finalize(&self, var_name: &str, var_type: &str) -> Vec<String> {
vec![
format!(" ; Lastprivate finalize: {}", var_name),
format!(
" %{}.lp.val = load {}, {}* %{}.local",
var_name, var_type, var_type, var_name
),
format!(
" %{}.lp.ptr = bitcast i8* %.captured.{} to {}*",
var_name, var_name, var_type
),
format!(
" store {} %{}.lp.val, {}* %{}.lp.ptr",
var_type, var_name, var_type, var_name
),
]
}
pub fn generate_linear_init(&self, var_name: &str, var_type: &str, step: i64) -> Vec<String> {
vec![
format!(" ; Linear variable: {} step={}", var_name, step),
format!(
" %{}.lin.val = load {}, {}* %.captured.{}",
var_name, var_type, var_type, var_name
),
format!(" %{}.lin.ptr = alloca {}", var_name, var_type),
format!(
" store {} %{}.lin.val, {}* %{}.lin.ptr",
var_type, var_name, var_type, var_name
),
]
}
pub fn generate_reduction_init(&mut self, red: &X86OmpIRReduction) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!(
" ; Reduction init: {} ({})",
red.original_var,
red.kind.name()
));
ir.push(format!(" %{} = alloca {}", red.local_var, red.type_name));
let neutral = red
.kind
.neutral_value()
.map(|v| v.to_string())
.unwrap_or_default();
ir.push(format!(
" store {} {}, {}* %{}",
red.type_name, neutral, red.type_name, red.local_var
));
ir
}
pub fn generate_reduction_combiner(&mut self, red: &X86OmpIRReduction) -> Vec<String> {
let mut ir = Vec::new();
let op_ir_str = match red.kind {
X86ReductionOp::Add => "add",
X86ReductionOp::Sub => "sub",
X86ReductionOp::Mul => "mul",
X86ReductionOp::BAnd | X86ReductionOp::And => "and",
X86ReductionOp::BOr | X86ReductionOp::Or => "or",
X86ReductionOp::Xor => "xor",
X86ReductionOp::Min => {
ir.push(format!(
" %cmp_{} = icmp slt i64 %local_{}, %iter_val",
red.original_var, red.original_var
));
ir.push(format!(
" %acc_{} = select i1 %cmp_{}, i64 %local_{}, i64 %iter_val",
red.original_var, red.original_var, red.original_var
));
return ir;
}
X86ReductionOp::Max => {
ir.push(format!(
" %cmp_{} = icmp sgt i64 %local_{}, %iter_val",
red.original_var, red.original_var
));
ir.push(format!(
" %acc_{} = select i1 %cmp_{}, i64 %local_{}, i64 %iter_val",
red.original_var, red.original_var, red.original_var
));
return ir;
}
X86ReductionOp::LogicalAnd => {
ir.push(format!(
" %acc_{} = and i1 %local_{}, %iter_val",
red.original_var, red.original_var
));
return ir;
}
X86ReductionOp::LogicalOr => {
ir.push(format!(
" %acc_{} = or i1 %local_{}, %iter_val",
red.original_var, red.original_var
));
return ir;
}
X86ReductionOp::Custom => {
ir.push(" ; Custom reduction".to_string());
return ir;
}
};
ir.push(format!(
" %acc_{} = {} {} %local_{}, %iter_val",
red.original_var, op_ir_str, red.type_name, red.original_var
));
ir
}
pub fn generate_reduction_finalize(&mut self, red: &X86OmpIRReduction) -> Vec<String> {
vec![
format!(
" ; Reduction finalize: {} ({})",
red.original_var,
red.kind.name()
),
format!(
" call i32 @{}(@ident_t* @.loc, i32 %gtid, i32 1, i64 8, i8* %{}, i8* @{}, i8* null)",
kmpc_x86::REDUCE,
red.local_var,
red.combiner_label
),
format!(
" call void @{}(@ident_t* @.loc, i32 %gtid, i8* @{})",
kmpc_x86::END_REDUCE,
red.local_var
),
]
}
pub fn generate_reduction_pipeline(
&mut self,
reductions: &[(String, X86ReductionOp)],
var_type: &str,
) -> Vec<String> {
let mut ir = Vec::new();
let mut ir_reductions = Vec::new();
for (var_name, op) in reductions {
let local_var = format!("%omp_red_{}", var_name);
let combiner_label = self.gen_label("red_combiner");
let finalize_label = self.gen_label("red_finalize");
let red = X86OmpIRReduction {
kind: *op,
original_var: var_name.clone(),
local_var: local_var.clone(),
type_name: var_type.to_string(),
init_expr: op.neutral_value().map(|v| v.to_string()),
combiner_label,
finalize_label,
};
ir.extend(self.generate_reduction_init(&red));
ir_reductions.push(red);
}
self.reductions.extend(ir_reductions);
ir
}
pub fn generate_static_loop_ir(&mut self, loop_info: &X86OmpIRLoopInfo) -> Vec<String> {
let mut ir = Vec::new();
let hdr = self.gen_label("omp_for_hdr");
let body = self.gen_label("omp_for_body");
let ext = self.gen_label("omp_for_exit");
ir.push(format!("; Static loop chunk={}", loop_info.chunk_size));
ir.push(" %plastiter = alloca i32".to_string());
ir.push(" %plower = alloca i32".to_string());
ir.push(" %pupper = alloca i32".to_string());
ir.push(" %pstride = alloca i32".to_string());
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid, i32 34, i32* %plastiter, i32* %plower, i32* %pupper, i32* %pstride, i32 1, i32 {})",
kmpc_x86::FOR_STATIC_INIT_4,
loop_info.chunk_size
));
ir.push(" %lower = load i32, i32* %plower".to_string());
ir.push(" %upper = load i32, i32* %pupper".to_string());
ir.push(format!(" br label %{}", hdr));
ir.push(format!("{}:", hdr));
ir.push(format!(
" %i = phi i32 [ %lower, %entry ], [ %next_i, %{} ]",
body
));
ir.push(" %loop_cond = icmp sle i32 %i, %upper".to_string());
ir.push(format!(
" br i1 %loop_cond, label %{}, label %{}",
body, ext
));
ir.push(format!("{}:", body));
ir.push(" %next_i = add i32 %i, 1".to_string());
ir.push(format!(" br label %{}", hdr));
ir.push(format!("{}:", ext));
ir.push(format!(" call void @{}()", kmpc_x86::FOR_STATIC_FINI));
if !loop_info.has_nowait {
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::BARRIER
));
}
ir
}
pub fn generate_dynamic_loop_ir(&mut self, loop_info: &X86OmpIRLoopInfo) -> Vec<String> {
let mut ir = Vec::new();
let chk = self.gen_label("omp_dispatch_chk");
let body = self.gen_label("omp_dispatch_body");
let ext = self.gen_label("omp_dispatch_ext");
ir.push(format!("; Dynamic loop chunk={}", loop_info.chunk_size));
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid, i32 35, i32 {}, i32 1, i32 {}, i32 {})",
kmpc_x86::DISPATCH_INIT_4,
loop_info.lower_bound,
loop_info.upper_bound,
loop_info.chunk_size
));
ir.push(format!(" br label %{}", chk));
ir.push(format!("{}:", chk));
ir.push(" %plastiter = alloca i32".to_string());
ir.push(" %plower_dyn = alloca i32".to_string());
ir.push(" %pupper_dyn = alloca i32".to_string());
ir.push(" %pstride_dyn = alloca i32".to_string());
ir.push(format!(
" %has_work = call i32 @{}(@ident_t* @.loc, i32 %gtid, i32* %plastiter, i32* %plower_dyn, i32* %pupper_dyn, i32* %pstride_dyn)",
kmpc_x86::DISPATCH_NEXT_4
));
ir.push(" %hw_bool = icmp ne i32 %has_work, 0".to_string());
ir.push(format!(" br i1 %hw_bool, label %{}, label %{}", body, ext));
ir.push(format!("{}:", body));
ir.push(" %lb_dyn = load i32, i32* %plower_dyn".to_string());
ir.push(" %ub_dyn = load i32, i32* %pupper_dyn".to_string());
ir.push(format!(" br label %{}", chk));
ir.push(format!("{}:", ext));
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::DISPATCH_FINI_4
));
if !loop_info.has_nowait {
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::BARRIER
));
}
ir
}
pub fn generate_guided_loop_ir(&mut self, loop_info: &X86OmpIRLoopInfo) -> Vec<String> {
let mut ir = Vec::new();
let chk = self.gen_label("omp_guided_chk");
let body = self.gen_label("omp_guided_body");
let ext = self.gen_label("omp_guided_ext");
ir.push(format!("; Guided loop min_chunk={}", loop_info.chunk_size));
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid, i32 36, i32 {}, i32 1, i32 {}, i32 {})",
kmpc_x86::DISPATCH_INIT_4,
loop_info.lower_bound,
loop_info.upper_bound,
loop_info.chunk_size.max(1)
));
ir.push(format!(" br label %{}", chk));
ir.push(format!("{}:", chk));
ir.push(" %plastiter_g = alloca i32".to_string());
ir.push(" %plower_g = alloca i32".to_string());
ir.push(" %pupper_g = alloca i32".to_string());
ir.push(" %pstride_g = alloca i32".to_string());
ir.push(format!(
" %gw_work = call i32 @{}(@ident_t* @.loc, i32 %gtid, i32* %plastiter_g, i32* %plower_g, i32* %pupper_g, i32* %pstride_g)",
kmpc_x86::DISPATCH_NEXT_4
));
ir.push(" %gw_bool = icmp ne i32 %gw_work, 0".to_string());
ir.push(format!(" br i1 %gw_bool, label %{}, label %{}", body, ext));
ir.push(format!("{}:", body));
ir.push(format!(" br label %{}", chk));
ir.push(format!("{}:", ext));
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::DISPATCH_FINI_4
));
if !loop_info.has_nowait {
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::BARRIER
));
}
ir
}
pub fn generate_loop_ir(&mut self, loop_info: &X86OmpIRLoopInfo) -> Vec<String> {
match loop_info.schedule_id {
1 => self.generate_static_loop_ir(loop_info),
2 => self.generate_dynamic_loop_ir(loop_info),
3 => self.generate_guided_loop_ir(loop_info),
4 | 5 => {
let mut info = loop_info.clone();
info.schedule_id = 2;
self.generate_dynamic_loop_ir(&info)
}
_ => self.generate_static_loop_ir(loop_info),
}
}
pub fn generate_simd_loop_ir(&mut self, simd_info: &X86OmpIRSimd) -> Vec<String> {
let mut ir = Vec::new();
let vw = simd_info.vector_width;
let elem_type = &simd_info.element_type;
let vec_type = format!("<{} x {}>", vw, elem_type);
let align = self.context.simd_level.vector_width_bits() / 8;
ir.push(format!("; SIMD vectorized loop: {} x {}", vw, elem_type));
ir.push(format!(
" %vec_ptr = getelementptr {}, {}* %base, i64 %iv",
elem_type, elem_type
));
ir.push(format!(
" %vec_load = load <{} x {}>, <{} x {}>* %vec_ptr, align {}",
vw, elem_type, vw, elem_type, align
));
ir.push(format!(
" %vec_result = fmul <{} x {}> %vec_load, <{} x {}> zeroinitializer",
vw, elem_type, vw, elem_type
));
ir.push(format!(
" store <{} x {}> %vec_result, <{} x {}>* %vec_ptr, align {}",
vw, elem_type, vw, elem_type, align
));
ir
}
pub fn generate_x86_simd_ir(
&mut self,
elem_type: &str,
vector_width: u32,
_has_mask: bool,
) -> Vec<String> {
vec![format!(
"; X86 SIMD: {} x {} using {:?}",
vector_width, elem_type, self.context.simd_level
)]
}
pub fn create_simd_info(&self, simd_clause: &X86SimdClause) -> X86OmpIRSimd {
let vw =
simd_clause.effective_vector_width(self.context.simd_level.f32_element_count() as u32);
X86OmpIRSimd {
vector_width: vw,
element_type: "float".to_string(),
safelen_clause: simd_clause.safelen,
simdlen_clause: simd_clause.simdlen,
aligned_ptrs: simd_clause.aligned_vars.clone(),
nontemporal_stores: simd_clause.nontemporal_vars.clone(),
linear_steps: Vec::new(),
private_vars: simd_clause.private_vars.clone(),
reductions: Vec::new(),
}
}
pub fn generate_task_ir(&self, outlined_fn: &str, flags: u32) -> Vec<String> {
vec![
format!("; Task creation: {}", outlined_fn),
format!(
" %task = call i8* @{}(@ident_t* @.loc, i32 %gtid, i32 {}, i64 0, i64 0, i8* @{})",
kmpc_x86::TASK_ALLOC,
flags,
outlined_fn
),
format!(
" call i32 @{}(@ident_t* @.loc, i32 %gtid, i8* %task)",
kmpc_x86::TASK
),
]
}
pub fn generate_target_ir(&self, outlined_fn: &str, device_id: i32) -> Vec<String> {
vec![
format!("; Target region: {}", outlined_fn),
format!(
" call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::TARGET_INIT
),
format!(
" call i32 @{}(@ident_t* @.loc, i32 {}, i8* @binary, i32 0, i8** %args, i8** %args, i64* %sizes, i64* %types, i32 0, i32 0)",
kmpc_x86::TARGET,
device_id
),
format!(
" call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::TARGET_DEINIT
),
]
}
pub fn declare_runtime_functions(&self) -> Vec<String> {
vec![
format!(
"declare void @{}(@ident_t*, i32, void (i32*, i32*, ...)*, ...)",
kmpc_x86::FORK_CALL
),
format!(
"declare void @{}(@ident_t*, i32, i32)",
kmpc_x86::PUSH_NUM_THREADS
),
format!("declare i32 @{}()", kmpc_x86::GLOBAL_THREAD_NUM),
format!(
"declare void @{}(@ident_t*, i32, i32, i32*, i32*, i32*, i32*, i32, i32)",
kmpc_x86::FOR_STATIC_INIT_4
),
format!("declare void @{}()", kmpc_x86::FOR_STATIC_FINI),
format!(
"declare void @{}(@ident_t*, i32, i32, i32, i32, i32, i32)",
kmpc_x86::DISPATCH_INIT_4
),
format!(
"declare i32 @{}(@ident_t*, i32, i32*, i32*, i32*, i32*)",
kmpc_x86::DISPATCH_NEXT_4
),
format!(
"declare void @{}(@ident_t*, i32)",
kmpc_x86::DISPATCH_FINI_4
),
format!("declare void @{}(@ident_t*, i32)", kmpc_x86::BARRIER),
format!("declare void @{}(@ident_t*, i32, i8*)", kmpc_x86::CRITICAL),
format!(
"declare void @{}(@ident_t*, i32, i8*)",
kmpc_x86::END_CRITICAL
),
format!("declare i32 @{}(@ident_t*, i32)", kmpc_x86::MASTER_BEGIN),
format!("declare void @{}(@ident_t*, i32)", kmpc_x86::MASTER_END),
format!("declare i32 @{}(@ident_t*, i32)", kmpc_x86::SINGLE_BEGIN),
format!("declare void @{}(@ident_t*, i32)", kmpc_x86::SINGLE_END),
format!("declare void @{}(@ident_t*, i32)", kmpc_x86::ORDERED_BEGIN),
format!("declare void @{}(@ident_t*, i32)", kmpc_x86::ORDERED_END),
format!(
"declare i32 @{}(@ident_t*, i32, i32*)",
kmpc_x86::ATOMIC_RD_4
),
format!(
"declare void @{}(@ident_t*, i32, i32*, i32)",
kmpc_x86::ATOMIC_WR_4
),
format!("declare void @{}(@ident_t*, i32)", kmpc_x86::FLUSH),
format!("declare i32 @{}(@ident_t*, i32, i32)", kmpc_x86::CANCEL),
format!(
"declare i32 @{}(@ident_t*, i32, i32)",
kmpc_x86::CANCELLATION_POINT
),
format!(
"declare void @{}(@ident_t*, i32, i8*)",
kmpc_x86::THREADPRIVATE_REGISTER
),
format!(
"declare i8* @{}(@ident_t*, i32, i32, i64, i64, i8*)",
kmpc_x86::TASK_ALLOC
),
format!("declare i32 @{}(@ident_t*, i32, i8*)", kmpc_x86::TASK),
format!(
"declare i32 @{}(@ident_t*, i32, i8*, i32, ...)",
kmpc_x86::TASK_WITH_DEPS
),
format!("declare i32 @{}(@ident_t*, i32)", kmpc_x86::TASKWAIT),
format!("declare i32 @{}(@ident_t*, i32)", kmpc_x86::TASKYIELD),
format!("declare void @{}(@ident_t*, i32)", kmpc_x86::TASKGROUP),
format!("declare void @{}(@ident_t*, i32)", kmpc_x86::TASKGROUP_END),
format!("declare void @{}(@ident_t*, i32)", kmpc_x86::TARGET_INIT),
format!("declare void @{}(@ident_t*, i32)", kmpc_x86::TARGET_DEINIT),
format!(
"declare void @{}(@ident_t*, i32, ...)",
kmpc_x86::TARGET_DATA_BEGIN
),
format!(
"declare void @{}(@ident_t*, i32, ...)",
kmpc_x86::TARGET_DATA_END
),
format!(
"declare i32 @{}(@ident_t*, i32, i8*, ...)",
kmpc_x86::TARGET
),
format!(
"declare i32 @{}(@ident_t*, i32, i32, i64, i8*, i8*, i8*)",
kmpc_x86::REDUCE
),
format!(
"declare void @{}(@ident_t*, i32, i8*)",
kmpc_x86::END_REDUCE
),
format!(
"declare void @{}(@ident_t*, i32, i32, i32*, i64*, i64*, i64*, i64, i64)",
kmpc_x86::DISTRIBUTE_STATIC_INIT
),
format!(
"declare void @{}(@ident_t*, i32)",
kmpc_x86::DISTRIBUTE_STATIC_FINI
),
format!("declare void @{}(@ident_t*, i32)", kmpc_x86::SECTIONS_INIT),
format!("declare i32 @{}(@ident_t*, i32)", kmpc_x86::NEXT_SECTION),
format!(
"declare void @{}(@ident_t*, i32, i32)",
kmpc_x86::PUSH_PROC_BIND
),
format!(
"declare void @{}(@ident_t*, i32, i32, i32)",
kmpc_x86::FORK_TEAMS
),
]
}
pub fn compile_directives(&mut self, directives: &[X86OmpDirective]) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!("; ModuleID = '{}'", self.module_name));
ir.push(format!(
"; target triple = \"{}\"",
self.context.target_triple
));
ir.push(format!(
"; OpenMP version = {}.{}",
OMP_VERSION_MAJOR, OMP_VERSION_MINOR
));
ir.push("".to_string());
ir.push("@.loc = private unnamed_addr constant %ident_t ...".to_string());
ir.push("".to_string());
ir.extend(self.declare_runtime_functions());
ir.push("".to_string());
for directive in directives {
match directive.kind {
X86OmpDirectiveKind::Parallel | X86OmpDirectiveKind::ParallelFor => {
let fn_name =
format!("__omp_outlined_parallel_{}", self.outlined_functions.len());
let mut captured = Vec::new();
for v in &directive.shared_vars {
captured.push(X86CapturedVariable {
name: v.clone(),
sharing: X86DataSharingAttr::Shared,
var_type: "i64".to_string(),
is_pointer: true,
reduction_op: None,
linear_step: Some(0),
});
}
for v in &directive.private_vars {
captured.push(X86CapturedVariable {
name: v.clone(),
sharing: X86DataSharingAttr::Private,
var_type: "i64".to_string(),
is_pointer: false,
reduction_op: None,
linear_step: Some(0),
});
}
let outlined =
self.generate_outlined_parallel_function(&fn_name, &captured, false, &[]);
ir.extend(self.emit_outlined_fn_signature(&outlined));
ir.extend(self.emit_outlined_fn_epilogue(&outlined));
}
X86OmpDirectiveKind::For => {
let loop_info = X86OmpIRLoopInfo {
induction_var: "i".to_string(),
lower_bound: 0,
upper_bound: 100,
stride: 1,
chunk_size: directive.chunk_size.unwrap_or(1) as i64,
schedule_id: directive.schedule.to_runtime_id(),
is_static: matches!(directive.schedule, X86OmpScheduleKind::Static),
has_nowait: directive.nowait,
has_ordered: matches!(directive.ordered, X86OrderedKind::Ordered),
collapse_count: directive.collapse.unwrap_or(1),
};
ir.extend(self.generate_loop_ir(&loop_info));
}
X86OmpDirectiveKind::Simd | X86OmpDirectiveKind::ForSimd => {
let sc = &directive.simd_clause;
if sc.has_any() {
let simd = self.create_simd_info(sc);
ir.extend(self.generate_simd_loop_ir(&simd));
}
}
X86OmpDirectiveKind::Task => {
ir.extend(self.generate_task_ir("__omp_outlined_task_0", 0));
}
X86OmpDirectiveKind::Target => {
ir.extend(self.generate_target_ir("__omp_outlined_target_0", 0));
}
_ => {
ir.push(format!(
"; Directive {:?} IR gen not implemented",
directive.kind
));
}
}
}
ir
}
}
impl Default for X86OpenMPIRGen {
fn default() -> Self {
Self::new(X86OpenMPFull::default())
}
}
#[derive(Debug, Clone)]
pub struct X86OmpRuntimeFn {
pub name: String,
pub return_type: String,
pub params: Vec<(String, String)>,
pub is_vararg: bool,
pub description: String,
}
#[derive(Debug, Clone, Default)]
pub struct X86ThreadTeam {
pub team_id: u32,
pub num_threads: u32,
pub master_tid: u32,
pub is_active: bool,
pub proc_bind: Option<X86ProcBind>,
pub nested: bool,
pub parent_team_id: Option<u32>,
}
#[derive(Debug, Clone, Default)]
pub struct X86WorkQueue {
pub lower_bound: i64,
pub upper_bound: i64,
pub stride: i64,
pub chunk_size: i64,
pub next_iter: i64,
pub remaining: i64,
pub is_static: bool,
pub schedule_id: i32,
}
#[derive(Debug, Clone)]
pub struct X86TaskDepNode {
pub task_id: u64,
pub variable: String,
pub dep_type: X86DependType,
pub satisfied: bool,
pub dependents: Vec<u64>,
pub predecessors: Vec<u64>,
}
#[derive(Debug, Clone, Default)]
pub struct X86TaskDepGraph {
pub nodes: Vec<X86TaskDepNode>,
pub ready_queue: Vec<u64>,
pub completed: Vec<u64>,
}
#[derive(Debug, Clone, Default)]
pub struct X86CancelState {
pub parallel_cancelled: bool,
pub loop_cancelled: bool,
pub sections_cancelled: bool,
pub taskgroup_cancelled: bool,
}
#[derive(Debug)]
pub struct X86OpenMPRuntime {
pub context: X86OpenMPFull,
pub functions: Vec<X86OmpRuntimeFn>,
pub thread_teams: Vec<X86ThreadTeam>,
pub work_queues: Vec<X86WorkQueue>,
pub dep_graph: X86TaskDepGraph,
pub cancel_state: X86CancelState,
pub cancellation_enabled: bool,
pub threadprivate_vars: Vec<String>,
pub critical_locks: HashMap<String, u64>,
pub pending_reductions: Vec<X86OmpIRReduction>,
pub verbose: bool,
}
impl X86OpenMPRuntime {
pub fn new(context: X86OpenMPFull) -> Self {
let functions = Self::build_runtime_signatures();
Self {
context,
functions,
thread_teams: Vec::new(),
work_queues: Vec::new(),
dep_graph: X86TaskDepGraph::default(),
cancel_state: X86CancelState::default(),
cancellation_enabled: false,
threadprivate_vars: Vec::new(),
critical_locks: HashMap::new(),
pending_reductions: Vec::new(),
verbose: false,
}
}
fn build_runtime_signatures() -> Vec<X86OmpRuntimeFn> {
vec![
X86OmpRuntimeFn {
name: kmpc_x86::FORK_CALL.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("argc".to_string(), "i32".to_string()),
("microtask".to_string(), "void(i32*,i32*,...)*".to_string()),
],
is_vararg: true,
description: "Fork a team of threads to execute a parallel region".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::FORK_TEAMS.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("num_teams".to_string(), "i32".to_string()),
("thread_limit".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Launch multiple teams".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::PUSH_NUM_THREADS.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("num_threads".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Set the number of threads for the next parallel region".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::PUSH_PROC_BIND.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("proc_bind".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Set the proc_bind policy for the next parallel region".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::GLOBAL_THREAD_NUM.to_string(),
return_type: "i32".to_string(),
params: vec![],
is_vararg: false,
description: "Return the global thread identifier".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::BOUND_THREAD_NUM.to_string(),
return_type: "i32".to_string(),
params: vec![],
is_vararg: false,
description: "Return the bound thread identifier".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::FOR_STATIC_INIT_4.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("schedtype".to_string(), "i32".to_string()),
("plastiter".to_string(), "i32*".to_string()),
("plower".to_string(), "i32*".to_string()),
("pupper".to_string(), "i32*".to_string()),
("pstride".to_string(), "i32*".to_string()),
("incr".to_string(), "i32".to_string()),
("chunk".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Initialize static loop scheduling for 32-bit bounds".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::FOR_STATIC_INIT_8.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("schedtype".to_string(), "i32".to_string()),
("plastiter".to_string(), "i32*".to_string()),
("plower".to_string(), "i64*".to_string()),
("pupper".to_string(), "i64*".to_string()),
("pstride".to_string(), "i64*".to_string()),
("incr".to_string(), "i64".to_string()),
("chunk".to_string(), "i64".to_string()),
],
is_vararg: false,
description: "Initialize static loop scheduling for 64-bit bounds".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::FOR_STATIC_FINI.to_string(),
return_type: "void".to_string(),
params: vec![],
is_vararg: false,
description: "Finalize static loop scheduling".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::DISPATCH_INIT_4.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("sched".to_string(), "i32".to_string()),
("lower".to_string(), "i32".to_string()),
("upper".to_string(), "i32".to_string()),
("stride".to_string(), "i32".to_string()),
("chunk".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Initialize dynamic/guided dispatch for 32-bit bounds".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::DISPATCH_NEXT_4.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("plastiter".to_string(), "i32*".to_string()),
("plower".to_string(), "i32*".to_string()),
("pupper".to_string(), "i32*".to_string()),
("pstride".to_string(), "i32*".to_string()),
],
is_vararg: false,
description: "Get next chunk from dynamic/guided dispatch".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::DISPATCH_FINI_4.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Finalize dynamic/guided dispatch".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::BARRIER.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Barrier: all threads in the team synchronize".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::CANCEL_BARRIER.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Cancellable barrier".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::CRITICAL.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("lck".to_string(), "i8*".to_string()),
],
is_vararg: false,
description: "Enter a named critical section".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::END_CRITICAL.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("lck".to_string(), "i8*".to_string()),
],
is_vararg: false,
description: "Exit a named critical section".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::MASTER_BEGIN.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Returns 1 for the master thread, 0 for others".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::MASTER_END.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "End of master region".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::SINGLE_BEGIN.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Returns 1 for the first arriving thread, 0 for others".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::SINGLE_END.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "End of single region with implicit barrier".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::ORDERED_BEGIN.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Begin ordered region".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::ORDERED_END.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "End ordered region".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::ATOMIC_RD_4.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("ptr".to_string(), "i32*".to_string()),
],
is_vararg: false,
description: "Atomic read of 4-byte integer".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::ATOMIC_WR_4.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("ptr".to_string(), "i32*".to_string()),
("val".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Atomic write of 4-byte integer".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::ATOMIC_RD_8.to_string(),
return_type: "i64".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("ptr".to_string(), "i64*".to_string()),
],
is_vararg: false,
description: "Atomic read of 8-byte integer".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::ATOMIC_WR_8.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("ptr".to_string(), "i64*".to_string()),
("val".to_string(), "i64".to_string()),
],
is_vararg: false,
description: "Atomic write of 8-byte integer".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::ATOMIC_DOUBLE_RD.to_string(),
return_type: "double".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("ptr".to_string(), "double*".to_string()),
],
is_vararg: false,
description: "Atomic read of double-precision float".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::ATOMIC_DOUBLE_WR.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("ptr".to_string(), "double*".to_string()),
("val".to_string(), "double".to_string()),
],
is_vararg: false,
description: "Atomic write of double-precision float".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::FLUSH.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Memory flush: make memory consistent across threads".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::CANCEL.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("cancel_flag".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Request cancellation of a construct".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::CANCELLATION_POINT.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("cancel_flag".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Check for cancellation and act if requested".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::THREADPRIVATE_REGISTER.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("ptr".to_string(), "i8*".to_string()),
],
is_vararg: false,
description: "Register a threadprivate variable".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::THREADPRIVATE_CACHED.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("ptr".to_string(), "i8*".to_string()),
],
is_vararg: false,
description: "Access cached threadprivate copy".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TASK_ALLOC.to_string(),
return_type: "i8*".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("flags".to_string(), "i32".to_string()),
("sizeof_kmp_task".to_string(), "i64".to_string()),
("sizeof_shareds".to_string(), "i64".to_string()),
("task_entry".to_string(), "i8*".to_string()),
],
is_vararg: false,
description: "Allocate a task descriptor".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TASK.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("task".to_string(), "i8*".to_string()),
],
is_vararg: false,
description: "Submit a task for execution".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TASK_WITH_DEPS.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("task".to_string(), "i8*".to_string()),
("ndeps".to_string(), "i32".to_string()),
],
is_vararg: true,
description: "Submit a task with dependences".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TASKWAIT.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Wait for all child tasks to complete".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TASKYIELD.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Yield execution to another task".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TASKGROUP.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Begin a taskgroup (synchronize on end)".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TASKGROUP_END.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "End taskgroup: wait for all tasks in group".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TASKLOOP.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: true,
description: "Create tasks for a parallel loop".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TARGET_INIT.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Initialize the target device environment".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TARGET_DEINIT.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Deinitialize the target device environment".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TARGET_DATA_BEGIN.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("device_id".to_string(), "i32".to_string()),
],
is_vararg: true,
description: "Begin target data region (map data to device)".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TARGET_DATA_END.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("device_id".to_string(), "i32".to_string()),
],
is_vararg: true,
description: "End target data region (unmap data from device)".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TARGET.to_string(),
return_type: "i32".to_string(),
params: vec![("loc".to_string(), "ident_t*".to_string())],
is_vararg: true,
description: "Execute a target region on device".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TARGET_UPDATE.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("device_id".to_string(), "i32".to_string()),
],
is_vararg: true,
description: "Update device data (host-to-device or device-to-host)".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::TARGET_MEMCPY.to_string(),
return_type: "void".to_string(),
params: vec![("loc".to_string(), "ident_t*".to_string())],
is_vararg: true,
description: "Memory copy between host and device".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::REDUCE.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("num_args".to_string(), "i32".to_string()),
("reduce_size".to_string(), "i64".to_string()),
("reduce_data".to_string(), "i8*".to_string()),
("reduce_func".to_string(), "i8*".to_string()),
("lck".to_string(), "i8*".to_string()),
],
is_vararg: false,
description: "Perform inter-thread reduction".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::END_REDUCE.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("lck".to_string(), "i8*".to_string()),
],
is_vararg: false,
description: "Finalize reduction and broadcast result".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::SECTIONS_INIT.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Initialize sections worksharing".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::NEXT_SECTION.to_string(),
return_type: "i32".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Claim the next available section".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::DISTRIBUTE_STATIC_INIT.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("sched".to_string(), "i32".to_string()),
("plastiter".to_string(), "i32*".to_string()),
("plower".to_string(), "i64*".to_string()),
("pupper".to_string(), "i64*".to_string()),
("pstride".to_string(), "i64*".to_string()),
("incr".to_string(), "i64".to_string()),
("chunk".to_string(), "i64".to_string()),
],
is_vararg: false,
description: "Initialize distribute static scheduling".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::DISTRIBUTE_STATIC_FINI.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "Finalize distribute static scheduling".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::ALLOC_SHARED.to_string(),
return_type: "i8*".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("size".to_string(), "i64".to_string()),
],
is_vararg: false,
description: "Allocate shared memory".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::FREE_SHARED.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("ptr".to_string(), "i8*".to_string()),
],
is_vararg: false,
description: "Free shared memory allocation".to_string(),
},
X86OmpRuntimeFn {
name: kmpc_x86::SIMD.to_string(),
return_type: "void".to_string(),
params: vec![
("loc".to_string(), "ident_t*".to_string()),
("gtid".to_string(), "i32".to_string()),
("simdlen".to_string(), "i32".to_string()),
],
is_vararg: false,
description: "SIMD loop hint".to_string(),
},
]
}
pub fn start_thread_team(&mut self, num_threads: u32, proc_bind: Option<X86ProcBind>) {
let team_id = self.thread_teams.len() as u32;
let parent_id = self.thread_teams.last().map(|t| t.team_id);
let team = X86ThreadTeam {
team_id,
num_threads,
master_tid: 0,
is_active: true,
proc_bind,
nested: parent_id.is_some(),
parent_team_id: parent_id,
};
if self.verbose {
eprintln!(
"[X86OpenMPRuntime] Starting team {} with {} threads (nested={})",
team_id, num_threads, team.nested
);
}
self.thread_teams.push(team);
}
pub fn end_thread_team(&mut self) {
if let Some(team) = self.thread_teams.last_mut() {
team.is_active = false;
if self.verbose {
eprintln!("[X86OpenMPRuntime] Ending team {}", team.team_id);
}
}
}
pub fn current_team(&self) -> Option<&X86ThreadTeam> {
self.thread_teams.last()
}
pub fn team_count(&self) -> usize {
self.thread_teams.len()
}
pub fn create_work_queue(
&mut self,
lower: i64,
upper: i64,
stride: i64,
chunk_size: i64,
schedule_id: i32,
) {
let remaining = ((upper - lower) / stride) + 1;
let queue = X86WorkQueue {
lower_bound: lower,
upper_bound: upper,
stride,
chunk_size: chunk_size.max(1),
next_iter: lower,
remaining,
is_static: schedule_id == 1 || schedule_id == 34,
schedule_id,
};
self.work_queues.push(queue);
}
pub fn next_chunk(&mut self, queue_index: usize) -> Option<(i64, i64)> {
if queue_index >= self.work_queues.len() {
return None;
}
let q = &mut self.work_queues[queue_index];
if q.remaining <= 0 {
return None;
}
let chunk = match q.schedule_id {
1 | 34 => q.chunk_size,
35 => q.chunk_size.min(q.remaining),
36 => {
let dynamic_chunk = (q.remaining / 2).max(1).min(q.chunk_size);
dynamic_chunk
}
_ => q.chunk_size,
};
let start = q.next_iter;
let end = (start + (chunk - 1) * q.stride).min(q.upper_bound);
let actual_count = ((end - start) / q.stride) + 1;
q.next_iter = end + q.stride;
q.remaining -= actual_count;
if actual_count > 0 {
Some((start, end))
} else {
None
}
}
pub fn finalize_work_queue(&mut self, queue_index: usize) {
if queue_index < self.work_queues.len() {
self.work_queues.remove(queue_index);
}
}
pub fn add_task_node(&mut self, task_id: u64, variable: &str, dep_type: X86DependType) {
let node = X86TaskDepNode {
task_id,
variable: variable.to_string(),
dep_type,
satisfied: false,
dependents: Vec::new(),
predecessors: Vec::new(),
};
self.dep_graph.nodes.push(node);
}
pub fn add_dependency(&mut self, from_task: u64, to_task: u64) {
if let Some(node) = self
.dep_graph
.nodes
.iter_mut()
.find(|n| n.task_id == from_task)
{
node.dependents.push(to_task);
}
if let Some(node) = self
.dep_graph
.nodes
.iter_mut()
.find(|n| n.task_id == to_task)
{
node.predecessors.push(from_task);
}
}
pub fn complete_task(&mut self, task_id: u64) {
self.dep_graph.completed.push(task_id);
let mut ready_now = Vec::new();
for node in &mut self.dep_graph.nodes {
if node.predecessors.contains(&task_id) {
node.predecessors.retain(|p| *p != task_id);
let all_done = node
.predecessors
.iter()
.all(|p| self.dep_graph.completed.contains(p));
if all_done && !node.satisfied {
node.satisfied = true;
ready_now.push(node.task_id);
}
}
}
self.dep_graph.ready_queue.extend(ready_now);
}
pub fn next_ready_task(&mut self) -> Option<u64> {
self.dep_graph.ready_queue.pop()
}
pub fn set_cancel(&mut self, kind: &str) {
match kind {
"parallel" => self.cancel_state.parallel_cancelled = true,
"loop" | "for" => self.cancel_state.loop_cancelled = true,
"sections" => self.cancel_state.sections_cancelled = true,
"taskgroup" => self.cancel_state.taskgroup_cancelled = true,
_ => {}
}
if self.verbose {
eprintln!("[X86OpenMPRuntime] Cancel requested: {}", kind);
}
}
pub fn is_cancelled(&self, kind: &str) -> bool {
match kind {
"parallel" => self.cancel_state.parallel_cancelled,
"loop" | "for" => self.cancel_state.loop_cancelled,
"sections" => self.cancel_state.sections_cancelled,
"taskgroup" => self.cancel_state.taskgroup_cancelled,
_ => false,
}
}
pub fn reset_cancel(&mut self) {
self.cancel_state = X86CancelState::default();
}
pub fn set_cancellation(&mut self, enabled: bool) {
self.cancellation_enabled = enabled;
}
pub fn register_threadprivate(&mut self, var_name: &str) {
if !self.threadprivate_vars.contains(&var_name.to_string()) {
self.threadprivate_vars.push(var_name.to_string());
}
}
pub fn is_threadprivate(&self, var_name: &str) -> bool {
self.threadprivate_vars.contains(&var_name.to_string())
}
pub fn register_critical_lock(&mut self, name: &str) -> u64 {
let id = self.critical_locks.len() as u64;
self.critical_locks.insert(name.to_string(), id);
id
}
pub fn get_critical_lock(&self, name: &str) -> Option<u64> {
self.critical_locks.get(name).copied()
}
pub fn register_reduction(&mut self, red: X86OmpIRReduction) {
self.pending_reductions.push(red);
}
pub fn clear_reductions(&mut self) {
self.pending_reductions.clear();
}
pub fn get_function(&self, name: &str) -> Option<&X86OmpRuntimeFn> {
self.functions.iter().find(|f| f.name == name)
}
pub fn has_function(&self, name: &str) -> bool {
self.functions.iter().any(|f| f.name == name)
}
pub fn function_names(&self) -> Vec<&str> {
self.functions.iter().map(|f| f.name.as_str()).collect()
}
pub fn function_count(&self) -> usize {
self.functions.len()
}
pub fn status_report(&self) -> String {
format!(
"X86OpenMPRuntime: {} functions, {} teams, {} work queues, {} tasks, cancel={}",
self.functions.len(),
self.thread_teams.len(),
self.work_queues.len(),
self.dep_graph.nodes.len(),
self.cancellation_enabled
)
}
}
impl Default for X86OpenMPRuntime {
fn default() -> Self {
Self::new(X86OpenMPFull::default())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_omp_version() {
assert_eq!(OMP_VERSION_MAJOR, 5);
assert_eq!(OMP_VERSION_MINOR, 2);
assert_eq!(OMP_VERSION, 502);
}
#[test]
fn test_x86_constants() {
assert_eq!(X86_CACHE_LINE_SIZE, 64);
assert!(X86_OMP_MAX_THREADS > 0);
assert_eq!(X86_DEFAULT_SIMD_WIDTH, 4);
assert_eq!(X86_MAX_SIMD_WIDTH, 16);
}
#[test]
fn test_kmpc_function_names() {
assert_eq!(kmpc_x86::FORK_CALL, "__kmpc_fork_call");
assert_eq!(kmpc_x86::FORK_TEAMS, "__kmpc_fork_teams");
assert_eq!(kmpc_x86::PUSH_NUM_THREADS, "__kmpc_push_num_threads");
assert_eq!(kmpc_x86::BARRIER, "__kmpc_barrier");
assert_eq!(kmpc_x86::CRITICAL, "__kmpc_critical");
assert_eq!(kmpc_x86::END_CRITICAL, "__kmpc_end_critical");
assert_eq!(kmpc_x86::TASK_ALLOC, "__kmpc_omp_task_alloc");
assert_eq!(kmpc_x86::TASK, "__kmpc_omp_task");
assert_eq!(kmpc_x86::TASKWAIT, "__kmpc_omp_taskwait");
assert_eq!(kmpc_x86::TASKGROUP, "__kmpc_taskgroup");
assert_eq!(kmpc_x86::TARGET, "__kmpc_target");
assert_eq!(kmpc_x86::REDUCE, "__kmpc_reduce");
}
#[test]
fn test_tgt_function_names() {
assert_eq!(tgt_x86::REGISTER_LIB, "__tgt_register_lib");
assert_eq!(tgt_x86::TARGET, "__tgt_target");
assert_eq!(tgt_x86::TARGET_DATA_BEGIN, "__tgt_target_data_begin");
assert_eq!(tgt_x86::TARGET_DATA_END, "__tgt_target_data_end");
assert_eq!(tgt_x86::TARGET_MEMCPY, "__tgt_target_memcpy");
}
#[test]
fn test_directive_kind_is_parallel() {
assert!(X86OmpDirectiveKind::Parallel.is_parallel());
assert!(X86OmpDirectiveKind::ParallelFor.is_parallel());
assert!(!X86OmpDirectiveKind::For.is_parallel());
}
#[test]
fn test_directive_kind_is_worksharing() {
assert!(X86OmpDirectiveKind::For.is_worksharing());
assert!(X86OmpDirectiveKind::Sections.is_worksharing());
assert!(!X86OmpDirectiveKind::Barrier.is_worksharing());
}
#[test]
fn test_directive_kind_is_target() {
assert!(X86OmpDirectiveKind::Target.is_target());
assert!(X86OmpDirectiveKind::TargetData.is_target());
assert!(!X86OmpDirectiveKind::Parallel.is_target());
}
#[test]
fn test_directive_kind_is_simd() {
assert!(X86OmpDirectiveKind::Simd.is_simd());
assert!(X86OmpDirectiveKind::ForSimd.is_simd());
assert!(!X86OmpDirectiveKind::Target.is_simd());
}
#[test]
fn test_directive_kind_name() {
assert_eq!(X86OmpDirectiveKind::Parallel.name(), "parallel");
assert_eq!(X86OmpDirectiveKind::For.name(), "for");
assert_eq!(X86OmpDirectiveKind::Target.name(), "target");
assert_eq!(X86OmpDirectiveKind::Barrier.name(), "barrier");
}
#[test]
fn test_directive_kind_from_pragma_name() {
assert_eq!(
X86OmpDirectiveKind::from_pragma_name("parallel"),
Some(X86OmpDirectiveKind::Parallel)
);
assert_eq!(
X86OmpDirectiveKind::from_pragma_name("parallel for"),
Some(X86OmpDirectiveKind::ParallelFor)
);
assert_eq!(
X86OmpDirectiveKind::from_pragma_name("for"),
Some(X86OmpDirectiveKind::For)
);
assert_eq!(
X86OmpDirectiveKind::from_pragma_name("target data"),
Some(X86OmpDirectiveKind::TargetData)
);
assert_eq!(X86OmpDirectiveKind::from_pragma_name("nonexistent"), None);
}
#[test]
fn test_schedule_kind_to_runtime_id() {
assert_eq!(X86OmpScheduleKind::Static.to_runtime_id(), 1);
assert_eq!(X86OmpScheduleKind::Dynamic.to_runtime_id(), 2);
assert_eq!(X86OmpScheduleKind::Guided.to_runtime_id(), 3);
assert_eq!(X86OmpScheduleKind::Auto.to_runtime_id(), 4);
assert_eq!(X86OmpScheduleKind::Runtime.to_runtime_id(), 5);
}
#[test]
fn test_schedule_kind_from_name() {
assert_eq!(
X86OmpScheduleKind::from_name("static"),
Some(X86OmpScheduleKind::Static)
);
assert_eq!(
X86OmpScheduleKind::from_name("dynamic"),
Some(X86OmpScheduleKind::Dynamic)
);
assert_eq!(
X86OmpScheduleKind::from_name("guided"),
Some(X86OmpScheduleKind::Guided)
);
assert_eq!(
X86OmpScheduleKind::from_name("auto"),
Some(X86OmpScheduleKind::Auto)
);
assert_eq!(
X86OmpScheduleKind::from_name("runtime"),
Some(X86OmpScheduleKind::Runtime)
);
assert_eq!(X86OmpScheduleKind::from_name("invalid"), None);
}
#[test]
fn test_schedule_kind_default() {
let default = X86OmpScheduleKind::default();
assert_eq!(default, X86OmpScheduleKind::Static);
}
#[test]
fn test_map_type_from_name() {
assert_eq!(X86OmpMapType::from_name("to"), Some(X86OmpMapType::To));
assert_eq!(X86OmpMapType::from_name("from"), Some(X86OmpMapType::From));
assert_eq!(
X86OmpMapType::from_name("tofrom"),
Some(X86OmpMapType::ToFrom)
);
assert_eq!(
X86OmpMapType::from_name("alloc"),
Some(X86OmpMapType::Alloc)
);
assert_eq!(
X86OmpMapType::from_name("delete"),
Some(X86OmpMapType::Delete)
);
}
#[test]
fn test_map_type_is_data_movement() {
assert!(X86OmpMapType::To.is_data_movement());
assert!(X86OmpMapType::From.is_data_movement());
assert!(X86OmpMapType::ToFrom.is_data_movement());
assert!(!X86OmpMapType::Alloc.is_data_movement());
}
#[test]
fn test_reduction_op_from_name() {
assert_eq!(X86ReductionOp::from_name("+"), Some(X86ReductionOp::Add));
assert_eq!(X86ReductionOp::from_name("*"), Some(X86ReductionOp::Mul));
assert_eq!(X86ReductionOp::from_name("-"), Some(X86ReductionOp::Sub));
assert_eq!(X86ReductionOp::from_name("min"), Some(X86ReductionOp::Min));
assert_eq!(X86ReductionOp::from_name("max"), Some(X86ReductionOp::Max));
assert_eq!(
X86ReductionOp::from_name("&&"),
Some(X86ReductionOp::LogicalAnd)
);
assert_eq!(
X86ReductionOp::from_name("||"),
Some(X86ReductionOp::LogicalOr)
);
assert_eq!(X86ReductionOp::from_name("&"), Some(X86ReductionOp::BAnd));
assert_eq!(X86ReductionOp::from_name("|"), Some(X86ReductionOp::BOr));
}
#[test]
fn test_reduction_op_commutative() {
assert!(X86ReductionOp::Add.is_commutative());
assert!(X86ReductionOp::Mul.is_commutative());
assert!(X86ReductionOp::BAnd.is_commutative());
assert!(X86ReductionOp::BOr.is_commutative());
assert!(X86ReductionOp::LogicalAnd.is_commutative());
assert!(X86ReductionOp::LogicalOr.is_commutative());
assert!(!X86ReductionOp::Sub.is_commutative());
}
#[test]
fn test_reduction_op_neutral_value() {
assert_eq!(X86ReductionOp::Add.neutral_value(), Some(0));
assert_eq!(X86ReductionOp::Mul.neutral_value(), Some(1));
assert_eq!(X86ReductionOp::BAnd.neutral_value(), Some(-1));
assert_eq!(X86ReductionOp::BOr.neutral_value(), Some(0));
assert_eq!(X86ReductionOp::LogicalAnd.neutral_value(), Some(1));
assert_eq!(X86ReductionOp::LogicalOr.neutral_value(), Some(0));
}
#[test]
fn test_data_sharing_attr_name() {
assert_eq!(X86DataSharingAttr::Shared.name(), "shared");
assert_eq!(X86DataSharingAttr::Private.name(), "private");
assert_eq!(X86DataSharingAttr::FirstPrivate.name(), "firstprivate");
assert_eq!(X86DataSharingAttr::LastPrivate.name(), "lastprivate");
assert_eq!(X86DataSharingAttr::Reduction.name(), "reduction");
assert_eq!(X86DataSharingAttr::Linear.name(), "linear");
}
#[test]
fn test_default_attr_from_name() {
assert_eq!(
X86DefaultAttr::from_name("shared"),
Some(X86DefaultAttr::Shared)
);
assert_eq!(
X86DefaultAttr::from_name("private"),
Some(X86DefaultAttr::Private)
);
assert_eq!(
X86DefaultAttr::from_name("firstprivate"),
Some(X86DefaultAttr::FirstPrivate)
);
assert_eq!(
X86DefaultAttr::from_name("none"),
Some(X86DefaultAttr::None)
);
}
#[test]
fn test_proc_bind_name() {
assert_eq!(X86ProcBind::False.name(), "false");
assert_eq!(X86ProcBind::True.name(), "true");
assert_eq!(X86ProcBind::Master.name(), "master");
assert_eq!(X86ProcBind::Close.name(), "close");
assert_eq!(X86ProcBind::Spread.name(), "spread");
}
#[test]
fn test_proc_bind_from_name() {
assert_eq!(X86ProcBind::from_name("spread"), Some(X86ProcBind::Spread));
assert_eq!(X86ProcBind::from_name("close"), Some(X86ProcBind::Close));
assert_eq!(X86ProcBind::from_name("master"), Some(X86ProcBind::Master));
assert_eq!(
X86ProcBind::from_name("primary"),
Some(X86ProcBind::Primary)
);
}
#[test]
fn test_depend_type_from_name() {
assert_eq!(X86DependType::from_name("in"), Some(X86DependType::In));
assert_eq!(X86DependType::from_name("out"), Some(X86DependType::Out));
assert_eq!(
X86DependType::from_name("inout"),
Some(X86DependType::InOut)
);
assert_eq!(
X86DependType::from_name("mutexinoutset"),
Some(X86DependType::MutexInOutSet)
);
}
#[test]
fn test_depend_type_runtime_id() {
assert_eq!(X86DependType::In.runtime_id(), 1);
assert_eq!(X86DependType::Out.runtime_id(), 2);
assert_eq!(X86DependType::InOut.runtime_id(), 3);
}
#[test]
fn test_simd_clause_new() {
let clause = X86SimdClause::new();
assert_eq!(clause.safelen, None);
assert_eq!(clause.simdlen, None);
assert!(clause.aligned_vars.is_empty());
assert!(!clause.has_any());
}
#[test]
fn test_simd_clause_effective_vector_width() {
let clause = X86SimdClause {
simdlen: Some(8),
..X86SimdClause::new()
};
let hw_max = X86SimdLevel::AVX2.f32_element_count() as u32;
assert_eq!(clause.effective_vector_width(hw_max), 8);
}
#[test]
fn test_simd_clause_default() {
let clause = X86SimdClause::default();
assert!(!clause.has_any());
}
#[test]
fn test_map_clause_new() {
let map = X86MapClause::new("A", X86OmpMapType::To);
assert_eq!(map.variable_name, "A");
assert_eq!(map.map_type, X86OmpMapType::To);
assert!(!map.array_section);
}
#[test]
fn test_map_clause_with_array() {
let map = X86MapClause::new("A", X86OmpMapType::ToFrom);
let arr = map.with_array(0, 10);
assert!(arr.array_section);
assert_eq!(arr.array_start, Some("0".to_string()));
assert_eq!(arr.array_length, Some("10".to_string()));
}
#[test]
fn test_omp_directive_new() {
let dir = X86OmpDirective::new(X86OmpDirectiveKind::Parallel);
assert_eq!(dir.kind, X86OmpDirectiveKind::Parallel);
assert_eq!(dir.schedule, X86OmpScheduleKind::default());
assert!(dir.private_vars.is_empty());
assert!(dir.shared_vars.is_empty());
}
#[test]
fn test_omp_directive_with_schedule() {
let dir = X86OmpDirective::new(X86OmpDirectiveKind::For)
.with_schedule(X86OmpScheduleKind::Static, None);
assert_eq!(dir.schedule, X86OmpScheduleKind::Static);
}
#[test]
fn test_omp_directive_with_num_threads() {
let dir = X86OmpDirective::new(X86OmpDirectiveKind::Parallel).with_num_threads("4");
assert_eq!(dir.num_threads, Some("4".to_string()));
}
#[test]
fn test_omp_directive_with_nowait() {
let dir = X86OmpDirective::new(X86OmpDirectiveKind::For).with_nowait();
assert!(dir.nowait);
}
#[test]
fn test_omp_directive_with_collapse() {
let dir = X86OmpDirective::new(X86OmpDirectiveKind::For).with_collapse(3);
assert_eq!(dir.collapse, Some(3));
}
#[test]
fn test_omp_directive_add_private() {
let mut dir = X86OmpDirective::new(X86OmpDirectiveKind::Parallel);
dir.add_private("x");
assert!(dir.private_vars.contains(&"x".to_string()));
}
#[test]
fn test_omp_directive_add_shared() {
let mut dir = X86OmpDirective::new(X86OmpDirectiveKind::Parallel);
dir.add_shared("A");
assert!(dir.shared_vars.contains(&"A".to_string()));
}
#[test]
fn test_omp_directive_add_reduction() {
let mut dir = X86OmpDirective::new(X86OmpDirectiveKind::Parallel);
dir.add_reduction("sum", X86ReductionOp::Add);
assert_eq!(dir.reductions.len(), 1);
assert_eq!(dir.reductions[0], ("sum".to_string(), X86ReductionOp::Add));
}
#[test]
fn test_omp_directive_has_data_environment() {
let dir = X86OmpDirective::new(X86OmpDirectiveKind::Target);
assert!(dir.has_data_environment());
let dir2 = X86OmpDirective::new(X86OmpDirectiveKind::Parallel);
assert!(!dir2.has_data_environment());
}
#[test]
fn test_omp_directive_all_captured_vars() {
let mut dir = X86OmpDirective::new(X86OmpDirectiveKind::Parallel);
dir.add_shared("A".to_string());
dir.add_private("i".to_string());
let vars = dir.all_captured_vars();
assert!(vars.contains(&"A".to_string()));
assert!(vars.contains(&"i".to_string()));
}
#[test]
fn test_simd_level_vector_width_bits() {
assert_eq!(X86SimdLevel::None.vector_width_bits(), 0);
assert_eq!(X86SimdLevel::SSE.vector_width_bits(), 128);
assert_eq!(X86SimdLevel::SSE2.vector_width_bits(), 128);
assert_eq!(X86SimdLevel::AVX.vector_width_bits(), 256);
assert_eq!(X86SimdLevel::AVX2.vector_width_bits(), 256);
assert_eq!(X86SimdLevel::AVX512F.vector_width_bits(), 512);
}
#[test]
fn test_simd_level_element_count() {
let sse = X86SimdLevel::SSE2;
assert_eq!(sse.f32_element_count(), 4);
assert_eq!(sse.f64_element_count(), 2);
assert_eq!(sse.i32_element_count(), 4);
assert_eq!(sse.i8_element_count(), 16);
let avx2 = X86SimdLevel::AVX2;
assert_eq!(avx2.f32_element_count(), 8);
assert_eq!(avx2.f64_element_count(), 4);
let avx512 = X86SimdLevel::AVX512F;
assert_eq!(avx512.f32_element_count(), 16);
assert_eq!(avx512.f64_element_count(), 8);
}
#[test]
fn test_simd_level_supports_masked() {
assert!(!X86SimdLevel::SSE2.supports_masked_operations());
assert!(!X86SimdLevel::AVX2.supports_masked_operations());
assert!(X86SimdLevel::AVX512F.supports_masked_operations());
}
#[test]
fn test_simd_level_from_features() {
let level = X86SimdLevel::from_features(true, false, false);
assert_eq!(level, X86SimdLevel::AVX512F);
let level2 = X86SimdLevel::from_features(false, true, false);
assert_eq!(level2, X86SimdLevel::AVX2);
let level3 = X86SimdLevel::from_features(false, false, true);
assert_eq!(level3, X86SimdLevel::SSE2);
let level4 = X86SimdLevel::from_features(false, false, false);
assert_eq!(level4, X86SimdLevel::None);
}
#[test]
fn test_icv_storage_new() {
let icv = X86ICVStorage::new();
assert_eq!(icv.nthreads_var, 0);
assert_eq!(icv.dyn_var, 0);
assert_eq!(icv.nest_var, 0);
assert_eq!(icv.cancellation_var, 0);
}
#[test]
fn test_icv_storage_effective_num_threads() {
let mut icv = X86ICVStorage::new();
icv.nthreads_var = 4;
assert_eq!(icv.effective_num_threads(), 4);
}
#[test]
fn test_icv_storage_default() {
let icv = X86ICVStorage::default();
assert_eq!(icv.nthreads_var, 0);
}
#[test]
fn test_openmp_full_new_x86_64() {
let omp = X86OpenMPFull::new_x86_64();
assert!(omp.target_triple.contains("x86_64"));
assert_eq!(omp.simd_level, X86SimdLevel::AVX2);
assert_eq!(omp.omp_version, 502);
assert!(omp.x86_optimizations);
}
#[test]
fn test_openmp_full_new_x86_64_avx512() {
let omp = X86OpenMPFull::new_x86_64_avx512();
assert_eq!(omp.simd_level, X86SimdLevel::AVX512F);
}
#[test]
fn test_openmp_full_new_i386() {
let omp = X86OpenMPFull::new_i386();
assert!(omp.target_triple.contains("i386"));
assert_eq!(omp.simd_level, X86SimdLevel::SSE2);
}
#[test]
fn test_openmp_full_disable_optimizations() {
let mut omp = X86OpenMPFull::new_x86_64();
assert!(omp.x86_optimizations);
omp.disable_x86_optimizations();
assert!(!omp.x86_optimizations);
}
#[test]
fn test_openmp_full_vector_width() {
let omp = X86OpenMPFull::new_x86_64();
assert_eq!(omp.vector_width_bits(), 256);
assert_eq!(omp.f32_vector_len(), 8);
assert_eq!(omp.f64_vector_len(), 4);
assert_eq!(omp.i32_vector_len(), 8);
}
#[test]
fn test_openmp_full_default() {
let omp = X86OpenMPFull::default();
assert_eq!(omp.simd_level, X86SimdLevel::AVX2);
}
#[test]
fn test_parser_new() {
let parser = X86OmpDirectiveParser::new();
assert_eq!(parser.recognized_count(), 0);
assert!(!parser.has_errors());
assert!(parser.errors().is_empty());
}
#[test]
fn test_parser_parse_parallel_directive() {
let mut parser = X86OmpDirectiveParser::new();
let result = parser.parse_directive("omp parallel private(i) shared(A) num_threads(4)");
assert!(result.is_ok());
let dir = result.unwrap();
assert_eq!(dir.kind, X86OmpDirectiveKind::Parallel);
assert!(dir.private_vars.contains(&"i".to_string()));
assert!(dir.shared_vars.contains(&"A".to_string()));
assert_eq!(dir.num_threads, Some("4".to_string()));
}
#[test]
fn test_parser_parse_for_directive() {
let mut parser = X86OmpDirectiveParser::new();
let result = parser.parse_directive("omp for schedule(static, 8) nowait");
assert!(result.is_ok());
let dir = result.unwrap();
assert_eq!(dir.kind, X86OmpDirectiveKind::For);
assert_eq!(dir.schedule, Some(X86OmpScheduleKind::Static));
assert_eq!(dir.chunk_size, Some(8));
assert!(dir.nowait);
}
#[test]
fn test_parser_parse_target_directive() {
let mut parser = X86OmpDirectiveParser::new();
let result = parser.parse_directive("omp target map(to: A) map(from: B) device(0)");
assert!(result.is_ok());
let dir = result.unwrap();
assert_eq!(dir.kind, X86OmpDirectiveKind::Target);
assert_eq!(dir.map_clauses.len(), 2);
assert_eq!(dir.device_id, Some(0));
}
#[test]
fn test_parser_parse_task_directive() {
let mut parser = X86OmpDirectiveParser::new();
let result = parser.parse_directive("omp task depend(in: x) priority(5) untied");
assert!(result.is_ok());
let dir = result.unwrap();
assert_eq!(dir.kind, X86OmpDirectiveKind::Task);
assert_eq!(dir.depend_clauses.len(), 1);
assert!(dir.untied);
assert_eq!(dir.priority_clause, Some(5));
}
#[test]
fn test_parser_parse_simd_directive() {
let mut parser = X86OmpDirectiveParser::new();
let result = parser.parse_directive("omp simd simdlen(8) safelen(16) aligned(A: 64)");
assert!(result.is_ok());
let dir = result.unwrap();
assert_eq!(dir.kind, X86OmpDirectiveKind::Simd);
assert!(dir.simd_clause.is_some());
if let Some(ref sc) = dir.simd_clause {
assert_eq!(sc.simdlen, Some(8));
assert_eq!(sc.safelen, Some(16));
assert!(sc.aligned_vars.contains(&"A".to_string()));
}
}
#[test]
fn test_parser_parse_invalid() {
let mut parser = X86OmpDirectiveParser::new();
let result = parser.parse_directive("omp nonexistent_directive");
assert!(result.is_err());
assert!(parser.has_errors());
}
#[test]
fn test_parser_parse_barrier() {
let mut parser = X86OmpDirectiveParser::new();
let result = parser.parse_directive("omp barrier");
assert!(result.is_ok());
let dir = result.unwrap();
assert_eq!(dir.kind, X86OmpDirectiveKind::Barrier);
}
#[test]
fn test_parser_parse_critical() {
let mut parser = X86OmpDirectiveParser::new();
let result = parser.parse_directive("omp critical (my_lock)");
assert!(result.is_ok());
let dir = result.unwrap();
assert_eq!(dir.kind, X86OmpDirectiveKind::Critical);
assert_eq!(dir.critical_name, Some("my_lock".to_string()));
}
#[test]
fn test_parser_parse_reduction() {
let mut parser = X86OmpDirectiveParser::new();
let result = parser.parse_directive("omp parallel for reduction(+:sum) reduction(*:prod)");
assert!(result.is_ok());
let dir = result.unwrap();
assert_eq!(dir.reductions.len(), 2);
}
#[test]
fn test_parser_parse_taskloop() {
let mut parser = X86OmpDirectiveParser::new();
let result = parser.parse_directive("omp taskloop grainsize(64) num_tasks(16)");
assert!(result.is_ok());
let dir = result.unwrap();
assert_eq!(dir.kind, X86OmpDirectiveKind::TaskLoop);
assert_eq!(dir.grainsize, Some(64));
assert_eq!(dir.num_tasks, Some(16));
}
#[test]
fn test_codegen_new() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
assert_eq!(cg.parallel_count, 0);
assert_eq!(cg.loop_count, 0);
assert_eq!(cg.task_count, 0);
assert_eq!(cg.target_count, 0);
}
#[test]
fn test_codegen_lower_parallel() {
let ctx = X86OpenMPFull::new_x86_64();
let mut cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective::new(X86OmpDirectiveKind::Parallel);
let region = cg.lower_parallel(&dir);
assert_eq!(cg.parallel_count, 1);
assert!(region.outlined_fn_name.starts_with("__omp_parallel_"));
}
#[test]
fn test_codegen_lower_for_loop() {
let ctx = X86OpenMPFull::new_x86_64();
let mut cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective::new(X86OmpDirectiveKind::For)
.with_schedule(X86OmpScheduleKind::Static);
let loop_info = cg.lower_for_loop(&dir);
assert_eq!(cg.loop_count, 1);
assert_eq!(loop_info.schedule, X86OmpScheduleKind::Static);
}
#[test]
fn test_codegen_generate_static_schedule() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let loop_info = X86OmpLoopLowering {
directive: X86OmpDirective::new(X86OmpDirectiveKind::For),
loop_var: "i".to_string(),
lower_bound: "0".to_string(),
upper_bound: "100".to_string(),
stride: "1".to_string(),
chunk_size: 4,
schedule: X86OmpScheduleKind::Static,
ordered_clause: X86OrderedKind::None,
collapse_depth: 1,
simd_clause: X86SimdClause::default(),
nowait: false,
body_label: ".body".to_string(),
epilogue_label: ".epilogue".to_string(),
};
let ir = cg.generate_static_schedule(&loop_info);
assert!(!ir.is_empty());
assert!(ir.iter().any(|s| s.contains(kmpc_x86::FOR_STATIC_INIT_4)));
assert!(ir.iter().any(|s| s.contains(kmpc_x86::FOR_STATIC_FINI)));
}
#[test]
fn test_codegen_generate_dynamic_schedule() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let loop_info = X86OmpLoopLowering {
directive: X86OmpDirective::new(X86OmpDirectiveKind::For),
loop_var: "i".to_string(),
lower_bound: "0".to_string(),
upper_bound: "100".to_string(),
stride: "1".to_string(),
chunk_size: 8,
schedule: X86OmpScheduleKind::Dynamic,
ordered_clause: X86OrderedKind::None,
collapse_depth: 1,
simd_clause: X86SimdClause::default(),
nowait: false,
body_label: ".body".to_string(),
epilogue_label: ".epilogue".to_string(),
};
let ir = cg.generate_dynamic_schedule(&loop_info);
assert!(!ir.is_empty());
assert!(ir.iter().any(|s| s.contains(kmpc_x86::DISPATCH_INIT_4)));
assert!(ir.iter().any(|s| s.contains(kmpc_x86::DISPATCH_NEXT_4)));
}
#[test]
fn test_codegen_lower_barrier() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let ir = cg.lower_barrier();
assert!(ir.iter().any(|s| s.contains(kmpc_x86::BARRIER)));
}
#[test]
fn test_codegen_lower_critical() {
let ctx = X86OpenMPFull::new_x86_64();
let mut cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective {
critical_name: Some("my_lock".to_string()),
..X86OmpDirective::new(X86OmpDirectiveKind::Critical)
};
let crit = cg.lower_critical(&dir);
assert!(crit.name.as_deref() == Some("my_lock"));
let ir = cg.generate_critical_code(&crit);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::CRITICAL)));
assert!(ir.iter().any(|s| s.contains(kmpc_x86::END_CRITICAL)));
}
#[test]
fn test_codegen_lower_task() {
let ctx = X86OpenMPFull::new_x86_64();
let mut cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective::new(X86OmpDirectiveKind::Task);
let task = cg.lower_task(&dir);
assert_eq!(cg.task_count, 1);
assert!(task.outlined_fn_name.starts_with("__omp_task_"));
}
#[test]
fn test_codegen_lower_target() {
let ctx = X86OpenMPFull::new_x86_64();
let mut cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective::new(X86OmpDirectiveKind::Target);
let target = cg.lower_target(&dir);
assert_eq!(cg.target_count, 1);
assert!(target.outlined_fn_name.starts_with("__omp_target_"));
}
#[test]
fn test_codegen_lower_simd() {
let ctx = X86OpenMPFull::new_x86_64();
let mut cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective {
simd_clause: Some(X86SimdClause {
simdlen: Some(8),
..Default::default()
}),
..X86OmpDirective::new(X86OmpDirectiveKind::Simd)
};
let simd = cg.lower_simd_loop(&dir);
assert_eq!(cg.simd_count, 1);
}
#[test]
fn test_codegen_lower_reduction() {
let ctx = X86OpenMPFull::new_x86_64();
let mut cg = X86OpenMPCodeGen::new(ctx);
let red = cg.lower_reduction(X86ReductionOp::Add, "sum", "i32");
assert_eq!(red.reduction_op, X86ReductionOp::Add);
assert_eq!(red.target_var, "sum");
assert!(!red.local_var.is_empty());
}
#[test]
fn test_codegen_reduction_init_code() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let red = X86OmpReductionLowering {
reduction_op: X86ReductionOp::Add,
target_var: "sum".to_string(),
local_var: "_sum_omp_local".to_string(),
var_type: "i32".to_string(),
init_value: "0".to_string(),
combiner_fn: "__omp_reduction_combiner_sum_add".to_string(),
finalizer_fn: "__omp_reduction_finalizer_sum_add".to_string(),
};
let ir = cg.generate_reduction_init_code(&red);
assert!(!ir.is_empty());
assert!(ir.iter().any(|s| s.contains("alloca")));
}
#[test]
fn test_codegen_stats() {
let ctx = X86OpenMPFull::new_x86_64();
let mut cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective::new(X86OmpDirectiveKind::Parallel);
cg.lower_parallel(&dir);
let dir2 = X86OmpDirective::new(X86OmpDirectiveKind::For);
cg.lower_for_loop(&dir2);
let stats = cg.stats();
assert_eq!(stats.parallel_regions, 1);
assert_eq!(stats.worksharing_loops, 1);
let display = format!("{}", stats);
assert!(display.contains("parallel=1"));
assert!(display.contains("loops=1"));
}
#[test]
fn test_codegen_lower_directive_dispatch() {
let ctx = X86OpenMPFull::new_x86_64();
let mut cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective::new(X86OmpDirectiveKind::Barrier);
let ir = cg.lower_directive(&dir);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::BARRIER)));
}
#[test]
fn test_codegen_lower_flush() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let ir = cg.lower_flush(None);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::FLUSH)));
}
#[test]
fn test_codegen_lower_threadprivate() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let ir = cg.lower_threadprivate(&["A".to_string()]);
assert!(ir
.iter()
.any(|s| s.contains(kmpc_x86::THREADPRIVATE_REGISTER)));
}
#[test]
fn test_irgen_new() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
assert_eq!(irgen.outlined_functions.len(), 0);
assert_eq!(irgen.instruction_counter, 0);
}
#[test]
fn test_irgen_outlined_parallel_function() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let captured = vec![X86CapturedVariable {
name: "A".to_string(),
sharing: X86DataSharingAttr::Shared,
var_type: "i64".to_string(),
is_pointer: true,
reduction_op: None,
linear_step: Some(0),
}];
let outlined =
irgen.generate_outlined_parallel_function("__omp_outlined_test", &captured, false, &[]);
assert!(outlined.name.contains("test"));
assert_eq!(outlined.shared_vars.len(), 1);
assert_eq!(irgen.outlined_functions.len(), 1);
}
#[test]
fn test_irgen_emit_signature() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let captured = vec![X86CapturedVariable {
name: "x".to_string(),
sharing: X86DataSharingAttr::Shared,
var_type: "i64".to_string(),
is_pointer: true,
reduction_op: None,
linear_step: Some(0),
}];
let outlined =
irgen.generate_outlined_parallel_function("__omp_test", &captured, false, &[]);
let sig = irgen.emit_outlined_fn_signature(&outlined);
assert!(sig.iter().any(|s| s.contains("define internal void")));
assert!(sig.iter().any(|s| s.contains("__omp_test")));
assert!(sig.iter().any(|s| s.contains("entry:")));
}
#[test]
fn test_irgen_emit_epilogue() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let outlined = X86OmpIROutlinedFunction {
name: "__omp_test".to_string(),
shared_vars: vec![],
body_label: "entry".to_string(),
has_barrier: true,
has_reduction: false,
reduction_ops: vec![],
nowait: false,
};
let epi = irgen.emit_outlined_fn_epilogue(&outlined);
assert!(epi.iter().any(|s| s.contains("ret void")));
assert!(epi.iter().any(|s| s.contains("}")));
assert!(epi.iter().any(|s| s.contains(kmpc_x86::BARRIER)));
}
#[test]
fn test_irgen_private_var_init() {
let ctx = X86OpenMPFull::new_x86_64();
let irgen = X86OpenMPIRGen::new(ctx);
let ir = irgen.generate_private_var_init("i", "i32", "0");
assert!(ir.iter().any(|s| s.contains("i.private")));
assert!(ir.iter().any(|s| s.contains("alloca i32")));
}
#[test]
fn test_irgen_firstprivate_init() {
let ctx = X86OpenMPFull::new_x86_64();
let irgen = X86OpenMPIRGen::new(ctx);
let ir = irgen.generate_firstprivate_init("x", "i32");
assert!(ir.iter().any(|s| s.contains("fp.val")));
assert!(ir.iter().any(|s| s.contains("_.captured.x")));
}
#[test]
fn test_irgen_reduction_init() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let red = X86OmpIRReduction {
kind: X86ReductionOp::Add,
original_var: "sum".to_string(),
local_var: "%omp_red_sum".to_string(),
type_name: "i32".to_string(),
init_expr: "0".to_string(),
combiner_label: "red_combiner_0".to_string(),
finalize_label: "red_finalize_0".to_string(),
};
let ir = irgen.generate_reduction_init(&red);
assert!(ir.iter().any(|s| s.contains("alloca")));
}
#[test]
fn test_irgen_reduction_combiner() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let red = X86OmpIRReduction {
kind: X86ReductionOp::Add,
original_var: "sum".to_string(),
local_var: "%omp_red_sum".to_string(),
type_name: "i32".to_string(),
init_expr: "0".to_string(),
combiner_label: "red_combiner_0".to_string(),
finalize_label: "red_finalize_0".to_string(),
};
let ir = irgen.generate_reduction_combiner(&red);
assert!(ir.iter().any(|s| s.contains("add")));
}
#[test]
fn test_irgen_reduction_finalize() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let red = X86OmpIRReduction {
kind: X86ReductionOp::Add,
original_var: "sum".to_string(),
local_var: "%omp_red_sum".to_string(),
type_name: "i32".to_string(),
init_expr: "0".to_string(),
combiner_label: "red_combiner_0".to_string(),
finalize_label: "red_finalize_0".to_string(),
};
let ir = irgen.generate_reduction_finalize(&red);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::REDUCE)));
assert!(ir.iter().any(|s| s.contains(kmpc_x86::END_REDUCE)));
}
#[test]
fn test_irgen_reduction_pipeline() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let reductions = vec![("sum".to_string(), X86ReductionOp::Add)];
let ir = irgen.generate_reduction_pipeline(&reductions, "i32");
assert!(!ir.is_empty());
assert!(irgen.reductions.len() == 1);
}
#[test]
fn test_irgen_static_loop_ir() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let loop_info = X86OmpIRLoopInfo {
induction_var: "i".to_string(),
lower_bound: 0,
upper_bound: 100,
stride: 1,
chunk_size: 4,
schedule_id: 1,
is_static: true,
has_nowait: false,
has_ordered: false,
collapse_count: 1,
};
let ir = irgen.generate_static_loop_ir(&loop_info);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::FOR_STATIC_INIT_4)));
assert!(ir.iter().any(|s| s.contains(kmpc_x86::FOR_STATIC_FINI)));
}
#[test]
fn test_irgen_dynamic_loop_ir() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let loop_info = X86OmpIRLoopInfo {
induction_var: "i".to_string(),
lower_bound: 0,
upper_bound: 100,
stride: 1,
chunk_size: 8,
schedule_id: 2,
is_static: false,
has_nowait: true,
has_ordered: false,
collapse_count: 1,
};
let ir = irgen.generate_dynamic_loop_ir(&loop_info);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::DISPATCH_INIT_4)));
assert!(ir.iter().any(|s| s.contains(kmpc_x86::DISPATCH_NEXT_4)));
assert!(!ir.iter().any(|s| s.contains(kmpc_x86::BARRIER))); }
#[test]
fn test_irgen_guided_loop_ir() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let loop_info = X86OmpIRLoopInfo {
induction_var: "i".to_string(),
lower_bound: 0,
upper_bound: 1000,
stride: 1,
chunk_size: 16,
schedule_id: 3,
is_static: false,
has_nowait: false,
has_ordered: false,
collapse_count: 1,
};
let ir = irgen.generate_guided_loop_ir(&loop_info);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::DISPATCH_INIT_4)));
assert!(ir.iter().any(|s| s.contains("36"))); }
#[test]
fn test_irgen_simd_loop_ir() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let simd = X86OmpIRSimd {
vector_width: 8,
element_type: "float".to_string(),
safelen_clause: None,
simdlen_clause: Some(8),
aligned_ptrs: vec![],
nontemporal_stores: vec![],
linear_steps: vec![],
private_vars: vec![],
reductions: vec![],
};
let ir = irgen.generate_simd_loop_ir(&simd);
assert!(ir.iter().any(|s| s.contains("<8 x float>")));
assert!(ir.iter().any(|s| s.contains("vec_load")));
}
#[test]
fn test_irgen_task_ir() {
let ctx = X86OpenMPFull::new_x86_64();
let irgen = X86OpenMPIRGen::new(ctx);
let ir = irgen.generate_task_ir("__omp_test_task", 0);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::TASK_ALLOC)));
assert!(ir.iter().any(|s| s.contains(kmpc_x86::TASK)));
}
#[test]
fn test_irgen_target_ir() {
let ctx = X86OpenMPFull::new_x86_64();
let irgen = X86OpenMPIRGen::new(ctx);
let ir = irgen.generate_target_ir("__omp_test_target", 0);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::TARGET_INIT)));
assert!(ir.iter().any(|s| s.contains(kmpc_x86::TARGET_DEINIT)));
}
#[test]
fn test_irgen_create_simd_info() {
let ctx = X86OpenMPFull::new_x86_64();
let irgen = X86OpenMPIRGen::new(ctx);
let clause = X86SimdClause {
simdlen: Some(4),
safelen: Some(16),
aligned_vars: vec!["A".to_string()],
..Default::default()
};
let info = irgen.create_simd_info(&clause);
assert_eq!(info.simdlen_clause, Some(4));
assert_eq!(info.safelen_clause, Some(16));
assert!(info.aligned_ptrs.contains(&"A".to_string()));
}
#[test]
fn test_irgen_compile_directives() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let dirs = vec![
X86OmpDirective::new(X86OmpDirectiveKind::Parallel),
X86OmpDirective::new(X86OmpDirectiveKind::For)
.with_schedule(X86OmpScheduleKind::Static),
];
let ir = irgen.compile_directives(&dirs);
assert!(!ir.is_empty());
assert!(ir.iter().any(|s| s.contains("ModuleID")));
}
#[test]
fn test_runtime_new() {
let ctx = X86OpenMPFull::new_x86_64();
let rt = X86OpenMPRuntime::new(ctx);
assert!(rt.function_count() > 0);
assert_eq!(rt.thread_teams.len(), 0);
assert_eq!(rt.work_queues.len(), 0);
}
#[test]
fn test_runtime_function_lookup() {
let ctx = X86OpenMPFull::new_x86_64();
let rt = X86OpenMPRuntime::new(ctx);
assert!(rt.has_function(kmpc_x86::FORK_CALL));
assert!(rt.has_function(kmpc_x86::BARRIER));
assert!(rt.has_function(kmpc_x86::TASK));
assert!(rt.has_function(kmpc_x86::TARGET));
assert!(!rt.has_function("nonexistent"));
}
#[test]
fn test_runtime_get_function() {
let ctx = X86OpenMPFull::new_x86_64();
let rt = X86OpenMPRuntime::new(ctx);
let fn_info = rt.get_function(kmpc_x86::FORK_CALL);
assert!(fn_info.is_some());
let info = fn_info.unwrap();
assert_eq!(info.name, "__kmpc_fork_call");
assert_eq!(info.return_type, "void");
}
#[test]
fn test_runtime_thread_team() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.start_thread_team(8, Some(X86ProcBind::Close));
assert_eq!(rt.team_count(), 1);
let team = rt.current_team().unwrap();
assert_eq!(team.num_threads, 8);
assert_eq!(team.proc_bind, Some(X86ProcBind::Close));
assert!(!team.nested);
rt.end_thread_team();
assert!(!rt.current_team().unwrap().is_active);
}
#[test]
fn test_runtime_nested_teams() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.start_thread_team(4, None);
assert_eq!(rt.team_count(), 1);
rt.start_thread_team(2, None);
assert_eq!(rt.team_count(), 2);
assert!(rt.current_team().unwrap().nested);
rt.end_thread_team();
rt.end_thread_team();
assert_eq!(rt.team_count(), 2);
}
#[test]
fn test_runtime_work_queue_static() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.create_work_queue(0, 100, 1, 25, 1);
assert_eq!(rt.work_queues.len(), 1);
let chunk = rt.next_chunk(0);
assert!(chunk.is_some());
}
#[test]
fn test_runtime_work_queue_dynamic() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.create_work_queue(0, 100, 1, 10, 35);
let mut total = 0;
while let Some((start, end)) = rt.next_chunk(0) {
total += (end - start) / 1 + 1;
if total > 200 {
break;
}
}
assert!(total > 0);
}
#[test]
fn test_runtime_task_dep_graph() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.add_task_node(0, "x", X86DependType::In);
rt.add_task_node(1, "x", X86DependType::Out);
rt.add_dependency(0, 1);
assert_eq!(rt.dep_graph.nodes.len(), 2);
rt.complete_task(0);
assert!(rt.dep_graph.completed.contains(&0));
}
#[test]
fn test_runtime_cancel() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.set_cancellation(true);
assert!(rt.cancellation_enabled);
rt.set_cancel("parallel");
assert!(rt.is_cancelled("parallel"));
assert!(!rt.is_cancelled("loop"));
rt.reset_cancel();
assert!(!rt.is_cancelled("parallel"));
}
#[test]
fn test_runtime_threadprivate() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.register_threadprivate("A");
assert!(rt.is_threadprivate("A"));
assert!(!rt.is_threadprivate("B"));
}
#[test]
fn test_runtime_critical_lock() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
let id = rt.register_critical_lock("my_lock");
assert_eq!(rt.get_critical_lock("my_lock"), Some(id));
}
#[test]
fn test_runtime_function_names() {
let ctx = X86OpenMPFull::new_x86_64();
let rt = X86OpenMPRuntime::new(ctx);
let names = rt.function_names();
assert!(names.iter().any(|n| *n == kmpc_x86::FORK_CALL));
assert!(names.iter().any(|n| *n == kmpc_x86::BARRIER));
}
#[test]
fn test_runtime_status_report() {
let ctx = X86OpenMPFull::new_x86_64();
let rt = X86OpenMPRuntime::new(ctx);
let report = rt.status_report();
assert!(report.contains("X86OpenMPRuntime"));
assert!(report.contains("functions"));
}
#[test]
fn test_full_pipeline_parallel_for() {
let ctx = X86OpenMPFull::new_x86_64();
let mut parser = X86OmpDirectiveParser::new();
let mut codegen = X86OpenMPCodeGen::new(ctx);
let mut irgen = X86OpenMPIRGen::new(X86OpenMPFull::new_x86_64());
let result = parser.parse_directive(
"omp parallel for private(i) shared(A) schedule(static,4) num_threads(8)",
);
assert!(result.is_ok());
let dir = result.unwrap();
let (region, loop_info) = codegen.lower_parallel_for(&dir);
assert!(region.outlined_fn_name.starts_with("__omp_parallel_"));
assert_eq!(loop_info.schedule, X86OmpScheduleKind::Static);
let fork_call = codegen.generate_fork_call(®ion, ®ion.outlined_fn_name);
assert!(!fork_call.is_empty());
let schedule_ir = codegen.generate_schedule_code(&loop_info);
assert!(!schedule_ir.is_empty());
let ir = irgen.compile_directives(&[dir]);
assert!(!ir.is_empty());
}
#[test]
fn test_full_pipeline_target() {
let ctx = X86OpenMPFull::new_x86_64();
let mut parser = X86OmpDirectiveParser::new();
let mut codegen = X86OpenMPCodeGen::new(ctx);
let result = parser.parse_directive("omp target map(to: A) map(from: B) device(0)");
assert!(result.is_ok());
let dir = result.unwrap();
let target = codegen.lower_target(&dir);
assert!(target.outlined_fn_name.starts_with("__omp_target_"));
assert_eq!(target.map_clauses.len(), 2);
assert_eq!(target.device_id, Some(0));
let target_ir = codegen.generate_target_code(&target);
assert!(target_ir.iter().any(|s| s.contains(kmpc_x86::TARGET)));
}
#[test]
fn test_full_pipeline_task_with_deps() {
let ctx = X86OpenMPFull::new_x86_64();
let mut parser = X86OmpDirectiveParser::new();
let mut codegen = X86OpenMPCodeGen::new(ctx);
let result = parser.parse_directive("omp task depend(in: x) depend(out: y) priority(10)");
assert!(result.is_ok());
let dir = result.unwrap();
let task = codegen.lower_task(&dir);
assert_eq!(codegen.task_count, 1);
assert!(task.outlined_fn_name.starts_with("__omp_task_"));
let task_ir = codegen.generate_task_code(&task);
assert!(task_ir.iter().any(|s| s.contains(kmpc_x86::TASK_ALLOC)));
}
#[test]
fn test_full_pipeline_simd() {
let ctx = X86OpenMPFull::new_x86_64();
let mut parser = X86OmpDirectiveParser::new();
let mut codegen = X86OpenMPCodeGen::new(ctx);
let result = parser.parse_directive("omp simd simdlen(8) safelen(16)");
assert!(result.is_ok());
let dir = result.unwrap();
let simd = codegen.lower_simd_loop(&dir);
assert_eq!(codegen.simd_count, 1);
let simd_ir = codegen.generate_simd_code(&simd);
assert!(!simd_ir.is_empty());
}
#[test]
fn test_full_pipeline_reduction() {
let ctx = X86OpenMPFull::new_x86_64();
let mut codegen = X86OpenMPCodeGen::new(ctx);
let red = codegen.lower_reduction(X86ReductionOp::Add, "result", "f64");
assert_eq!(red.reduction_op, X86ReductionOp::Add);
let init_ir = codegen.generate_reduction_init_code(&red);
assert!(!init_ir.is_empty());
let finalize_ir = codegen.generate_reduction_finalize_code(&red);
assert!(finalize_ir.iter().any(|s| s.contains(kmpc_x86::REDUCE)));
}
#[test]
fn test_x86_specific_simd_ir() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let avx2_ir = irgen.generate_x86_simd_ir("float", 8, false);
assert!(avx2_ir.iter().any(|s| s.contains("AVX")));
let avx512_ctx = X86OpenMPFull::new_x86_64_avx512();
let mut avx512_irgen = X86OpenMPIRGen::new(avx512_ctx);
let avx512_ir = avx512_irgen.generate_x86_simd_ir("double", 8, true);
assert!(avx512_ir.iter().any(|s| s.contains("512")));
}
#[test]
fn test_schedule_id_match() {
assert_eq!(X86OmpScheduleKind::Static.to_runtime_id(), 1);
assert_eq!(X86OmpScheduleKind::Dynamic.to_runtime_id(), 2);
assert_eq!(X86OmpScheduleKind::Guided.to_runtime_id(), 3);
let dir = X86OmpDirective::new(X86OmpDirectiveKind::For)
.with_schedule(X86OmpScheduleKind::Static);
let code = dir.get_runtime_call();
assert!(
code.contains("fork_call") || code.contains("static") || code.contains("for_static")
);
}
#[test]
fn test_codegen_default() {
let cg = X86OpenMPCodeGen::default();
assert_eq!(cg.parallel_count, 0);
let irgen = X86OpenMPIRGen::default();
assert_eq!(irgen.outlined_functions.len(), 0);
let rt = X86OpenMPRuntime::default();
assert!(rt.function_count() > 0);
}
#[test]
fn test_simd_level_display() {
assert_eq!(format!("{}", X86SimdLevel::AVX2), "AVX2");
assert_eq!(format!("{}", X86SimdLevel::AVX512F), "AVX-512F");
assert_eq!(format!("{}", X86SimdLevel::SSE2), "SSE2");
}
#[test]
fn test_reduction_op_display() {
assert_eq!(format!("{}", X86ReductionOp::Add), "+");
assert_eq!(format!("{}", X86ReductionOp::Mul), "*");
assert_eq!(format!("{}", X86ReductionOp::LogicalAnd), "&&");
}
}
impl X86OpenMPCodeGen {
pub fn lower_parallel_sections_combined(
&mut self,
directive: &X86OmpDirective,
num_sections: u32,
) -> Vec<String> {
let region = self.lower_parallel(directive);
let mut ir = self.generate_fork_call(®ion, ®ion.outlined_fn_name);
ir.extend(self.lower_sections(directive, num_sections));
ir
}
pub fn lower_target_teams(&mut self, directive: &X86OmpDirective) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Target teams: offloaded hierarchical parallelism".to_string());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::TARGET_INIT
));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid, i32 {}, i32 {})",
kmpc_x86::FORK_TEAMS,
directive.num_threads.as_deref().unwrap_or("0"),
directive.num_threads.as_deref().unwrap_or("0")
));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::TARGET_DEINIT
));
ir
}
pub fn lower_target_teams_distribute_parallel_for(
&mut self,
directive: &X86OmpDirective,
) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Full target teams distribute parallel for pipeline".to_string());
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::TARGET_INIT
));
let loop_info = self.lower_for_loop(directive);
ir.extend(self.generate_schedule_code(&loop_info));
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::TARGET_DEINIT
));
ir
}
pub fn lower_declare_target(&mut self, fn_name: &str) {
self.context.declare_target_fns.push(fn_name.to_string());
}
pub fn lower_declare_simd(&mut self, fn_name: &str, clause: X86SimdClause) {
self.context
.declare_simd_fns
.push((fn_name.to_string(), clause));
}
pub fn lower_atomic_cas(
&self,
ptr: &str,
expected: &str,
desired: &str,
width: u32,
) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!("; Atomic CAS {}: {} -> {}", ptr, expected, desired));
match width {
4 => {
ir.push(format!(
"call i32 @{}(@ident_t* @.loc, i32 %gtid, i32* %{}, i32 {}, i32 {})",
kmpc_x86::ATOMIC_CAS_4,
ptr,
expected,
desired
));
}
8 => {
ir.push(format!(
"call i64 @{}(@ident_t* @.loc, i32 %gtid, i64* %{}, i64 {}, i64 {})",
kmpc_x86::ATOMIC_CAS_8,
ptr,
expected,
desired
));
}
_ => {
ir.push("; Unsupported CAS width".to_string());
}
}
ir
}
pub fn lower_atomic_swap(&self, ptr: &str, val: &str, width: u32) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!("; Atomic swap {}: {}", ptr, val));
match width {
4 => {
ir.push(format!(
"call i32 @{}(@ident_t* @.loc, i32 %gtid, i32* %{}, i32 {})",
kmpc_x86::ATOMIC_SWAP_4,
ptr,
val
));
}
8 => {
ir.push(format!(
"call i64 @{}(@ident_t* @.loc, i32 %gtid, i64* %{}, i64 {})",
kmpc_x86::ATOMIC_SWAP_8,
ptr,
val
));
}
_ => {
ir.push("; Unsupported swap width".to_string());
}
}
ir
}
pub fn lower_loop_transform(
&self,
directive: &X86OmpDirective,
transform: &X86LoopTransformClause,
) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!(
"; Loop transform for {}: tile={:?}, unroll={:?}, interchange={:?}",
directive.kind.name(),
transform.tile_sizes,
transform.unroll_factor,
transform.interchange_indices
));
if transform.unroll_full {
ir.push("; Full loop unrolling requested".to_string());
ir.push("%unroll_hint = alloca i32".to_string());
ir.push("store i32 0, i32* %unroll_hint, !llvm.loop !0".to_string());
} else if let Some(factor) = transform.unroll_factor {
ir.push(format!("; Loop unrolling with factor {}", factor));
}
if !transform.tile_sizes.is_empty() {
ir.push(format!(
"; Loop tiling with sizes: {:?}",
transform.tile_sizes
));
}
if !transform.interchange_indices.is_empty() {
ir.push(format!(
"; Loop interchange: {:?}",
transform.interchange_indices
));
}
ir
}
pub fn lower_requires(
&self,
unified_address: bool,
unified_shared_memory: bool,
reverse_offload: bool,
dynamic_allocators: bool,
atomic_default_mem_order: Option<&str>,
) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Requires directive".to_string());
if unified_address {
ir.push("; unified_address".to_string());
}
if unified_shared_memory {
ir.push("; unified_shared_memory".to_string());
}
if reverse_offload {
ir.push("; reverse_offload".to_string());
}
if dynamic_allocators {
ir.push("; dynamic_allocators".to_string());
}
if let Some(order) = atomic_default_mem_order {
ir.push(format!("; atomic_default_mem_order({})", order));
}
ir
}
pub fn lower_is_device_ptr(&self, vars: &[String]) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; is_device_ptr: device pointer variables".to_string());
for var in vars {
ir.push(format!("; Device pointer: {}", var));
}
ir
}
pub fn lower_masked(&self, filter: Option<u32>) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Masked region: selected threads execute".to_string());
let filter_val = filter.unwrap_or(0);
if filter_val > 0 {
ir.push(format!("; Filter: only thread {} executes", filter_val));
}
ir.push("%tid = call i32 @omp_get_thread_num()".to_string());
ir.push(format!("%is_masked = icmp eq i32 %tid, {}", filter_val));
ir.push("br i1 %is_masked, label %masked_body, label %masked_end".to_string());
ir.push("masked_body:".to_string());
ir.push("masked_end:".to_string());
ir
}
pub fn lower_scope(&self, directive: &X86OmpDirective) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Scope region: structured block with data environment".to_string());
if !directive.private_vars.is_empty() {
ir.push(format!("; Private vars: {:?}", directive.private_vars));
}
if directive.nowait {
ir.push("; nowait: no barrier after scope".to_string());
} else {
ir.push(format!(
"call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::BARRIER
));
}
ir
}
pub fn lower_nothing(&self) -> Vec<String> {
vec!["; Nothing: no operation".to_string()]
}
pub fn lower_error(&self, severity: &str, message: &str) -> Vec<String> {
vec![
format!("; Error {}: {}", severity, message),
format!(
"call void @llvm.trap() ; fatal error {}: {}",
severity, message
),
]
}
pub fn lower_metadirective(
&self,
when_clauses: &[(String, String)],
default_variant: Option<&str>,
) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Metadirective: compile-time context selection".to_string());
for (context, variant) in when_clauses {
ir.push(format!("; when({}): {}", context, variant));
}
if let Some(def) = default_variant {
ir.push(format!("; default: {}", def));
}
ir
}
pub fn lower_loop_directive(&mut self, directive: &X86OmpDirective) -> Vec<String> {
let loop_info = self.lower_for_loop(directive);
self.generate_schedule_code(&loop_info)
}
pub fn lower_parallel_masked(
&mut self,
directive: &X86OmpDirective,
filter: Option<u32>,
) -> Vec<String> {
let mut ir = Vec::new();
let region = self.lower_parallel(directive);
ir.extend(self.generate_fork_call(®ion, ®ion.outlined_fn_name));
ir.extend(self.lower_masked(filter));
ir
}
pub fn generate_x86_barrier_optimized(&self) -> Vec<String> {
vec![
"; X86 optimized barrier with PAUSE hint".to_string(),
" ; Use MONITOR/MWAIT or PAUSE for power-efficient spinning".to_string(),
"barrier_spin:".to_string(),
format!(" call void asm sideeffect \"pause\", \"~{memory}\"()"),
format!(
" call void @{}(@ident_t* @.loc, i32 %gtid) ; actual barrier",
kmpc_x86::BARRIER
),
]
}
pub fn generate_x86_parallel_detection(&self) -> Vec<String> {
vec![
"; X86: detect optimal thread count from CPU topology".to_string(),
" ; Use CPUID to detect logical processors".to_string(),
" %num_logical = call i32 @omp_get_num_procs()".to_string(),
" %num_threads = select i1 %has_num_threads, i32 %requested, i32 %num_logical"
.to_string(),
]
}
}
impl X86OpenMPIRGen {
pub fn generate_collapse_loop_ir(
&mut self,
loop_info: &X86OmpIRLoopInfo,
collapse_depth: u32,
extents: &[i64],
) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!("; Collapsed loop: depth {}", collapse_depth));
let mut total = 1i64;
for e in extents {
total = total.saturating_mul(*e);
}
ir.push(format!(" ; Total iterations: {}", total));
ir.push(" %flat_iter = phi i64 [ 0, %entry ], [ %next_flat, %collapse_body ]".to_string());
for d in 0..collapse_depth {
ir.push(format!(
" %idx_{} = urem i64 %flat_div_{}, {}",
d,
d,
extents.get(d as usize).unwrap_or(&1)
));
if d < collapse_depth - 1 {
ir.push(format!(
" %flat_div_{} = udiv i64 %flat_iter, {}",
d + 1,
extents.get(d as usize).unwrap_or(&1)
));
}
}
ir.push(" %next_flat = add i64 %flat_iter, 1".to_string());
ir.push(format!(" %coll_cond = icmp slt i64 %next_flat, {}", total));
ir.push(" br i1 %coll_cond, label %collapse_body, label %collapse_exit".to_string());
ir
}
pub fn generate_threadprivate_ir(&self, var_name: &str) -> Vec<String> {
vec![
format!("; Threadprivate: {}", var_name),
format!(
" call void @{}(@ident_t* @.loc, i32 %gtid, i8* @{})",
kmpc_x86::THREADPRIVATE_REGISTER,
var_name
),
]
}
pub fn generate_copyin_ir(&self, vars: &[String]) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Copyin: master thread propagates threadprivate values".to_string());
ir.push(format!(
" %is_master = call i32 @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::MASTER_BEGIN
));
for var in vars {
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid, i8* @{})",
kmpc_x86::THREADPRIVATE_CACHED,
var
));
}
ir
}
pub fn generate_single_copyprivate_ir(&self, copyprivate_vars: &[String]) -> Vec<String> {
let mut ir = Vec::new();
ir.push("; Single + copyprivate: broadcast values".to_string());
ir.push(format!(
" %is_single = call i32 @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::SINGLE_BEGIN
));
ir.push(" br i1 %is_single, label %single_body, label %single_after".to_string());
ir.push("single_body:".to_string());
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::SINGLE_END
));
ir.push("single_after:".to_string());
for var in copyprivate_vars {
ir.push(format!(
" ; Broadcast {} from single thread to all others",
var
));
}
ir
}
pub fn generate_distribute_static_ir(&mut self, loop_info: &X86OmpIRLoopInfo) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!(
"; Distribute static loop: chunk={}",
loop_info.chunk_size
));
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid, i32 33, i32* %plastiter, i64* %plower, i64* %pupper, i64* %pstride, i64 1, i64 {})",
kmpc_x86::DISTRIBUTE_STATIC_INIT,
loop_info.chunk_size
));
ir.push(" %dist_lower = load i64, i64* %plower".to_string());
ir.push(" %dist_upper = load i64, i64* %pupper".to_string());
ir.push(format!(
" call void @{}(@ident_t* @.loc, i32 %gtid)",
kmpc_x86::DISTRIBUTE_STATIC_FINI
));
ir
}
pub fn generate_outlined_task_body(&mut self, task: &X86OmpTaskLowering) -> Vec<String> {
let mut ir = Vec::new();
ir.push(format!(
"define internal void @{}(i32* %.global_tid., i32* %.bound_tid.) {{",
task.outlined_fn_name
));
ir.push("entry:".to_string());
ir.push(" %gtid = load i32, i32* %.global_tid.".to_string());
for var in &task.captured_vars {
ir.push(format!(
" ; Captured var: {} ({})",
var.name,
var.sharing.name()
));
}
if task.is_untied {
ir.push(
" ; Untied task: can be suspended and resumed on different thread".to_string(),
);
}
ir.push(" ret void".to_string());
ir.push("}".to_string());
ir
}
pub fn generate_task_modify_flags_ir(
&self,
task_ptr: &str,
mergeable: bool,
untied: bool,
final_flag: bool,
) -> Vec<String> {
let mut flags: u32 = 0;
if mergeable {
flags |= 1;
}
if untied {
flags |= 2;
}
if final_flag {
flags |= 4;
}
vec![
format!("; Task modify flags: {}", flags),
format!(
" call void @{}(@ident_t* @.loc, i32 %gtid, i8* {}, i32 {})",
kmpc_x86::TASK_MODIFY_FLAGS,
task_ptr,
flags
),
]
}
pub fn generate_offloading_entry_ir(
&self,
entry_name: &str,
entry_addr: &str,
entry_size: u64,
entry_flags: u32,
) -> Vec<String> {
vec![
format!("; Offloading entry: {}", entry_name),
format!(
"@{} = private constant %__tgt_offload_entry {{ i8* {}, i8* getelementptr... }}",
entry_name, entry_addr
),
format!(
" %entry_ptr = getelementptr %__tgt_offload_entry, %__tgt_offload_entry* @{}, i32 0",
entry_name
),
format!(
" call void @{}(i8* %entry_ptr, i32 1)",
tgt_x86::REGISTER_LIB
),
]
}
}
impl X86OpenMPRuntime {
pub fn initialize_from_environment(&mut self) {
let icv = X86ICVStorage::from_environment();
self.context.icv = icv;
self.cancellation_enabled = self.context.icv.cancellation_var != 0;
if self.verbose {
eprintln!("[X86OpenMPRuntime] Initialized from environment");
eprintln!(" num_threads={}", self.context.icv.nthreads_var);
eprintln!(" dyn_var={}", self.context.icv.dyn_var);
eprintln!(" nest_var={}", self.context.icv.nest_var);
eprintln!(" cancellation={}", self.cancellation_enabled);
}
}
pub fn simulate_fork_call(&mut self, num_args: u32, microtask_name: &str) {
let num_threads = self.context.icv.effective_num_threads();
self.start_thread_team(num_threads, self.context.icv.proc_bind_var);
if self.verbose {
eprintln!(
"[X86OpenMPRuntime] Fork: {} ({} args, {} threads)",
microtask_name, num_args, num_threads
);
}
self.end_thread_team();
}
pub fn barrier_with_cancel(&mut self, cancel_kind: &str) -> bool {
if self.cancellation_enabled && self.is_cancelled(cancel_kind) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Cancelled at barrier: {}", cancel_kind);
}
return true;
}
if self.verbose {
eprintln!("[X86OpenMPRuntime] Barrier");
}
false
}
pub fn enter_critical(&mut self, name: &str) -> u64 {
let id = self.register_critical_lock(name);
if self.verbose {
eprintln!("[X86OpenMPRuntime] Enter critical: {} (lock {})", name, id);
}
id
}
pub fn exit_critical(&mut self, name: &str) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Exit critical: {}", name);
}
}
pub fn allocate_task(&mut self, flags: u32, task_entry: &str) -> u64 {
let task_id = self.dep_graph.nodes.len() as u64 + 1;
if self.verbose {
eprintln!(
"[X86OpenMPRuntime] Allocate task {}: flags={} entry={}",
task_id, flags, task_entry
);
}
task_id
}
pub fn submit_task(&mut self, task_id: u64) {
self.dep_graph.ready_queue.push(task_id);
if self.verbose {
eprintln!("[X86OpenMPRuntime] Submit task {}", task_id);
}
}
pub fn wait_tasks(&mut self) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Taskwait: waiting for child tasks");
}
self.dep_graph.ready_queue.clear();
}
pub fn begin_taskgroup(&mut self) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Taskgroup: begin");
}
}
pub fn end_taskgroup(&mut self) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Taskgroup: end — synchronizing");
}
}
pub fn execute_target(&mut self, device_id: i32, binary_desc: &str) -> i32 {
if self.verbose {
eprintln!(
"[X86OpenMPRuntime] Target: device={} binary={}",
device_id, binary_desc
);
}
0
}
pub fn begin_target_data(&mut self, device_id: i32) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Target data begin: device={}", device_id);
}
}
pub fn end_target_data(&mut self, device_id: i32) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Target data end: device={}", device_id);
}
}
pub fn update_target_data(&mut self, device_id: i32) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Target update: device={}", device_id);
}
}
pub fn perform_reduction(&mut self, reduction_var: &str, op: X86ReductionOp) {
let local_var = format!("%omp_red_{}", reduction_var);
let red = X86OmpIRReduction {
kind: op,
original_var: reduction_var.to_string(),
local_var,
type_name: "i64".to_string(),
init_expr: op.neutral_value().map(|v| v.to_string()),
combiner_label: format!("__omp_red_comb_{}", reduction_var),
finalize_label: format!("__omp_red_fini_{}", reduction_var),
};
self.pending_reductions.push(red);
if self.verbose {
eprintln!(
"[X86OpenMPRuntime] Reduction: {} ({})",
reduction_var,
op.name()
);
}
}
pub fn complete_reduction(&mut self, reduction_var: &str) {
self.pending_reductions
.retain(|r| r.original_var != reduction_var);
if self.verbose {
eprintln!("[X86OpenMPRuntime] End reduction: {}", reduction_var);
}
}
pub fn flush_memory(&mut self, var: Option<&str>) {
if self.verbose {
if let Some(v) = var {
eprintln!("[X86OpenMPRuntime] Flush: {}", v);
} else {
eprintln!("[X86OpenMPRuntime] Flush: all");
}
}
}
pub fn cancel_operation(&mut self, kind: &str) -> i32 {
self.set_cancel(kind);
if self.cancellation_enabled {
1
} else {
0
}
}
pub fn check_cancellation_point(&mut self, kind: &str) -> i32 {
if self.cancellation_enabled && self.is_cancelled(kind) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Cancellation point activated: {}", kind);
}
1
} else {
0
}
}
pub fn init_sections(&mut self) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Sections init");
}
}
pub fn next_section(&mut self) -> i32 {
1
}
pub fn set_proc_bind(&mut self, bind: X86ProcBind) {
self.context.icv.proc_bind_var = Some(bind);
if self.verbose {
eprintln!("[X86OpenMPRuntime] Push proc_bind: {}", bind.name());
}
}
pub fn push_num_threads(&mut self, n: u32) {
self.context.icv.nthreads_var = n;
if self.verbose {
eprintln!("[X86OpenMPRuntime] Push num_threads: {}", n);
}
}
pub fn global_thread_num(&self) -> i32 {
self.context.icv.active_levels_var as i32
}
pub fn bound_thread_num(&self) -> i32 {
self.context.icv.levels_var as i32
}
pub fn execute_taskloop(&mut self, lower: i64, upper: i64, grainsize: i64, num_tasks: i64) {
if self.verbose {
eprintln!(
"[X86OpenMPRuntime] Taskloop: range=[{}, {}] grainsize={} num_tasks={}",
lower, upper, grainsize, num_tasks
);
}
let total_iters = (upper - lower) + 1;
let task_count = if num_tasks > 0 {
num_tasks.min(total_iters)
} else if grainsize > 0 {
(total_iters + grainsize - 1) / grainsize
} else {
1
};
for t in 0..task_count {
let task_lower = lower + t * (total_iters / task_count);
let task_upper = if t == task_count - 1 {
upper
} else {
lower + (t + 1) * (total_iters / task_count) - 1
};
if self.verbose {
eprintln!(" Task {}: range=[{}, {}]", t, task_lower, task_upper);
}
}
}
pub fn get_num_devices(&self) -> i32 {
1
}
pub fn get_default_device(&self) -> i32 {
self.context.icv.default_device_var
}
pub fn set_default_device(&mut self, dev: i32) {
self.context.icv.default_device_var = dev;
}
pub fn is_initial_device(&self) -> i32 {
1
}
pub fn get_initial_device(&self) -> i32 {
0
}
pub fn associate_ptr(&self, host_ptr: &str, device_ptr: &str) {
if self.verbose {
eprintln!(
"[X86OpenMPRuntime] Associate: {} <-> {}",
host_ptr, device_ptr
);
}
}
pub fn disassociate_ptr(&self, host_ptr: &str) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Disassociate: {}", host_ptr);
}
}
pub fn target_memcpy_rect(&self, dst: &str, src: &str) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Target memcpy rect: {} <- {}", dst, src);
}
}
pub fn target_alloc(&self, size: u64) -> u64 {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Target alloc: {} bytes", size);
}
0 }
pub fn target_free(&self, dev_ptr: u64) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Target free: {:x}", dev_ptr);
}
}
pub fn register_offload_lib(&self, lib_desc: &str) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Register lib: {}", lib_desc);
}
}
pub fn unregister_offload_lib(&self, lib_desc: &str) {
if self.verbose {
eprintln!("[X86OpenMPRuntime] Unregister lib: {}", lib_desc);
}
}
}
#[cfg(test)]
mod extended_tests {
use super::*;
#[test]
fn test_codegen_parallel_sections_combined() {
let ctx = X86OpenMPFull::new_x86_64();
let mut cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective::new(X86OmpDirectiveKind::ParallelSections);
let ir = cg.lower_parallel_sections_combined(&dir, 3);
assert!(!ir.is_empty());
assert!(ir.iter().any(|s| s.contains(kmpc_x86::FORK_CALL)));
assert!(ir.iter().any(|s| s.contains(kmpc_x86::SECTIONS_INIT)));
}
#[test]
fn test_codegen_target_teams() {
let ctx = X86OpenMPFull::new_x86_64();
let mut cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective::new(X86OmpDirectiveKind::TargetTeams);
let ir = cg.lower_target_teams(&dir);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::TARGET_INIT)));
assert!(ir.iter().any(|s| s.contains(kmpc_x86::FORK_TEAMS)));
}
#[test]
fn test_codegen_atomic_cas() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let ir4 = cg.lower_atomic_cas("ptr", "expected", "desired", 4);
assert!(ir4.iter().any(|s| s.contains(kmpc_x86::ATOMIC_CAS_4)));
let ir8 = cg.lower_atomic_cas("ptr", "expected", "desired", 8);
assert!(ir8.iter().any(|s| s.contains(kmpc_x86::ATOMIC_CAS_8)));
}
#[test]
fn test_codegen_atomic_swap() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let ir = cg.lower_atomic_swap("ptr", "val", 4);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::ATOMIC_SWAP_4)));
}
#[test]
fn test_codegen_loop_transform() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let transform = X86LoopTransformClause {
tile_sizes: vec![4, 8],
unroll_factor: Some(4),
unroll_full: false,
interchange_indices: vec![1, 0],
scan_inclusive: false,
scan_exclusive: false,
};
let dir = X86OmpDirective::new(X86OmpDirectiveKind::For);
let ir = cg.lower_loop_transform(&dir, &transform);
assert!(!ir.is_empty());
assert!(ir.iter().any(|s| s.contains("tile")));
}
#[test]
fn test_codegen_requires() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let ir = cg.lower_requires(true, true, false, false, Some("seq_cst"));
assert!(ir.iter().any(|s| s.contains("unified_address")));
assert!(ir.iter().any(|s| s.contains("unified_shared_memory")));
assert!(ir.iter().any(|s| s.contains("seq_cst")));
}
#[test]
fn test_codegen_masked() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let ir = cg.lower_masked(Some(0));
assert!(ir.iter().any(|s| s.contains("masked_body")));
}
#[test]
fn test_codegen_scope() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective::new(X86OmpDirectiveKind::Scope);
let ir = cg.lower_scope(&dir);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::BARRIER)));
}
#[test]
fn test_codegen_nothing() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let ir = cg.lower_nothing();
assert_eq!(ir.len(), 1);
}
#[test]
fn test_codegen_error() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let ir = cg.lower_error("fatal", "test error message");
assert!(ir.iter().any(|s| s.contains("trap")));
}
#[test]
fn test_codegen_metadirective() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let whens = vec![("target={}".to_string(), "variant(1)".to_string())];
let ir = cg.lower_metadirective(&whens, Some("variant(2)"));
assert!(ir.iter().any(|s| s.contains("Metadirective")));
}
#[test]
fn test_codegen_parallel_masked() {
let ctx = X86OpenMPFull::new_x86_64();
let mut cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective::new(X86OmpDirectiveKind::Parallel);
let ir = cg.lower_parallel_masked(&dir, Some(0));
assert!(ir.iter().any(|s| s.contains(kmpc_x86::FORK_CALL)));
}
#[test]
fn test_codegen_x86_barrier_optimized() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let ir = cg.generate_x86_barrier_optimized();
assert!(ir.iter().any(|s| s.contains("pause")));
}
#[test]
fn test_codegen_lower_single_master_functions() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let single_ir = cg.lower_single(&X86OmpDirective::new(X86OmpDirectiveKind::Single));
assert!(single_ir.iter().any(|s| s.contains(kmpc_x86::SINGLE_BEGIN)));
let master_ir = cg.lower_master();
assert!(master_ir.iter().any(|s| s.contains(kmpc_x86::MASTER_BEGIN)));
}
#[test]
fn test_codegen_lower_atomic_all_kinds() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let dir = X86OmpDirective::new(X86OmpDirectiveKind::Atomic);
let read_ir = cg.lower_atomic(&dir, X86OmpAtomicKind::Read);
assert!(read_ir.iter().any(|s| s.contains("ATOMIC_RD")));
let write_ir = cg.lower_atomic(&dir, X86OmpAtomicKind::Write);
assert!(write_ir.iter().any(|s| s.contains("ATOMIC_WR")));
let update_ir = cg.lower_atomic(&dir, X86OmpAtomicKind::Update);
assert!(update_ir.iter().any(|s| s.contains("atomicrmw")));
let compare_ir = cg.lower_atomic(&dir, X86OmpAtomicKind::Compare);
assert!(compare_ir.iter().any(|s| s.contains("cmpxchg")));
}
#[test]
fn test_irgen_collapse_loop() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let loop_info = X86OmpIRLoopInfo {
induction_var: "i".to_string(),
lower_bound: 0,
upper_bound: 10,
stride: 1,
chunk_size: 1,
schedule_id: 1,
is_static: true,
has_nowait: false,
has_ordered: false,
collapse_count: 3,
};
let ir = irgen.generate_collapse_loop_ir(&loop_info, 3, &[10, 10, 10]);
assert!(!ir.is_empty());
assert!(ir.iter().any(|s| s.contains("Collapsed loop")));
assert!(ir.iter().any(|s| s.contains("flat_iter")));
}
#[test]
fn test_irgen_threadprivate() {
let ctx = X86OpenMPFull::new_x86_64();
let irgen = X86OpenMPIRGen::new(ctx);
let ir = irgen.generate_threadprivate_ir("A");
assert!(ir
.iter()
.any(|s| s.contains(kmpc_x86::THREADPRIVATE_REGISTER)));
}
#[test]
fn test_irgen_copyin() {
let ctx = X86OpenMPFull::new_x86_64();
let irgen = X86OpenMPIRGen::new(ctx);
let ir = irgen.generate_copyin_ir(&["A".to_string(), "B".to_string()]);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::MASTER_BEGIN)));
assert!(ir
.iter()
.any(|s| s.contains(kmpc_x86::THREADPRIVATE_CACHED)));
}
#[test]
fn test_irgen_single_copyprivate() {
let ctx = X86OpenMPFull::new_x86_64();
let irgen = X86OpenMPIRGen::new(ctx);
let ir = irgen.generate_single_copyprivate_ir(&["x".to_string()]);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::SINGLE_BEGIN)));
}
#[test]
fn test_irgen_distribute_static() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let loop_info = X86OmpIRLoopInfo {
induction_var: "i".to_string(),
lower_bound: 0,
upper_bound: 100,
stride: 1,
chunk_size: 4,
schedule_id: 33,
is_static: true,
has_nowait: false,
has_ordered: false,
collapse_count: 1,
};
let ir = irgen.generate_distribute_static_ir(&loop_info);
assert!(ir
.iter()
.any(|s| s.contains(kmpc_x86::DISTRIBUTE_STATIC_INIT)));
}
#[test]
fn test_irgen_offloading_entry() {
let ctx = X86OpenMPFull::new_x86_64();
let irgen = X86OpenMPIRGen::new(ctx);
let ir = irgen.generate_offloading_entry_ir("test_entry", "@test_fn", 0, 0);
assert!(ir.iter().any(|s| s.contains(tgt_x86::REGISTER_LIB)));
}
#[test]
fn test_irgen_lastprivate_finalize() {
let ctx = X86OpenMPFull::new_x86_64();
let irgen = X86OpenMPIRGen::new(ctx);
let ir = irgen.generate_lastprivate_finalize("x", "i32");
assert!(ir.iter().any(|s| s.contains("lp.ptr")));
assert!(ir.iter().any(|s| s.contains("_.captured.x")));
}
#[test]
fn test_irgen_linear_init() {
let ctx = X86OpenMPFull::new_x86_64();
let irgen = X86OpenMPIRGen::new(ctx);
let ir = irgen.generate_linear_init("i", "i32", 2);
assert!(ir.iter().any(|s| s.contains("lin.val")));
}
#[test]
fn test_runtime_initialize_environment() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.initialize_from_environment();
}
#[test]
fn test_runtime_simulate_fork_call() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.simulate_fork_call(3, "__omp_test");
assert_eq!(rt.team_count(), 1);
}
#[test]
fn test_runtime_barrier_with_cancel() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.set_cancellation(true);
rt.set_cancel("parallel");
let cancelled = rt.barrier_with_cancel("parallel");
assert!(cancelled);
}
#[test]
fn test_runtime_enter_exit_critical() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
let id = rt.enter_critical("my_lock");
assert!(id < 100);
rt.exit_critical("my_lock");
}
#[test]
fn test_runtime_allocate_submit_task() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
let task_id = rt.allocate_task(0, "__omp_task_fn");
rt.submit_task(task_id);
assert!(rt.dep_graph.ready_queue.contains(&task_id));
}
#[test]
fn test_runtime_task_wait_and_group() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.begin_taskgroup();
rt.wait_tasks();
rt.end_taskgroup();
}
#[test]
fn test_runtime_target_execute() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
let rc = rt.execute_target(0, "binary");
assert_eq!(rc, 0);
rt.begin_target_data(0);
rt.end_target_data(0);
rt.update_target_data(0);
}
#[test]
fn test_runtime_reduction_ops() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.perform_reduction("sum", X86ReductionOp::Add);
assert_eq!(rt.pending_reductions.len(), 1);
rt.complete_reduction("sum");
assert_eq!(rt.pending_reductions.len(), 0);
}
#[test]
fn test_runtime_reduction_all_ops() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
let ops = vec![
X86ReductionOp::Add,
X86ReductionOp::Mul,
X86ReductionOp::Min,
X86ReductionOp::Max,
X86ReductionOp::BAnd,
X86ReductionOp::BOr,
X86ReductionOp::Xor,
X86ReductionOp::LogicalAnd,
X86ReductionOp::LogicalOr,
];
for op in ops {
rt.perform_reduction("var", op);
}
assert_eq!(rt.pending_reductions.len(), 9);
rt.clear_reductions();
assert_eq!(rt.pending_reductions.len(), 0);
}
#[test]
fn test_runtime_flush() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.flush_memory(None);
rt.flush_memory(Some("A"));
}
#[test]
fn test_runtime_cancel_operation() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.set_cancellation(true);
let rc = rt.cancel_operation("loop");
assert_eq!(rc, 1);
assert!(rt.is_cancelled("loop"));
}
#[test]
fn test_runtime_cancellation_point() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.set_cancellation(true);
rt.set_cancel("loop");
let rc = rt.check_cancellation_point("loop");
assert_eq!(rc, 1);
rt.reset_cancel();
let rc2 = rt.check_cancellation_point("loop");
assert_eq!(rc2, 0);
}
#[test]
fn test_runtime_sections() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.init_sections();
let section = rt.next_section();
assert_eq!(section, 1);
}
#[test]
fn test_runtime_proc_bind() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.set_proc_bind(X86ProcBind::Spread);
assert_eq!(rt.context.icv.proc_bind_var, Some(X86ProcBind::Spread));
}
#[test]
fn test_runtime_push_num_threads() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.push_num_threads(8);
assert_eq!(rt.context.icv.nthreads_var, 8);
}
#[test]
fn test_runtime_global_bound_thread_num() {
let ctx = X86OpenMPFull::new_x86_64();
let rt = X86OpenMPRuntime::new(ctx);
let gtid = rt.global_thread_num();
let btid = rt.bound_thread_num();
assert!(gtid >= 0);
assert!(btid >= 0);
}
#[test]
fn test_runtime_taskloop() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
rt.execute_taskloop(0, 100, 0, 4);
rt.execute_taskloop(0, 100, 25, 0);
rt.execute_taskloop(0, 100, 0, 0);
}
#[test]
fn test_runtime_device_management() {
let ctx = X86OpenMPFull::new_x86_64();
let mut rt = X86OpenMPRuntime::new(ctx);
assert_eq!(rt.get_num_devices(), 1);
assert_eq!(rt.get_default_device(), 0);
rt.set_default_device(1);
assert_eq!(rt.get_default_device(), 1);
assert_eq!(rt.is_initial_device(), 1);
assert_eq!(rt.get_initial_device(), 0);
}
#[test]
fn test_runtime_association() {
let ctx = X86OpenMPFull::new_x86_64();
let rt = X86OpenMPRuntime::new(ctx);
rt.associate_ptr("host_ptr", "dev_ptr");
rt.disassociate_ptr("host_ptr");
}
#[test]
fn test_runtime_memcpy_rect() {
let ctx = X86OpenMPFull::new_x86_64();
let rt = X86OpenMPRuntime::new(ctx);
rt.target_memcpy_rect("dst", "src");
}
#[test]
fn test_runtime_alloc_free() {
let ctx = X86OpenMPFull::new_x86_64();
let rt = X86OpenMPRuntime::new(ctx);
let ptr = rt.target_alloc(1024);
rt.target_free(ptr);
}
#[test]
fn test_runtime_offload_lib() {
let ctx = X86OpenMPFull::new_x86_64();
let rt = X86OpenMPRuntime::new(ctx);
rt.register_offload_lib("test_lib");
rt.unregister_offload_lib("test_lib");
}
#[test]
fn test_full_pipeline_parallel_reduction() {
let ctx = X86OpenMPFull::new_x86_64();
let mut parser = X86OmpDirectiveParser::new();
let mut codegen = X86OpenMPCodeGen::new(ctx);
let result = parser.parse_directive("omp parallel reduction(+:sum) num_threads(4)");
assert!(result.is_ok());
let dir = result.unwrap();
assert_eq!(dir.reductions.len(), 1);
let region = codegen.lower_parallel(&dir);
assert_eq!(codegen.parallel_count, 1);
let red = codegen.lower_reduction(X86ReductionOp::Add, "sum", "i32");
let init = codegen.generate_reduction_init_code(&red);
let combiner = codegen.generate_reduction_combiner_code(&red);
let finalize = codegen.generate_reduction_finalize_code(&red);
assert!(!init.is_empty());
assert!(!combiner.is_empty());
assert!(!finalize.is_empty());
}
#[test]
fn test_full_pipeline_sections() {
let ctx = X86OpenMPFull::new_x86_64();
let mut parser = X86OmpDirectiveParser::new();
let mut codegen = X86OpenMPCodeGen::new(ctx);
let result = parser.parse_directive("omp sections");
assert!(result.is_ok());
let dir = result.unwrap();
let ir = codegen.lower_sections(&dir, 4);
assert!(ir.iter().any(|s| s.contains(kmpc_x86::SECTIONS_INIT)));
assert!(ir.iter().any(|s| s.contains(kmpc_x86::NEXT_SECTION)));
}
#[test]
fn test_full_pipeline_taskloop() {
let ctx = X86OpenMPFull::new_x86_64();
let mut parser = X86OmpDirectiveParser::new();
let mut codegen = X86OpenMPCodeGen::new(ctx);
let result = parser.parse_directive("omp taskloop grainsize(64) num_tasks(8)");
assert!(result.is_ok());
let dir = result.unwrap();
let taskloop = codegen.lower_taskloop(&dir);
assert!(taskloop.grainsize == Some(64));
assert!(taskloop.num_tasks == Some(8));
}
#[test]
fn test_all_schedule_kinds_generate_code() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
for schedule in &[
X86OmpScheduleKind::Static,
X86OmpScheduleKind::Dynamic,
X86OmpScheduleKind::Guided,
X86OmpScheduleKind::Auto,
X86OmpScheduleKind::Runtime,
] {
let loop_info = X86OmpLoopLowering {
directive: X86OmpDirective::new(X86OmpDirectiveKind::For),
loop_var: "i".to_string(),
lower_bound: "0".to_string(),
upper_bound: "100".to_string(),
stride: "1".to_string(),
chunk_size: 4,
schedule: *schedule,
ordered_clause: X86OrderedKind::None,
collapse_depth: 1,
simd_clause: X86SimdClause::default(),
nowait: false,
body_label: ".body".to_string(),
epilogue_label: ".epilogue".to_string(),
};
let ir = cg.generate_schedule_code(&loop_info);
assert!(!ir.is_empty(), "Schedule {:?} generated empty IR", schedule);
}
}
#[test]
fn test_all_irgen_loop_schedules() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
for schedule_id in 1..=5 {
let loop_info = X86OmpIRLoopInfo {
induction_var: "i".to_string(),
lower_bound: 0,
upper_bound: 100,
stride: 1,
chunk_size: 4,
schedule_id,
is_static: schedule_id == 1,
has_nowait: false,
has_ordered: false,
collapse_count: 1,
};
let ir = irgen.generate_loop_ir(&loop_info);
assert!(
!ir.is_empty(),
"Schedule {} generated empty IR",
schedule_id
);
}
}
#[test]
fn test_captured_variable_all_attributes() {
let mut dir = X86OmpDirective::new(X86OmpDirectiveKind::Parallel);
dir.add_shared("A".to_string());
dir.add_private("i".to_string());
dir.firstprivate_vars.push("x".to_string());
dir.lastprivate_vars.push("y".to_string());
dir.add_reduction("sum".to_string(), X86ReductionOp::Add);
dir.copyin_vars.push("tp_var".to_string());
let ctx = X86OpenMPFull::new_x86_64();
let cg = X86OpenMPCodeGen::new(ctx);
let vars = cg.collect_captured_variables(&dir);
assert!(vars
.iter()
.any(|v| v.name == "A" && matches!(v.sharing, X86DataSharingAttr::Shared)));
assert!(vars
.iter()
.any(|v| v.name == "i" && matches!(v.sharing, X86DataSharingAttr::Private)));
assert!(vars
.iter()
.any(|v| v.name == "x" && matches!(v.sharing, X86DataSharingAttr::FirstPrivate)));
assert!(vars
.iter()
.any(|v| v.name == "y" && matches!(v.sharing, X86DataSharingAttr::LastPrivate)));
assert!(vars
.iter()
.any(|v| v.name == "sum" && matches!(v.sharing, X86DataSharingAttr::Reduction(_))));
assert!(vars
.iter()
.any(|v| v.name == "tp_var" && matches!(v.sharing, X86DataSharingAttr::CopyIn)));
}
#[test]
fn test_all_directive_kinds_have_name() {
let kinds = vec![
X86OmpDirectiveKind::Parallel,
X86OmpDirectiveKind::ParallelFor,
X86OmpDirectiveKind::ParallelSections,
X86OmpDirectiveKind::For,
X86OmpDirectiveKind::Sections,
X86OmpDirectiveKind::Single,
X86OmpDirectiveKind::Master,
X86OmpDirectiveKind::Critical,
X86OmpDirectiveKind::Atomic,
X86OmpDirectiveKind::Barrier,
X86OmpDirectiveKind::Task,
X86OmpDirectiveKind::TaskWait,
X86OmpDirectiveKind::TaskGroup,
X86OmpDirectiveKind::TaskLoop,
X86OmpDirectiveKind::Simd,
X86OmpDirectiveKind::Target,
X86OmpDirectiveKind::TargetData,
X86OmpDirectiveKind::TargetUpdate,
X86OmpDirectiveKind::TargetEnterData,
X86OmpDirectiveKind::TargetExitData,
X86OmpDirectiveKind::Teams,
X86OmpDirectiveKind::Distribute,
X86OmpDirectiveKind::Ordered,
X86OmpDirectiveKind::Cancel,
X86OmpDirectiveKind::CancellationPoint,
X86OmpDirectiveKind::Flush,
X86OmpDirectiveKind::ThreadPrivate,
];
for kind in kinds {
let name = kind.name();
assert!(!name.is_empty(), "Kind {:?} has empty name", kind);
}
}
#[test]
fn test_reduction_neutral_values_match() {
assert_eq!(X86ReductionOp::Add.neutral_value(), Some(0));
assert_eq!(X86ReductionOp::Mul.neutral_value(), Some(1));
assert_eq!(X86ReductionOp::Min.neutral_value(), Some(i64::MAX));
assert_eq!(X86ReductionOp::Max.neutral_value(), Some(i64::MIN));
assert_eq!(X86ReductionOp::BAnd.neutral_value(), Some(-1));
assert_eq!(X86ReductionOp::BOr.neutral_value(), Some(0));
assert_eq!(X86ReductionOp::Xor.neutral_value(), Some(0));
assert_eq!(X86ReductionOp::LogicalAnd.neutral_value(), Some(1));
assert_eq!(X86ReductionOp::LogicalOr.neutral_value(), Some(0));
}
#[test]
fn test_simd_level_from_features_all_combinations() {
assert_eq!(
X86SimdLevel::from_features(true, true, true),
X86SimdLevel::AVX512F
);
assert_eq!(
X86SimdLevel::from_features(false, true, true),
X86SimdLevel::AVX2
);
assert_eq!(
X86SimdLevel::from_features(false, false, true),
X86SimdLevel::SSE2
);
assert_eq!(
X86SimdLevel::from_features(false, false, false),
X86SimdLevel::None
);
}
#[test]
fn test_codegen_default_initialization() {
let cg = X86OpenMPCodeGen::default();
assert_eq!(cg.parallel_count, 0);
assert_eq!(cg.loop_count, 0);
assert_eq!(cg.simd_count, 0);
assert_eq!(cg.task_count, 0);
assert_eq!(cg.target_count, 0);
assert_eq!(cg.section_count, 0);
assert_eq!(cg.verbose, false);
let irgen = X86OpenMPIRGen::default();
assert_eq!(irgen.outlined_functions.len(), 0);
assert_eq!(irgen.simd_regions.len(), 0);
assert_eq!(irgen.label_counter, 0);
assert_eq!(irgen.verbose, false);
let rt = X86OpenMPRuntime::default();
assert!(rt.function_count() > 0);
assert_eq!(rt.thread_teams.len(), 0);
assert_eq!(rt.verbose, false);
}
#[test]
fn test_icv_storage_environment() {
let icv = X86ICVStorage::from_environment();
assert!(icv.nthreads_var >= 0);
}
#[test]
fn test_icv_storage_inherit() {
let parent = X86ICVStorage::new();
let child = X86ICVStorage::inherit(&parent);
assert_eq!(child.nthreads_var, parent.nthreads_var);
assert_eq!(child.dyn_var, parent.dyn_var);
}
#[test]
fn test_outlined_func_generation_full() {
let ctx = X86OpenMPFull::new_x86_64();
let mut irgen = X86OpenMPIRGen::new(ctx);
let captured = vec![
X86CapturedVariable {
name: "A".to_string(),
sharing: X86DataSharingAttr::Shared,
var_type: "i64".to_string(),
is_pointer: true,
reduction_op: None,
linear_step: Some(0),
},
X86CapturedVariable {
name: "i".to_string(),
sharing: X86DataSharingAttr::Private,
var_type: "i32".to_string(),
is_pointer: false,
reduction_op: None,
linear_step: Some(0),
},
X86CapturedVariable {
name: "x".to_string(),
sharing: X86DataSharingAttr::FirstPrivate,
var_type: "i64".to_string(),
is_pointer: false,
reduction_op: None,
linear_step: Some(0),
},
X86CapturedVariable {
name: "y".to_string(),
sharing: X86DataSharingAttr::LastPrivate,
var_type: "i32".to_string(),
is_pointer: false,
reduction_op: None,
linear_step: Some(0),
},
X86CapturedVariable {
name: "sum".to_string(),
sharing: X86DataSharingAttr::Reduction(X86ReductionOp::Add),
var_type: "i64".to_string(),
is_pointer: false,
reduction_op: Some(X86ReductionOp::Add),
linear_step: Some(0),
},
];
let outlined =
irgen.generate_outlined_parallel_function("__omp_test_full", &captured, true, &[]);
assert_eq!(outlined.shared_vars.len(), 5);
assert!(outlined.has_reduction);
let sig = irgen.emit_outlined_fn_signature(&outlined);
assert!(sig.iter().any(|s| s.contains("__omp_test_full")));
assert!(sig.iter().any(|s| s.contains("%.global_tid.")));
let epilogue = irgen.emit_outlined_fn_epilogue(&outlined);
assert!(epilogue.iter().any(|s| s.contains("ret void")));
}
}
pub struct X86OmpUtils;
impl X86OmpUtils {
pub fn optimal_chunk_size(trip_count: u64, num_threads: u32) -> u64 {
if num_threads == 0 {
return trip_count;
}
let base_chunk = trip_count / num_threads as u64;
if base_chunk == 0 {
1
} else {
let cache_line_elements = X86_CACHE_LINE_SIZE as u64 / 4; if base_chunk > cache_line_elements {
(base_chunk / cache_line_elements) * cache_line_elements
} else {
base_chunk.max(1)
}
}
}
pub fn should_vectorize(trip_count: u64, simd_level: X86SimdLevel, is_aligned: bool) -> bool {
if simd_level == X86SimdLevel::None {
return false;
}
let min_trip = simd_level.vector_width_bits() as u64 / 32; trip_count >= min_trip && (is_aligned || simd_level.supports_masked_operations())
}
pub fn select_barrier_strategy(num_threads: u32) -> &'static str {
match num_threads {
0..=1 => "none",
2..=4 => "simple_centralized",
5..=16 => "tree_barrier",
17..=64 => "dissemination_barrier",
_ => "hierarchical_barrier",
}
}
pub fn pause_spin_threshold(num_threads: u32) -> u64 {
match num_threads {
0..=1 => 0,
2..=8 => X86_PAUSE_HINT as u64,
_ => X86_PAUSE_HINT as u64 * 2,
}
}
pub fn worker_stack_size(is_master: bool, icv: &X86ICVStorage) -> usize {
let base = 1024 * 1024; if is_master {
base * 8 } else if icv.stacksize_var > 0 {
icv.stacksize_var as usize
} else {
base * 4 }
}
pub fn offloading_supported(simd_level: X86SimdLevel) -> bool {
matches!(
simd_level,
X86SimdLevel::AVX | X86SimdLevel::AVX2 | X86SimdLevel::AVX512F
)
}
pub fn cache_line_padding() -> usize {
X86_CACHE_LINE_SIZE as usize
}
pub fn max_supported_threads() -> u32 {
X86_OMP_MAX_THREADS
}
pub fn incompatible_features_warning(feature1: &str, feature2: &str) -> String {
format!(
"OpenMP warning: '{}' and '{}' may have reduced performance when combined",
feature1, feature2
)
}
}
impl X86OpenMPFull {
pub fn create_codegen(&self) -> X86OpenMPCodeGen {
X86OpenMPCodeGen::new(self.clone())
}
pub fn create_irgen(&self) -> X86OpenMPIRGen {
X86OpenMPIRGen::new(self.clone())
}
pub fn create_runtime(&self) -> X86OpenMPRuntime {
X86OpenMPRuntime::new(self.clone())
}
fn clone(&self) -> Self {
Self {
target_triple: self.target_triple.clone(),
simd_level: self.simd_level,
omp_version: self.omp_version,
directive_count: self.directive_count,
codegen_count: self.codegen_count,
simd_count: self.simd_count,
offload_count: self.offload_count,
icv: X86ICVStorage::inherit(&self.icv),
runtime: OpenMPRuntime::default(),
offload_engine: None,
parallel_contexts: self.parallel_contexts.clone(),
declare_target_fns: self.declare_target_fns.clone(),
declare_simd_fns: self.declare_simd_fns.clone(),
reduction_registry: self.reduction_registry.clone(),
critical_names: self.critical_names.clone(),
x86_optimizations: self.x86_optimizations,
verbose: self.verbose,
}
}
}
#[cfg(test)]
mod final_integration_tests {
use super::*;
#[test]
fn test_e2e_pipeline_complex_directive() {
let ctx = X86OpenMPFull::new_x86_64_avx512();
let mut parser = X86OmpDirectiveParser::new();
let mut codegen = ctx.create_codegen();
let mut irgen = ctx.create_irgen();
let mut runtime = ctx.create_runtime();
let result = parser.parse_directive(
"omp parallel for private(i) shared(A,B) reduction(+:sum) schedule(static,8) num_threads(4)"
);
assert!(result.is_ok());
let dir = result.unwrap();
let (region, loop_info) = codegen.lower_parallel_for(&dir);
assert_eq!(codegen.parallel_count, 1);
assert_eq!(codegen.loop_count, 1);
let fork_call = codegen.generate_fork_call(®ion, ®ion.outlined_fn_name);
assert!(!fork_call.is_empty());
let schedule_code = codegen.generate_schedule_code(&loop_info);
assert!(!schedule_code.is_empty());
let red = codegen.lower_reduction(X86ReductionOp::Add, "sum", "f64");
let red_code = codegen.generate_reduction_init_code(&red);
assert!(!red_code.is_empty());
let dirs = vec![dir];
let ir = irgen.compile_directives(&dirs);
assert!(!ir.is_empty());
runtime.initialize_from_environment();
runtime.push_num_threads(4);
runtime.simulate_fork_call(2, ®ion.outlined_fn_name);
runtime.perform_reduction("sum", X86ReductionOp::Add);
runtime.complete_reduction("sum");
let report = runtime.status_report();
assert!(report.contains("X86OpenMPRuntime"));
}
#[test]
fn test_x86_omp_utils() {
assert_eq!(X86OmpUtils::optimal_chunk_size(100, 4), 24);
assert!(X86OmpUtils::should_vectorize(
16,
X86SimdLevel::AVX512F,
true
));
assert!(!X86OmpUtils::should_vectorize(1, X86SimdLevel::SSE2, true));
assert_eq!(
X86OmpUtils::select_barrier_strategy(2),
"simple_centralized"
);
assert_eq!(
X86OmpUtils::select_barrier_strategy(32),
"dissemination_barrier"
);
assert!(X86OmpUtils::offloading_supported(X86SimdLevel::AVX512F));
assert!(!X86OmpUtils::offloading_supported(X86SimdLevel::SSE2));
assert_eq!(X86OmpUtils::cache_line_padding(), 64);
assert_eq!(X86OmpUtils::max_supported_threads(), X86_OMP_MAX_THREADS);
}
#[test]
fn test_context_create_pipelines() {
let ctx = X86OpenMPFull::new_x86_64();
let cg = ctx.create_codegen();
assert_eq!(cg.parallel_count, 0);
let irgen = ctx.create_irgen();
assert_eq!(irgen.outlined_functions.len(), 0);
let rt = ctx.create_runtime();
assert!(rt.function_count() > 0);
}
#[test]
fn test_runtime_all_signatures_valid() {
let ctx = X86OpenMPFull::new_x86_64();
let rt = X86OpenMPRuntime::new(ctx);
let required = vec![
kmpc_x86::FORK_CALL,
kmpc_x86::FORK_TEAMS,
kmpc_x86::PUSH_NUM_THREADS,
kmpc_x86::PUSH_PROC_BIND,
kmpc_x86::GLOBAL_THREAD_NUM,
kmpc_x86::BOUND_THREAD_NUM,
kmpc_x86::FOR_STATIC_INIT_4,
kmpc_x86::FOR_STATIC_INIT_8,
kmpc_x86::FOR_STATIC_FINI,
kmpc_x86::DISPATCH_INIT_4,
kmpc_x86::DISPATCH_INIT_8,
kmpc_x86::DISPATCH_NEXT_4,
kmpc_x86::DISPATCH_NEXT_8,
kmpc_x86::DISPATCH_FINI_4,
kmpc_x86::DISPATCH_FINI_8,
kmpc_x86::BARRIER,
kmpc_x86::CANCEL_BARRIER,
kmpc_x86::CRITICAL,
kmpc_x86::END_CRITICAL,
kmpc_x86::MASTER_BEGIN,
kmpc_x86::MASTER_END,
kmpc_x86::SINGLE_BEGIN,
kmpc_x86::SINGLE_END,
kmpc_x86::ORDERED_BEGIN,
kmpc_x86::ORDERED_END,
kmpc_x86::ATOMIC_RD_4,
kmpc_x86::ATOMIC_WR_4,
kmpc_x86::ATOMIC_RD_8,
kmpc_x86::ATOMIC_WR_8,
kmpc_x86::ATOMIC_RD_16,
kmpc_x86::ATOMIC_WR_16,
kmpc_x86::ATOMIC_FLOAT_RD,
kmpc_x86::ATOMIC_FLOAT_WR,
kmpc_x86::ATOMIC_DOUBLE_RD,
kmpc_x86::ATOMIC_DOUBLE_WR,
kmpc_x86::FLUSH,
kmpc_x86::CANCEL,
kmpc_x86::CANCELLATION_POINT,
kmpc_x86::THREADPRIVATE_REGISTER,
kmpc_x86::THREADPRIVATE_CACHED,
kmpc_x86::TASK_ALLOC,
kmpc_x86::TASK,
kmpc_x86::TASK_WITH_DEPS,
kmpc_x86::TASKWAIT,
kmpc_x86::TASKYIELD,
kmpc_x86::TASKGROUP,
kmpc_x86::TASKGROUP_END,
kmpc_x86::TASKLOOP,
kmpc_x86::TARGET_INIT,
kmpc_x86::TARGET_DEINIT,
kmpc_x86::TARGET_DATA_BEGIN,
kmpc_x86::TARGET_DATA_END,
kmpc_x86::TARGET,
kmpc_x86::TARGET_UPDATE,
kmpc_x86::TARGET_MEMCPY,
kmpc_x86::REDUCE,
kmpc_x86::END_REDUCE,
kmpc_x86::SIMD,
kmpc_x86::SECTIONS_INIT,
kmpc_x86::NEXT_SECTION,
kmpc_x86::DISTRIBUTE_STATIC_INIT,
kmpc_x86::DISTRIBUTE_STATIC_FINI,
kmpc_x86::ALLOC_SHARED,
kmpc_x86::FREE_SHARED,
];
for name in required {
assert!(rt.has_function(name), "Missing runtime function: {}", name);
}
}
#[test]
fn test_tgt_all_function_names_defined() {
let names = vec![
tgt_x86::REGISTER_LIB,
tgt_x86::UNREGISTER_LIB,
tgt_x86::TARGET,
tgt_x86::TARGET_NOWAIT,
tgt_x86::TARGET_TEAMS,
tgt_x86::TARGET_TEAMS_NOWAIT,
tgt_x86::TARGET_DATA_BEGIN,
tgt_x86::TARGET_DATA_END,
tgt_x86::TARGET_DATA_UPDATE,
tgt_x86::TARGET_ALLOC,
tgt_x86::TARGET_FREE,
tgt_x86::TARGET_MEMCPY,
tgt_x86::TARGET_MEMCPY_RECT,
tgt_x86::TARGET_ASSOCIATE_PTR,
tgt_x86::TARGET_DISASSOCIATE_PTR,
tgt_x86::IS_INITIAL_DEVICE,
tgt_x86::GET_INITIAL_DEVICE,
tgt_x86::GET_NUM_DEVICES,
tgt_x86::GET_DEFAULT_DEVICE,
tgt_x86::SET_DEFAULT_DEVICE,
];
for name in names {
assert!(!name.is_empty(), "Empty target function name");
}
}
}